Add automatic checks

.flake8

+[flake8]
+ignore = W605,W503,W504,H306,H238,H301,H202
+max-line-length = 120
+per-file-ignores =
+    test/test_packages.py:E501

Makefile

@@ -80,9 +80,14 @@ lint/%: ARGS?=
 lint/%: ## lint the dockerfile(s) for a stack
 	@echo "Linting Dockerfiles in $(notdir $@)..."
 	@git ls-files --exclude='Dockerfile*' --ignored $(notdir $@) | grep -v ppc64 | xargs -L 1 $(HADOLINT) $(ARGS)
-	@echo "Linting done!"
+	@echo "Linting Dockerfiles done!"
 
-lint-all: $(foreach I,$(ALL_IMAGES),lint/$(I) ) ## lint all stacks
+flake8-lint:
+	@echo "Linting Python source files..."
+	@git ls-files '**.py' | xargs -L 1 flake8
+	@echo "Linting Python source files done!"
+
+lint-all: flake8-lint $(foreach I,$(ALL_IMAGES),lint/$(I) ) ## lint all stacks
 
 lint-build-test-all: $(foreach I,$(ALL_IMAGES),lint/$(I) arch_patch/$(I) build/$(I) test/$(I) ) ## lint, build and test all stacks
 
@@ -91,7 +96,7 @@ lint-install: ## install hadolint
 	@curl -sL -o $(HADOLINT) "https://github.com/hadolint/hadolint/releases/download/v1.18.0/hadolint-$(shell uname -s)-$(shell uname -m)"
 	@chmod 700 $(HADOLINT)
 	@echo "Installation done!"
-	@$(HADOLINT) --version	
+	@$(HADOLINT) --version
 
 img-clean: img-rm-dang img-rm ## clean dangling and jupyter images
 

all-spark-notebook/test/test_spark_notebooks.py

@@ -21,7 +21,10 @@ def test_nbconvert(container, test_file):
     output_dir = "/tmp"
     timeout_ms = 600
     LOGGER.info(f"Test that {test_file} notebook can be executed ...")
-    command = f"jupyter nbconvert --to markdown --ExecutePreprocessor.timeout={timeout_ms} --output-dir {output_dir} --execute {cont_data_dir}/{test_file}.ipynb"
+    command = "jupyter nbconvert --to markdown " + \
+        f"--ExecutePreprocessor.timeout={timeout_ms}" + \
+        f"--output-dir {output_dir}" +\
+        f"--execute {cont_data_dir}/{test_file}.ipynb"
     c = container.run(
         volumes={host_data_dir: {"bind": cont_data_dir, "mode": "ro"}},
         tty=True,

requirements-dev.txt

 docker
+flake8
 pytest
 recommonmark==0.5.0
 requests
@@ -6,4 +7,3 @@ sphinx>=1.6
 sphinx-intl
 tabulate
 transifex-client
-

scipy-notebook/test/test_pandas.py

@@ -8,17 +8,23 @@ LOGGER = logging.getLogger(__name__)
 
 
 @pytest.mark.parametrize(
-    "name,command",
+    "name,command_list",
     [
         (
             "Sum series",
-            "import pandas as pd; import numpy as np; np.random.seed(0); print(pd.Series(np.random.randint(0, 7, size=10)).sum())",
+            [
+                "import pandas as pd",
+                "import numpy as np",
+                "np.random.seed(0)",
+                "print(pd.Series(np.random.randint(0, 7, size=10)).sum())"
+            ]
         ),
     ],
 )
-def test_pandas(container, name, command):
+def test_pandas(container, name, command_list):
     """Basic pandas tests"""
     LOGGER.info(f"Testing pandas: {name} ...")
+    command = ';'.join(command_list)
     c = container.run(tty=True, command=["start.sh", "python", "-c", command])
     rv = c.wait(timeout=30)
     assert rv == 0 or rv["StatusCode"] == 0, f"Command {command} failed"