Merge pull request #1100 from romainx/hadolint

Docker linting -> Hadolint

.hadolint.yaml

+ignored:
+    - DL3006
+    - DL3008

.travis.yml

@@ -12,10 +12,11 @@ jobs:
       install:
       - pip install --upgrade pip
       - make dev-env
+      - make lint-install
       script:
       - set -e
       - if [ $(make n-docs-diff) -ne 0 ]; then make docs; fi;
-      - if [ $(make n-other-diff) -ne 0 ]; then make build-test-all DARGS="--build-arg TEST_ONLY_BUILD=1"; fi;
+      - if [ $(make n-other-diff) -ne 0 ]; then make lint-build-test-all DARGS="--build-arg TEST_ONLY_BUILD=1"; fi;
     - stage: push-tx
       install:
       - pip install --upgrade pip
@@ -26,10 +27,11 @@ jobs:
       install:
       - pip install --upgrade pip
       - make dev-env
+      - make lint-install
       script:
       - set -e
       - make docs
-      - make build-test-all DARGS="--build-arg TEST_ONLY_BUILD=1"
+      - make lint-build-test-all DARGS="--build-arg TEST_ONLY_BUILD=1"
 
 stages:
     - name: diff-test

Makefile

@@ -24,6 +24,9 @@ endif
 
 ALL_IMAGES:=$(ALL_STACKS)
 
+# Linter
+HADOLINT="${HOME}/hadolint"
+
 help:
 # http://marmelab.com/blog/2016/02/29/auto-documented-makefile.html
 	@echo "jupyter/docker-stacks"
@@ -73,6 +76,23 @@ dev/%: ## run a foreground container for a stack
 dev-env: ## install libraries required to build docs and run tests
 	pip install -r requirements-dev.txt
 
+lint/%: ARGS?=
+lint/%: ## lint the dockerfile(s) for a stack
+	@echo "Linting Dockerfiles in $(notdir $@)..."
+	@git ls-files --exclude='Dockerfile*' --ignored $(notdir $@) | grep -v ppc64 | xargs -L 1 $(HADOLINT) $(ARGS)
+	@echo "Linting done!"
+
+lint-all: $(foreach I,$(ALL_IMAGES),lint/$(I) ) ## lint all stacks
+
+lint-build-test-all: $(foreach I,$(ALL_IMAGES),lint/$(I) arch_patch/$(I) build/$(I) test/$(I) ) ## lint, build and test all stacks
+
+lint-install: ## install hadolint
+	@echo "Installing hadolint at $(HADOLINT) ..."
+	@curl -sL -o $(HADOLINT) "https://github.com/hadolint/hadolint/releases/download/v1.17.6/hadolint-$(shell uname -s)-$(shell uname -m)"
+	@chmod 700 $(HADOLINT)
+	@echo "Installation done!"
+	@$(HADOLINT) --version	
+
 img-clean: img-rm-dang img-rm ## clean dangling and jupyter images
 
 img-list: ## list jupyter images

all-spark-notebook/Dockerfile

@@ -30,22 +30,23 @@ RUN conda install --quiet --yes \
     'r-sparklyr=1.2*' \
     && \
     conda clean --all -f -y && \
-    fix-permissions $CONDA_DIR && \
-    fix-permissions /home/$NB_USER
+    fix-permissions "${CONDA_DIR}" && \
+    fix-permissions "/home/${NB_USER}"
 
 # Apache Toree kernel
+# hadolint ignore=DL3013
 RUN pip install --no-cache-dir \
     https://dist.apache.org/repos/dist/release/incubator/toree/0.3.0-incubating/toree-pip/toree-0.3.0.tar.gz \
     && \
     jupyter toree install --sys-prefix && \
-    rm -rf /home/$NB_USER/.local && \
-    fix-permissions $CONDA_DIR && \
-    fix-permissions /home/$NB_USER
+    rm -rf "/home/${NB_USER}/.local" && \
+    fix-permissions "${CONDA_DIR}" && \
+    fix-permissions "/home/${NB_USER}"
 
 # Spylon-kernel
 RUN conda install --quiet --yes 'spylon-kernel=0.4*' && \
     conda clean --all -f -y && \
     python -m spylon_kernel install --sys-prefix && \
-    rm -rf /home/$NB_USER/.local && \
-    fix-permissions $CONDA_DIR && \
-    fix-permissions /home/$NB_USER
+    rm -rf "/home/${NB_USER}/.local" && \
+    fix-permissions "${CONDA_DIR}" && \
+    fix-permissions "/home/${NB_USER}"

base-notebook/Dockerfile

@@ -13,6 +13,9 @@ ARG NB_USER="jovyan"
 ARG NB_UID="1000"
 ARG NB_GID="100"
 
+# Fix DL4006
+SHELL ["/bin/bash", "-o", "pipefail", "-c"]
+
 USER root
 
 # Install all OS dependencies for notebook server that starts but lacks all
@@ -76,8 +79,8 @@ ENV MINICONDA_VERSION=4.8.2 \
     MINICONDA_MD5=87e77f097f6ebb5127c77662dfc3165e \
     CONDA_VERSION=4.8.2
 
-RUN cd /tmp && \
-    wget --quiet https://repo.continuum.io/miniconda/Miniconda3-py37_${MINICONDA_VERSION}-Linux-x86_64.sh && \
+WORKDIR /tmp
+RUN wget --quiet https://repo.continuum.io/miniconda/Miniconda3-py37_${MINICONDA_VERSION}-Linux-x86_64.sh && \
     echo "${MINICONDA_MD5} *Miniconda3-py37_${MINICONDA_VERSION}-Linux-x86_64.sh" | md5sum -c - && \
     /bin/bash Miniconda3-py37_${MINICONDA_VERSION}-Linux-x86_64.sh -f -b -p $CONDA_DIR && \
     rm Miniconda3-py37_${MINICONDA_VERSION}-Linux-x86_64.sh && \
@@ -137,3 +140,5 @@ RUN fix-permissions /etc/jupyter/
 
 # Switch back to jovyan to avoid accidental container runs as root
 USER $NB_UID
+
+WORKDIR $HOME

datascience-notebook/Dockerfile

@@ -9,6 +9,9 @@ LABEL maintainer="Jupyter Project <jupyter@googlegroups.com>"
 # be skipped to shorten build time.
 ARG TEST_ONLY_BUILD
 
+# Fix DL4006
+SHELL ["/bin/bash", "-o", "pipefail", "-c"]
+
 USER root
 
 # R pre-requisites
@@ -25,21 +28,23 @@ ENV JULIA_DEPOT_PATH=/opt/julia
 ENV JULIA_PKGDIR=/opt/julia
 ENV JULIA_VERSION=1.4.1
 
-RUN mkdir /opt/julia-${JULIA_VERSION} && \
-    cd /tmp && \
-    wget -q https://julialang-s3.julialang.org/bin/linux/x64/`echo ${JULIA_VERSION} | cut -d. -f 1,2`/julia-${JULIA_VERSION}-linux-x86_64.tar.gz && \
+WORKDIR /tmp
+
+# hadolint ignore=SC2046
+RUN mkdir "/opt/julia-${JULIA_VERSION}" && \
+    wget -q https://julialang-s3.julialang.org/bin/linux/x64/$(echo "${JULIA_VERSION}" | cut -d. -f 1,2)"/julia-${JULIA_VERSION}-linux-x86_64.tar.gz" && \
     echo "fd6d8cadaed678174c3caefb92207a3b0e8da9f926af6703fb4d1e4e4f50610a *julia-${JULIA_VERSION}-linux-x86_64.tar.gz" | sha256sum -c - && \
-    tar xzf julia-${JULIA_VERSION}-linux-x86_64.tar.gz -C /opt/julia-${JULIA_VERSION} --strip-components=1 && \
-    rm /tmp/julia-${JULIA_VERSION}-linux-x86_64.tar.gz
+    tar xzf "julia-${JULIA_VERSION}-linux-x86_64.tar.gz" -C "/opt/julia-${JULIA_VERSION}" --strip-components=1 && \
+    rm "/tmp/julia-${JULIA_VERSION}-linux-x86_64.tar.gz"
 RUN ln -fs /opt/julia-*/bin/julia /usr/local/bin/julia
 
 # Show Julia where conda libraries are \
 RUN mkdir /etc/julia && \
     echo "push!(Libdl.DL_LOAD_PATH, \"$CONDA_DIR/lib\")" >> /etc/julia/juliarc.jl && \
     # Create JULIA_PKGDIR \
-    mkdir $JULIA_PKGDIR && \
-    chown $NB_USER $JULIA_PKGDIR && \
-    fix-permissions $JULIA_PKGDIR
+    mkdir "${JULIA_PKGDIR}" && \
+    chown "${NB_USER}" "${JULIA_PKGDIR}" && \
+    fix-permissions "${JULIA_PKGDIR}"
 
 USER $NB_UID
 
@@ -66,8 +71,8 @@ RUN conda install --quiet --yes \
     'rpy2=3.1*' \
     && \
     conda clean --all -f -y && \
-    fix-permissions $CONDA_DIR && \
-    fix-permissions /home/$NB_USER
+    fix-permissions "${CONDA_DIR}" && \
+    fix-permissions "/home/${NB_USER}"
 
 # Add Julia packages. Only add HDF5 if this is not a test-only build since
 # it takes roughly half the entire build time of all of the images on Travis
@@ -80,7 +85,9 @@ RUN julia -e 'import Pkg; Pkg.update()' && \
     (test $TEST_ONLY_BUILD || julia -e 'import Pkg; Pkg.add("HDF5")') && \
     julia -e "using Pkg; pkg\"add IJulia\"; pkg\"precompile\"" && \
     # move kernelspec out of home \
-    mv $HOME/.local/share/jupyter/kernels/julia* $CONDA_DIR/share/jupyter/kernels/ && \
-    chmod -R go+rx $CONDA_DIR/share/jupyter && \
-    rm -rf $HOME/.local && \
-    fix-permissions $JULIA_PKGDIR $CONDA_DIR/share/jupyter
+    mv "${HOME}/.local/share/jupyter/kernels/julia"* "${CONDA_DIR}/share/jupyter/kernels/" && \
+    chmod -R go+rx "${CONDA_DIR}/share/jupyter" && \
+    rm -rf "${HOME}/.local" && \
+    fix-permissions "${JULIA_PKGDIR}" "${CONDA_DIR}/share/jupyter"
+
+WORKDIR $HOME

docs/contributing/lint.md

+# Image Lint
+
+To comply with [Docker best practices][dbp], we are using the [Hadolint][hadolint] tool to analyse each `Dockerfile` .
+
+## Installation
+
+There is a specific `make` target to install the linter.
+By default `hadolint` will be installed in `${HOME}/hadolint`.
+
+```bash
+$ make lint-install
+
+# Installing hadolint at /Users/romain/hadolint ...
+# Installation done!
+# Haskell Dockerfile Linter v1.17.6-0-gc918759
+```
+
+## Lint
+
+### Per Stack
+
+The linter can be run per stack.
+
+```bash
+$ make lint/scipy-notebook  
+
+# Linting Dockerfiles in scipy-notebook...
+# scipy-notebook/Dockerfile:4 DL3006 Always tag the version of an image explicitly
+# scipy-notebook/Dockerfile:11 DL3008 Pin versions in apt get install. Instead of `apt-get install <package>` use `apt-get install <package>=<version>`
+# scipy-notebook/Dockerfile:18 SC2086 Double quote to prevent globbing and word splitting.
+# scipy-notebook/Dockerfile:68 SC2086 Double quote to prevent globbing and word splitting.
+# scipy-notebook/Dockerfile:68 DL3003 Use WORKDIR to switch to a directory
+# scipy-notebook/Dockerfile:79 SC2086 Double quote to prevent globbing and word splitting.
+# make: *** [lint/scipy-notebook] Error 1
+```
+
+Optionally you can pass arguments to the linter.
+
+```bash
+# Use a different export format
+$ make lint/scipy-notebook ARGS="--format codeclimate"  
+```
+
+### All the Stacks
+
+The linter can be run against all the stacks.
+
+```bash
+$ make lint-all
+```
+
+## Ignoring Rules
+
+Sometimes it is necessary to ignore [some rules][rules]. 
+The following rules are ignored by default and sor for all images in the `.hadolint.yaml` file.
+
+- [`DL3006`][DL3006]: We use a specific policy to manage image tags. 
+  - `base-notebook` `FROM` clause is fixed but based on an argument (`ARG`). 
+  - Building downstream images from (`FROM`) the latest is done on purpose.
+- [`DL3008`][DL3008]: System packages are always updated (`apt-get`) to the latest version.
+
+For other rules, the preferred way to do it is to flag ignored rules in the `Dockerfile`.
+
+> It is also possible to ignore rules by using a special comment directly above the Dockerfile instruction you want to make an exception for. Ignore rule comments look like `# hadolint ignore=DL3001,SC1081`. For example:
+
+```dockerfile
+
+FROM ubuntu
+
+# hadolint ignore=DL3003,SC1035
+RUN cd /tmp && echo "hello!"
+```
+
+[hadolint]: https://github.com/hadolint/hadolint
+[dbp]: https://docs.docker.com/develop/develop-images/dockerfile_best-practices
+[rules]: https://github.com/hadolint/hadolint#rules
+[DL3006]: https://github.com/hadolint/hadolint/wiki/DL3006
+[DL3008]: https://github.com/hadolint/hadolint/wiki/DL3008
\ No newline at end of file

docs/index.rst

@@ -47,6 +47,7 @@ Table of Contents
    contributing/packages
    contributing/recipes
    contributing/translations
+   contributing/lint
    contributing/tests
    contributing/features
    contributing/stacks

pyspark-notebook/Dockerfile

@@ -5,6 +5,9 @@ FROM $BASE_CONTAINER
 
 LABEL maintainer="Jupyter Project <jupyter@googlegroups.com>"
 
+# Fix DL4006
+SHELL ["/bin/bash", "-o", "pipefail", "-c"]
+
 USER root
 
 # Spark dependencies
@@ -16,13 +19,16 @@ RUN apt-get -y update && \
     rm -rf /var/lib/apt/lists/*
 
 # Using the preferred mirror to download Spark
-RUN cd /tmp && \
-    wget -q $(wget -qO- https://www.apache.org/dyn/closer.lua/spark/spark-${APACHE_SPARK_VERSION}/spark-${APACHE_SPARK_VERSION}-bin-hadoop${HADOOP_VERSION}.tgz\?as_json | \
+WORKDIR /tmp
+# hadolint ignore=SC2046
+RUN wget -q $(wget -qO- https://www.apache.org/dyn/closer.lua/spark/spark-${APACHE_SPARK_VERSION}/spark-${APACHE_SPARK_VERSION}-bin-hadoop${HADOOP_VERSION}.tgz\?as_json | \
     python -c "import sys, json; content=json.load(sys.stdin); print(content['preferred']+content['path_info'])") && \
     echo "2426a20c548bdfc07df288cd1d18d1da6b3189d0b78dee76fa034c52a4e02895f0ad460720c526f163ba63a17efae4764c46a1cd8f9b04c60f9937a554db85d2 *spark-${APACHE_SPARK_VERSION}-bin-hadoop${HADOOP_VERSION}.tgz" | sha512sum -c - && \
-    tar xzf spark-${APACHE_SPARK_VERSION}-bin-hadoop${HADOOP_VERSION}.tgz -C /usr/local --owner root --group root --no-same-owner && \
-    rm spark-${APACHE_SPARK_VERSION}-bin-hadoop${HADOOP_VERSION}.tgz
-RUN cd /usr/local && ln -s spark-${APACHE_SPARK_VERSION}-bin-hadoop${HADOOP_VERSION} spark
+    tar xzf "spark-${APACHE_SPARK_VERSION}-bin-hadoop${HADOOP_VERSION}.tgz" -C /usr/local --owner root --group root --no-same-owner && \
+    rm "spark-${APACHE_SPARK_VERSION}-bin-hadoop${HADOOP_VERSION}.tgz"
+
+WORKDIR /usr/local
+RUN ln -s "spark-${APACHE_SPARK_VERSION}-bin-hadoop${HADOOP_VERSION}" spark
 
 # Configure Spark
 ENV SPARK_HOME=/usr/local/spark
@@ -35,5 +41,7 @@ USER $NB_UID
 # Install pyarrow
 RUN conda install --quiet -y 'pyarrow' && \
     conda clean --all -f -y && \
-    fix-permissions $CONDA_DIR && \
-    fix-permissions /home/$NB_USER
+    fix-permissions "${CONDA_DIR}" && \
+    fix-permissions "/home/${NB_USER}"
+
+WORKDIR $HOME

r-notebook/Dockerfile

@@ -47,7 +47,7 @@ RUN conda install --quiet --yes \
     'unixodbc=2.3.*' \
     && \
     conda clean --all -f -y && \
-    fix-permissions $CONDA_DIR
+    fix-permissions "${CONDA_DIR}"
 
 # Install e1071 R package (dependency of the caret R package)
 RUN conda install --quiet --yes r-e1071

scipy-notebook/Dockerfile

@@ -59,24 +59,25 @@ RUN conda install --quiet --yes \
     jupyter lab build -y && \
     jupyter lab clean -y && \
     npm cache clean --force && \
-    rm -rf /home/$NB_USER/.cache/yarn && \
-    rm -rf /home/$NB_USER/.node-gyp && \
-    fix-permissions $CONDA_DIR && \
-    fix-permissions /home/$NB_USER
+    rm -rf "/home/${NB_USER}/.cache/yarn" && \
+    rm -rf "/home/${NB_USER}/.node-gyp" && \
+    fix-permissions "${CONDA_DIR}" && \
+    fix-permissions "/home/${NB_USER}"
 
 # Install facets which does not have a pip or conda package at the moment
-RUN cd /tmp && \
-    git clone https://github.com/PAIR-code/facets.git && \
-    cd facets && \
-    jupyter nbextension install facets-dist/ --sys-prefix && \
-    cd && \
+WORKDIR /tmp
+RUN git clone https://github.com/PAIR-code/facets.git && \
+    jupyter nbextension install facets/facets-dist/ --sys-prefix && \
     rm -rf /tmp/facets && \
-    fix-permissions $CONDA_DIR && \
-    fix-permissions /home/$NB_USER
+    fix-permissions "${CONDA_DIR}" && \
+    fix-permissions "/home/${NB_USER}"
 
 # Import matplotlib the first time to build the font cache.
-ENV XDG_CACHE_HOME /home/$NB_USER/.cache/
+ENV XDG_CACHE_HOME="/home/${NB_USER}/.cache/"
+
 RUN MPLBACKEND=Agg python -c "import matplotlib.pyplot" && \
-    fix-permissions /home/$NB_USER
+    fix-permissions "/home/${NB_USER}"
 
 USER $NB_UID
+
+WORKDIR $HOME

tensorflow-notebook/Dockerfile

@@ -8,5 +8,5 @@ LABEL maintainer="Jupyter Project <jupyter@googlegroups.com>"
 # Install Tensorflow
 RUN pip install --quiet --no-cache-dir \
     'tensorflow==2.2.0' && \
-    fix-permissions $CONDA_DIR && \
-    fix-permissions /home/$NB_USER
+    fix-permissions "${CONDA_DIR}" && \
+    fix-permissions "/home/${NB_USER}"