Merge branch 'master' into #1077

Makefile

@@ -52,6 +52,16 @@ build-test-all: $(foreach I,$(ALL_IMAGES),arch_patch/$(I) build/$(I) test/$(I) )
 check-outdated/%: ## check the outdated conda packages in a stack and produce a report (experimental)
 	@TEST_IMAGE="$(OWNER)/$(notdir $@)" pytest test/test_outdated.py
 
+cont-clean-all: cont-stop-all cont-rm-all ## clean all containers (stop + rm)
+
+cont-stop-all: ## stop all containers
+	@echo "Stopping all containers ..."
+	-docker stop -t0 $(shell docker ps -a -q) 2> /dev/null
+
+cont-rm-all: ## remove all containers
+	@echo "Removing all containers ..."
+	-docker rm --force $(shell docker ps -a -q) 2> /dev/null
+
 dev/%: ARGS?=
 dev/%: DARGS?=
 dev/%: PORT?=8888
@@ -61,6 +71,20 @@ dev/%: ## run a foreground container for a stack
 dev-env: ## install libraries required to build docs and run tests
 	pip install -r requirements-dev.txt
 
+img-clean: img-rm-dang img-rm ## clean dangling and jupyter images
+
+img-list: ## list jupyter images
+	@echo "Listing $(OWNER) images ..."
+	docker images "$(OWNER)/*"
+
+img-rm:  ## remove jupyter images
+	@echo "Removing $(OWNER) images ..."
+	-docker rmi --force $(shell docker images --quiet "$(OWNER)/*") 2> /dev/null
+
+img-rm-dang: ## remove dangling images (tagged None)
+	@echo "Removing dangling images ..."
+	-docker rmi --force $(shell docker images -f "dangling=true" -q) 2> /dev/null
+
 docs: ## build HTML documentation
 	make -C docs html
 
@@ -71,11 +95,17 @@ n-docs-diff: ## number of docs/ files changed since branch from master
 n-other-diff: ## number of files outside docs/ changed since branch from master
 	@git diff --name-only $(DIFF_RANGE) -- ':!docs/' | wc -l | awk '{print $$1}'
 
+pull/%: DARGS?=
+pull/%: ## pull a jupyter image
+	docker pull $(DARGS) $(OWNER)/$(notdir $@)
+
+run/%: DARGS?=
 run/%: ## run a bash in interactive mode in a stack
-	docker run -it --rm $(OWNER)/$(notdir $@) $(SHELL)
+	docker run -it --rm $(DARGS) $(OWNER)/$(notdir $@) $(SHELL)
 
+run-sudo/%: DARGS?=
 run-sudo/%: ## run a bash in interactive mode as root in a stack
-	docker run -it --rm -u root $(OWNER)/$(notdir $@) $(SHELL)
+	docker run -it --rm -u root $(DARGS) $(OWNER)/$(notdir $@) $(SHELL)
 
 tx-en: ## rebuild en locale strings and push to master (req: GH_TOKEN)
 	@git config --global user.email "travis@travis-ci.org"

all-spark-notebook/Dockerfile

@@ -27,7 +27,7 @@ RUN conda install --quiet --yes \
     'r-ggplot2=3.3*' \
     'r-irkernel=1.1*' \
     'r-rcurl=1.98*' \
-    'r-sparklyr=1.1*' \
+    'r-sparklyr=1.2*' \
     && \
     conda clean --all -f -y && \
     fix-permissions $CONDA_DIR && \

base-notebook/Dockerfile

@@ -4,7 +4,7 @@
 # Ubuntu 18.04 (bionic)
 # https://hub.docker.com/_/ubuntu/?tab=tags&name=bionic
 # OS/ARCH: linux/amd64
-ARG ROOT_CONTAINER=ubuntu:bionic-20200311@sha256:e5dd9dbb37df5b731a6688fa49f4003359f6f126958c9c928f937bec69836320
+ARG ROOT_CONTAINER=ubuntu:bionic-20200403@sha256:b58746c8a89938b8c9f5b77de3b8cf1fe78210c696ab03a1442e235eea65d84f
 ARG BASE_CONTAINER=$ROOT_CONTAINER
 FROM $BASE_CONTAINER
 
@@ -112,7 +112,7 @@ RUN conda install --quiet --yes 'tini=0.18.0' && \
 RUN conda install --quiet --yes \
     'notebook=6.0.3' \
     'jupyterhub=1.1.0' \
-    'jupyterlab=2.0.1' && \
+    'jupyterlab=2.1.1' && \
     conda clean --all -f -y && \
     npm cache clean --force && \
     jupyter notebook --generate-config && \

datascience-notebook/Dockerfile

@@ -23,12 +23,12 @@ RUN apt-get update && \
 # install Julia packages in /opt/julia instead of $HOME
 ENV JULIA_DEPOT_PATH=/opt/julia
 ENV JULIA_PKGDIR=/opt/julia
-ENV JULIA_VERSION=1.3.1
+ENV JULIA_VERSION=1.4.1
 
 RUN mkdir /opt/julia-${JULIA_VERSION} && \
     cd /tmp && \
     wget -q https://julialang-s3.julialang.org/bin/linux/x64/`echo ${JULIA_VERSION} | cut -d. -f 1,2`/julia-${JULIA_VERSION}-linux-x86_64.tar.gz && \
-    echo "faa707c8343780a6fe5eaf13490355e8190acf8e2c189b9e7ecbddb0fa2643ad *julia-${JULIA_VERSION}-linux-x86_64.tar.gz" | sha256sum -c - && \
+    echo "fd6d8cadaed678174c3caefb92207a3b0e8da9f926af6703fb4d1e4e4f50610a *julia-${JULIA_VERSION}-linux-x86_64.tar.gz" | sha256sum -c - && \
     tar xzf julia-${JULIA_VERSION}-linux-x86_64.tar.gz -C /opt/julia-${JULIA_VERSION} --strip-components=1 && \
     rm /tmp/julia-${JULIA_VERSION}-linux-x86_64.tar.gz
 RUN ln -fs /opt/julia-*/bin/julia /usr/local/bin/julia
@@ -48,8 +48,8 @@ RUN conda install --quiet --yes \
     'r-base=3.6.3' \
     'r-caret=6.0*' \
     'r-crayon=1.3*' \
-    'r-devtools=2.2*' \
-    'r-forecast=8.11*' \
+    'r-devtools=2.3*' \
+    'r-forecast=8.12*' \
     'r-hexbin=1.28*' \
     'r-htmltools=0.4*' \
     'r-htmlwidgets=1.5*' \

datascience-notebook/test/test_julia.py

+# Copyright (c) Jupyter Development Team.
+# Distributed under the terms of the Modified BSD License.
+import logging
+
+import pytest
+
+LOGGER = logging.getLogger(__name__)
+
+
+def test_julia(container):
+    """Basic julia test"""
+    LOGGER.info(f"Test that julia is correctly installed ...")
+    running_container = container.run(
+        tty=True, command=["start.sh", "bash", "-c", "sleep infinity"]
+    )
+    command = f"julia --version"
+    cmd = running_container.exec_run(command)
+    output = cmd.output.decode("utf-8")
+    assert cmd.exit_code == 0, f"Command {command} failed {output}"
+    LOGGER.debug(output)

minimal-notebook/test/test_nbconvert.py

@@ -16,7 +16,7 @@ def test_nbconvert(container, format):
     cont_data_dir = "/home/jovyan/data"
     test_file = "notebook1"
     output_dir = "/tmp"
-    LOGGER.info(f"Converting example notebook to {format.upper()} ...")
+    LOGGER.info(f"Test that an example notebook can be converted to {format.upper()} ...")
     command = f"jupyter nbconvert {cont_data_dir}/{test_file}.ipynb --output-dir {output_dir} --to {format}"
     c = container.run(
         volumes={host_data_dir: {"bind": cont_data_dir, "mode": "ro"}},
@@ -24,8 +24,8 @@ def test_nbconvert(container, format):
         command=["start.sh", "bash", "-c", command],
     )
     rv = c.wait(timeout=30)
-    assert rv == 0 or rv["StatusCode"] == 0
+    assert rv == 0 or rv["StatusCode"] == 0, f"Command {command} failed"
     logs = c.logs(stdout=True).decode("utf-8")
     LOGGER.debug(logs)
-    assert f"{output_dir}/{test_file}.{format}" in logs
-
+    expected_file = f"{output_dir}/{test_file}.{format}"
+    assert expected_file in logs, f"Expected file {expected_file} not generated"

r-notebook/Dockerfile

@@ -28,8 +28,8 @@ RUN conda install --quiet --yes \
     'r-base=3.6.3' \
     'r-caret=6.0*' \
     'r-crayon=1.3*' \
-    'r-devtools=2.2*' \
-    'r-forecast=8.11*' \
+    'r-devtools=2.3*' \
+    'r-forecast=8.12*' \
     'r-hexbin=1.28*' \
     'r-htmltools=0.4*' \
     'r-htmlwidgets=1.5*' \

scipy-notebook/Dockerfile

@@ -16,19 +16,20 @@ USER $NB_UID
 
 # Install Python 3 packages
 RUN conda install --quiet --yes \
-    'beautifulsoup4=4.8.*' \
+    'beautifulsoup4=4.9.*' \
     'conda-forge::blas=*=openblas' \
     'bokeh=2.0.*' \
     'bottleneck=1.3.*' \
-    'cloudpickle=1.3.*' \
+    'cloudpickle=1.4.*' \
     'cython=0.29.*' \
-    'dask=2.14.*' \
+    'dask=2.15.*' \
     'dill=0.3.*' \
     'h5py=2.10.*' \
     'hdf5=1.10.*' \
     'ipywidgets=7.5.*' \
     'ipympl=0.5.*'\
     'matplotlib-base=3.2.*' \
+    # numba update to 0.49 fails resolving deps.
     'numba=0.48.*' \
     'numexpr=2.7.*' \
     'pandas=1.0.*' \
@@ -44,7 +45,7 @@ RUN conda install --quiet --yes \
     'sympy=1.5.*' \
     'vincent=0.4.*' \
     'widgetsnbextension=3.5.*'\
-    'xlrd' \
+    'xlrd=1.2.*' \
     && \
     conda clean --all -f -y && \
     # Activate ipywidgets extension in the environment that runs the notebook server

scipy-notebook/test/data/matplotlib_1.py

+# Matplotlit: Create a simple plot example.
+# Refs: https://matplotlib.org/3.1.1/gallery/lines_bars_and_markers/simple_plot.html
+
+# Optional test with [Matplotlib Jupyter Integration](https://github.com/matplotlib/ipympl)
+# %matplotlib widget
+import matplotlib
+import matplotlib.pyplot as plt
+import numpy as np
+import os
+
+# Data for plotting
+t = np.arange(0.0, 2.0, 0.01)
+s = 1 + np.sin(2 * np.pi * t)
+
+fig, ax = plt.subplots()
+ax.plot(t, s)
+
+ax.set(xlabel='time (s)', ylabel='voltage (mV)',
+       title='About as simple as it gets, folks')
+ax.grid()
+# Note that the test can be run headless by checking if an image is produced
+file_path = os.path.join("/tmp", "test.png")
+fig.savefig(file_path)
+print(f"File {file_path} saved")
\ No newline at end of file

scipy-notebook/test/test_matplotlib.py

+# Copyright (c) Jupyter Development Team.
+# Distributed under the terms of the Modified BSD License.
+
+import logging
+
+import pytest
+import os
+
+LOGGER = logging.getLogger(__name__)
+
+
+def test_matplotlib(container):
+    """Test that matplotlib is able to plot a graph and write it as an image"""
+    host_data_dir = os.path.join(os.path.dirname(os.path.realpath(__file__)), "data")
+    cont_data_dir = "/home/jovyan/data"
+    test_file = "matplotlib_1.py"
+    output_dir = "/tmp"
+    LOGGER.info(f"Test that matplotlib is able to plot a graph and write it as an image ...")
+    command = "sleep infinity"
+    running_container = container.run(
+        volumes={host_data_dir: {"bind": cont_data_dir, "mode": "ro"}},
+        tty=True,
+        command=["start.sh", "bash", "-c", command],
+    )
+    command = f"python {cont_data_dir}/{test_file}"
+    cmd = running_container.exec_run(command)
+    assert cmd.exit_code == 0, f"Command {command} failed"
+    LOGGER.debug(cmd.output.decode("utf-8"))
+    # Checking if the file is generated
+    # https://stackoverflow.com/a/15895594/4413446
+    expected_file = f"{output_dir}/test.png"
+    command = f"test -s {expected_file}"
+    cmd = running_container.exec_run(command)
+    assert cmd.exit_code == 0, f"Command {command} failed"
+    LOGGER.debug(cmd.output.decode("utf-8"))

scipy-notebook/test/test_pandas.py

@@ -21,6 +21,6 @@ def test_pandas(container, name, command):
     LOGGER.info(f"Testing pandas: {name} ...")
     c = container.run(tty=True, command=["start.sh", "python", "-c", command])
     rv = c.wait(timeout=30)
-    assert rv == 0 or rv["StatusCode"] == 0
+    assert rv == 0 or rv["StatusCode"] == 0, f"Command {command} failed"
     logs = c.logs(stdout=True).decode("utf-8")
     LOGGER.debug(logs)

tensorflow-notebook/test/test_tensorflow.py

@@ -25,6 +25,6 @@ def test_tensorflow(container, name, command):
     LOGGER.info(f"Testing tensorflow: {name} ...")
     c = container.run(tty=True, command=["start.sh", "python", "-c", command])
     rv = c.wait(timeout=30)
-    assert rv == 0 or rv["StatusCode"] == 0
+    assert rv == 0 or rv["StatusCode"] == 0, f"Command {command} failed"
     logs = c.logs(stdout=True).decode("utf-8")
     LOGGER.debug(logs)

test/test_notebook.py

 # Copyright (c) Jupyter Development Team.
 # Distributed under the terms of the Modified BSD License.
 
+
 def test_secured_server(container, http_client):
     """Notebook server should eventually request user login."""
     container.run()
-    resp = http_client.get('http://localhost:8888')
+    resp = http_client.get("http://localhost:8888")
     resp.raise_for_status()
-    assert 'login_submit' in resp.text
+    assert "login_submit" in resp.text, "User login not requested"