Merge branch 'master' into auto_sklearn

Makefile

@@ -52,6 +52,16 @@ build-test-all: $(foreach I,$(ALL_IMAGES),arch_patch/$(I) build/$(I) test/$(I) )
 check-outdated/%: ## check the outdated conda packages in a stack and produce a report (experimental)
 	@TEST_IMAGE="$(OWNER)/$(notdir $@)" pytest test/test_outdated.py
 
+cont-clean-all: cont-stop-all cont-rm-all ## clean all containers (stop + rm)
+
+cont-stop-all: ## stop all containers
+	@echo "Stopping all containers ..."
+	-docker stop -t0 $(shell docker ps -a -q) 2> /dev/null
+
+cont-rm-all: ## remove all containers
+	@echo "Removing all containers ..."
+	-docker rm --force $(shell docker ps -a -q) 2> /dev/null
+
 dev/%: ARGS?=
 dev/%: DARGS?=
 dev/%: PORT?=8888
@@ -61,6 +71,20 @@ dev/%: ## run a foreground container for a stack
 dev-env: ## install libraries required to build docs and run tests
 	pip install -r requirements-dev.txt
 
+img-clean: img-rm-dang img-rm ## clean dangling and jupyter images
+
+img-list: ## list jupyter images
+	@echo "Listing $(OWNER) images ..."
+	docker images "$(OWNER)/*"
+
+img-rm:  ## remove jupyter images
+	@echo "Removing $(OWNER) images ..."
+	-docker rmi --force $(shell docker images --quiet "$(OWNER)/*") 2> /dev/null
+
+img-rm-dang: ## remove dangling images (tagged None)
+	@echo "Removing dangling images ..."
+	-docker rmi --force $(shell docker images -f "dangling=true" -q) 2> /dev/null
+
 docs: ## build HTML documentation
 	make -C docs html
 
@@ -71,11 +95,17 @@ n-docs-diff: ## number of docs/ files changed since branch from master
 n-other-diff: ## number of files outside docs/ changed since branch from master
 	@git diff --name-only $(DIFF_RANGE) -- ':!docs/' | wc -l | awk '{print $$1}'
 
+pull/%: DARGS?=
+pull/%: ## pull a jupyter image
+	docker pull $(DARGS) $(OWNER)/$(notdir $@)
+
+run/%: DARGS?=
 run/%: ## run a bash in interactive mode in a stack
-	docker run -it --rm $(OWNER)/$(notdir $@) $(SHELL)
+	docker run -it --rm $(DARGS) $(OWNER)/$(notdir $@) $(SHELL)
 
+run-sudo/%: DARGS?=
 run-sudo/%: ## run a bash in interactive mode as root in a stack
-	docker run -it --rm -u root $(OWNER)/$(notdir $@) $(SHELL)
+	docker run -it --rm -u root $(DARGS) $(OWNER)/$(notdir $@) $(SHELL)
 
 tx-en: ## rebuild en locale strings and push to master (req: GH_TOKEN)
 	@git config --global user.email "travis@travis-ci.org"

all-spark-notebook/Dockerfile

@@ -27,7 +27,7 @@ RUN conda install --quiet --yes \
     'r-ggplot2=3.3*' \
     'r-irkernel=1.1*' \
     'r-rcurl=1.98*' \
-    'r-sparklyr=1.1*' \
+    'r-sparklyr=1.2*' \
     && \
     conda clean --all -f -y && \
     fix-permissions $CONDA_DIR && \

base-notebook/Dockerfile

@@ -4,7 +4,7 @@
 # Ubuntu 18.04 (bionic)
 # https://hub.docker.com/_/ubuntu/?tab=tags&name=bionic
 # OS/ARCH: linux/amd64
-ARG ROOT_CONTAINER=ubuntu:bionic-20200311@sha256:e5dd9dbb37df5b731a6688fa49f4003359f6f126958c9c928f937bec69836320
+ARG ROOT_CONTAINER=ubuntu:bionic-20200403@sha256:b58746c8a89938b8c9f5b77de3b8cf1fe78210c696ab03a1442e235eea65d84f
 ARG BASE_CONTAINER=$ROOT_CONTAINER
 FROM $BASE_CONTAINER
 
@@ -112,7 +112,7 @@ RUN conda install --quiet --yes 'tini=0.18.0' && \
 RUN conda install --quiet --yes \
     'notebook=6.0.3' \
     'jupyterhub=1.1.0' \
-    'jupyterlab=2.0.1' && \
+    'jupyterlab=2.1.1' && \
     conda clean --all -f -y && \
     npm cache clean --force && \
     jupyter notebook --generate-config && \

base-notebook/start.sh

@@ -93,7 +93,7 @@ if [ $(id -u) == 0 ] ; then
     fi
 
     # Add $CONDA_DIR/bin to sudo secure_path
-    sed -r "s#Defaults\s+secure_path=\"([^\"]+)\"#Defaults secure_path=\"\1:$CONDA_DIR/bin\"#" /etc/sudoers | grep secure_path > /etc/sudoers.d/path
+    sed -r "s#Defaults\s+secure_path\s*=\s*\"?([^\"]+)\"?#Defaults secure_path=\"\1:$CONDA_DIR/bin\"#" /etc/sudoers | grep secure_path > /etc/sudoers.d/path
 
     # Exec the command as NB_USER with the PATH and the rest of
     # the environment preserved

datascience-notebook/Dockerfile

@@ -23,12 +23,12 @@ RUN apt-get update && \
 # install Julia packages in /opt/julia instead of $HOME
 ENV JULIA_DEPOT_PATH=/opt/julia
 ENV JULIA_PKGDIR=/opt/julia
-ENV JULIA_VERSION=1.3.1
+ENV JULIA_VERSION=1.4.1
 
 RUN mkdir /opt/julia-${JULIA_VERSION} && \
     cd /tmp && \
     wget -q https://julialang-s3.julialang.org/bin/linux/x64/`echo ${JULIA_VERSION} | cut -d. -f 1,2`/julia-${JULIA_VERSION}-linux-x86_64.tar.gz && \
-    echo "faa707c8343780a6fe5eaf13490355e8190acf8e2c189b9e7ecbddb0fa2643ad *julia-${JULIA_VERSION}-linux-x86_64.tar.gz" | sha256sum -c - && \
+    echo "fd6d8cadaed678174c3caefb92207a3b0e8da9f926af6703fb4d1e4e4f50610a *julia-${JULIA_VERSION}-linux-x86_64.tar.gz" | sha256sum -c - && \
     tar xzf julia-${JULIA_VERSION}-linux-x86_64.tar.gz -C /opt/julia-${JULIA_VERSION} --strip-components=1 && \
     rm /tmp/julia-${JULIA_VERSION}-linux-x86_64.tar.gz
 RUN ln -fs /opt/julia-*/bin/julia /usr/local/bin/julia
@@ -48,8 +48,8 @@ RUN conda install --quiet --yes \
     'r-base=3.6.3' \
     'r-caret=6.0*' \
     'r-crayon=1.3*' \
-    'r-devtools=2.2*' \
-    'r-forecast=8.11*' \
+    'r-devtools=2.3*' \
+    'r-forecast=8.12*' \
     'r-hexbin=1.28*' \
     'r-htmltools=0.4*' \
     'r-htmlwidgets=1.5*' \

datascience-notebook/test/test_julia.py

+# Copyright (c) Jupyter Development Team.
+# Distributed under the terms of the Modified BSD License.
+import logging
+
+import pytest
+
+LOGGER = logging.getLogger(__name__)
+
+
+def test_julia(container):
+    """Basic julia test"""
+    LOGGER.info(f"Test that julia is correctly installed ...")
+    running_container = container.run(
+        tty=True, command=["start.sh", "bash", "-c", "sleep infinity"]
+    )
+    command = f"julia --version"
+    cmd = running_container.exec_run(command)
+    output = cmd.output.decode("utf-8")
+    assert cmd.exit_code == 0, f"Command {command} failed {output}"
+    LOGGER.debug(output)

docs/using/selecting.md

@@ -82,7 +82,7 @@ packages from [conda-forge](https://conda-forge.github.io/feedstocks)
 [statsmodel](http://www.statsmodels.org/stable/index.html), [cloudpickle](https://github.com/cloudpipe/cloudpickle), [dill](https://pypi.python.org/pypi/dill),
 [numba](https://numba.pydata.org/), [bokeh](https://bokeh.pydata.org/en/latest/), [sqlalchemy](https://www.sqlalchemy.org/), [hdf5](http://www.h5py.org/),
 [vincent](http://vincent.readthedocs.io/en/latest/), [beautifulsoup](https://www.crummy.com/software/BeautifulSoup/),
-[protobuf](https://developers.google.com/protocol-buffers/docs/pythontutorial), and [xlrd](http://www.python-excel.org/) packages
+[protobuf](https://developers.google.com/protocol-buffers/docs/pythontutorial), [xlrd](http://www.python-excel.org/), [bottleneck](https://bottleneck.readthedocs.io/en/latest/), and [pytables](https://www.pytables.org/) packages
 * [ipywidgets](https://ipywidgets.readthedocs.io/en/stable/) and [ipympl](https://github.com/matplotlib/jupyter-matplotlib) for interactive visualizations and plots in Python notebooks
 * [Facets](https://github.com/PAIR-code/facets) for visualizing machine learning datasets
 
@@ -182,8 +182,7 @@ the datascience-notebook image. Click here to launch it on
 
   `crosscompass/ihaskell-notebook` is based on [IHaskell](https://github.com/gibiansky/IHaskell). Includes popular packages and example notebooks.
 
-  Try it on binder: [![launch Learn You a Haskell for Great
-Good!](https://mybinder.org/badge_logo.svg)](https://mybinder.org/v2/gh/jamesdbrock/learn-you-a-haskell-notebook/master?urlpath=lab/tree/learn_you_a_haskell/00-preface.ipynb)
+  Try it on [![Binder](https://mybinder.org/badge_logo.svg)](https://mybinder.org/v2/gh/jamesdbrock/learn-you-a-haskell-notebook/master?urlpath=lab/tree/ihaskell_examples/ihaskell/IHaskell.ipynb)
 
 * [java-notebook is a community Jupyter Docker Stack image](https://github.com/jbindinga/java-notebook). The image includes
 [IJava](https://github.com/SpencerPark/IJava) kernel on top of the minimal-notebook image. Click here to launch it on

minimal-notebook/test/test_nbconvert.py

@@ -16,7 +16,7 @@ def test_nbconvert(container, format):
     cont_data_dir = "/home/jovyan/data"
     test_file = "notebook1"
     output_dir = "/tmp"
-    LOGGER.info(f"Converting example notebook to {format.upper()} ...")
+    LOGGER.info(f"Test that an example notebook can be converted to {format.upper()} ...")
     command = f"jupyter nbconvert {cont_data_dir}/{test_file}.ipynb --output-dir {output_dir} --to {format}"
     c = container.run(
         volumes={host_data_dir: {"bind": cont_data_dir, "mode": "ro"}},
@@ -24,8 +24,8 @@ def test_nbconvert(container, format):
         command=["start.sh", "bash", "-c", command],
     )
     rv = c.wait(timeout=30)
-    assert rv == 0 or rv["StatusCode"] == 0
+    assert rv == 0 or rv["StatusCode"] == 0, f"Command {command} failed"
     logs = c.logs(stdout=True).decode("utf-8")
     LOGGER.debug(logs)
-    assert f"{output_dir}/{test_file}.{format}" in logs
-
+    expected_file = f"{output_dir}/{test_file}.{format}"
+    assert expected_file in logs, f"Expected file {expected_file} not generated"

r-notebook/Dockerfile

@@ -28,8 +28,8 @@ RUN conda install --quiet --yes \
     'r-base=3.6.3' \
     'r-caret=6.0*' \
     'r-crayon=1.3*' \
-    'r-devtools=2.2*' \
-    'r-forecast=8.11*' \
+    'r-devtools=2.3*' \
+    'r-forecast=8.12*' \
     'r-hexbin=1.28*' \
     'r-htmltools=0.4*' \
     'r-htmlwidgets=1.5*' \

scipy-notebook/Dockerfile

@@ -16,23 +16,26 @@ USER $NB_UID
 
 # Install Python 3 packages
 RUN conda install --quiet --yes \
-    'beautifulsoup4=4.8.*' \
+    'beautifulsoup4=4.9.*' \
     'conda-forge::blas=*=openblas' \
     'bokeh=2.0.*' \
-    'cloudpickle=1.3.*' \
+    'bottleneck=1.3.*' \
+    'cloudpickle=1.4.*' \
     'cython=0.29.*' \
-    'dask=2.14.*' \
+    'dask=2.15.*' \
     'dill=0.3.*' \
     'h5py=2.10.*' \
     'hdf5=1.10.*' \
     'ipywidgets=7.5.*' \
     'ipympl=0.5.*'\
     'matplotlib-base=3.2.*' \
+    # numba update to 0.49 fails resolving deps.
     'numba=0.48.*' \
     'numexpr=2.7.*' \
     'pandas=1.0.*' \
     'patsy=0.5.*' \
     'protobuf=3.11.*' \
+    'pytables=3.6.*' \
     'scikit-image=0.16.*' \
     'scikit-learn=0.22.*' \
     'scipy=1.4.*' \
@@ -42,7 +45,7 @@ RUN conda install --quiet --yes \
     'sympy=1.5.*' \
     'vincent=0.4.*' \
     'widgetsnbextension=3.5.*'\
-    'xlrd' \
+    'xlrd=1.2.*' \
     && \
     conda clean --all -f -y && \
     # Activate ipywidgets extension in the environment that runs the notebook server

scipy-notebook/test/data/matplotlib_1.py

+# Matplotlit: Create a simple plot example.
+# Refs: https://matplotlib.org/3.1.1/gallery/lines_bars_and_markers/simple_plot.html
+
+# Optional test with [Matplotlib Jupyter Integration](https://github.com/matplotlib/ipympl)
+# %matplotlib widget
+import matplotlib
+import matplotlib.pyplot as plt
+import numpy as np
+import os
+
+# Data for plotting
+t = np.arange(0.0, 2.0, 0.01)
+s = 1 + np.sin(2 * np.pi * t)
+
+fig, ax = plt.subplots()
+ax.plot(t, s)
+
+ax.set(xlabel='time (s)', ylabel='voltage (mV)',
+       title='About as simple as it gets, folks')
+ax.grid()
+# Note that the test can be run headless by checking if an image is produced
+file_path = os.path.join("/tmp", "test.png")
+fig.savefig(file_path)
+print(f"File {file_path} saved")
\ No newline at end of file

scipy-notebook/test/test_matplotlib.py

+# Copyright (c) Jupyter Development Team.
+# Distributed under the terms of the Modified BSD License.
+
+import logging
+
+import pytest
+import os
+
+LOGGER = logging.getLogger(__name__)
+
+
+def test_matplotlib(container):
+    """Test that matplotlib is able to plot a graph and write it as an image"""
+    host_data_dir = os.path.join(os.path.dirname(os.path.realpath(__file__)), "data")
+    cont_data_dir = "/home/jovyan/data"
+    test_file = "matplotlib_1.py"
+    output_dir = "/tmp"
+    LOGGER.info(f"Test that matplotlib is able to plot a graph and write it as an image ...")
+    command = "sleep infinity"
+    running_container = container.run(
+        volumes={host_data_dir: {"bind": cont_data_dir, "mode": "ro"}},
+        tty=True,
+        command=["start.sh", "bash", "-c", command],
+    )
+    command = f"python {cont_data_dir}/{test_file}"
+    cmd = running_container.exec_run(command)
+    assert cmd.exit_code == 0, f"Command {command} failed"
+    LOGGER.debug(cmd.output.decode("utf-8"))
+    # Checking if the file is generated
+    # https://stackoverflow.com/a/15895594/4413446
+    expected_file = f"{output_dir}/test.png"
+    command = f"test -s {expected_file}"
+    cmd = running_container.exec_run(command)
+    assert cmd.exit_code == 0, f"Command {command} failed"
+    LOGGER.debug(cmd.output.decode("utf-8"))

scipy-notebook/test/test_pandas.py

@@ -21,6 +21,6 @@ def test_pandas(container, name, command):
     LOGGER.info(f"Testing pandas: {name} ...")
     c = container.run(tty=True, command=["start.sh", "python", "-c", command])
     rv = c.wait(timeout=30)
-    assert rv == 0 or rv["StatusCode"] == 0
+    assert rv == 0 or rv["StatusCode"] == 0, f"Command {command} failed"
     logs = c.logs(stdout=True).decode("utf-8")
     LOGGER.debug(logs)

tensorflow-notebook/test/test_tensorflow.py

@@ -25,6 +25,6 @@ def test_tensorflow(container, name, command):
     LOGGER.info(f"Testing tensorflow: {name} ...")
     c = container.run(tty=True, command=["start.sh", "python", "-c", command])
     rv = c.wait(timeout=30)
-    assert rv == 0 or rv["StatusCode"] == 0
+    assert rv == 0 or rv["StatusCode"] == 0, f"Command {command} failed"
     logs = c.logs(stdout=True).decode("utf-8")
     LOGGER.debug(logs)

test/test_notebook.py

 # Copyright (c) Jupyter Development Team.
 # Distributed under the terms of the Modified BSD License.
 
+
 def test_secured_server(container, http_client):
     """Notebook server should eventually request user login."""
     container.run()
-    resp = http_client.get('http://localhost:8888')
+    resp = http_client.get("http://localhost:8888")
     resp.raise_for_status()
-    assert 'login_submit' in resp.text
+    assert "login_submit" in resp.text, "User login not requested"

test/test_packages.py

@@ -21,7 +21,7 @@ Example:
     $ make test/datascience-notebook
 
     # [...]
-    # test/test_packages.py::test_python_packages 
+    # test/test_packages.py::test_python_packages
     # --------------------------------------------------------------------------------------------- live log setup ----------------------------------------------------------------------------------------------
     # 2020-03-08 09:56:04 [    INFO] Starting container jupyter/datascience-notebook ... (helpers.py:51)
     # 2020-03-08 09:56:04 [    INFO] Running jupyter/datascience-notebook with args {'detach': True, 'ports': {'8888/tcp': 8888}, 'tty': True, 'command': ['start.sh', 'bash', '-c', 'sleep infinity']} ... (conftest.py:78)
@@ -51,6 +51,7 @@ PACKAGE_MAPPING = {
     "scikit-learn": "sklearn",
     "scikit-image": "skimage",
     "spylon-kernel": "spylon_kernel",
+    "pytables": "tables",
     # R
     "randomforest": "randomForest",
     "rsqlite": "DBI",