Merge branch 'master' into regexp-centos-friendly

.travis.yml

-dist: xenial
+dist: bionic
 language: python
 python:
     - 3.7

Makefile

@@ -52,6 +52,16 @@ build-test-all: $(foreach I,$(ALL_IMAGES),arch_patch/$(I) build/$(I) test/$(I) )
 check-outdated/%: ## check the outdated conda packages in a stack and produce a report (experimental)
 	@TEST_IMAGE="$(OWNER)/$(notdir $@)" pytest test/test_outdated.py
 
+cont-clean-all: cont-stop-all cont-rm-all ## clean all containers (stop + rm)
+
+cont-stop-all: ## stop all containers
+	@echo "Stopping all containers ..."
+	-docker stop -t0 $(shell docker ps -a -q) 2> /dev/null
+
+cont-rm-all: ## remove all containers
+	@echo "Removing all containers ..."
+	-docker rm --force $(shell docker ps -a -q) 2> /dev/null
+
 dev/%: ARGS?=
 dev/%: DARGS?=
 dev/%: PORT?=8888
@@ -61,6 +71,20 @@ dev/%: ## run a foreground container for a stack
 dev-env: ## install libraries required to build docs and run tests
 	pip install -r requirements-dev.txt
 
+img-clean: img-rm-dang img-rm ## clean dangling and jupyter images
+
+img-list: ## list jupyter images
+	@echo "Listing $(OWNER) images ..."
+	docker images "$(OWNER)/*"
+
+img-rm:  ## remove jupyter images
+	@echo "Removing $(OWNER) images ..."
+	-docker rmi --force $(shell docker images --quiet "$(OWNER)/*") 2> /dev/null
+
+img-rm-dang: ## remove dangling images (tagged None)
+	@echo "Removing dangling images ..."
+	-docker rmi --force $(shell docker images -f "dangling=true" -q) 2> /dev/null
+
 docs: ## build HTML documentation
 	make -C docs html
 
@@ -71,11 +95,17 @@ n-docs-diff: ## number of docs/ files changed since branch from master
 n-other-diff: ## number of files outside docs/ changed since branch from master
 	@git diff --name-only $(DIFF_RANGE) -- ':!docs/' | wc -l | awk '{print $$1}'
 
+pull/%: DARGS?=
+pull/%: ## pull a jupyter image
+	docker pull $(DARGS) $(OWNER)/$(notdir $@)
+
+run/%: DARGS?=
 run/%: ## run a bash in interactive mode in a stack
-	docker run -it --rm $(OWNER)/$(notdir $@) $(SHELL)
+	docker run -it --rm $(DARGS) $(OWNER)/$(notdir $@) $(SHELL)
 
+run-sudo/%: DARGS?=
 run-sudo/%: ## run a bash in interactive mode as root in a stack
-	docker run -it --rm -u root $(OWNER)/$(notdir $@) $(SHELL)
+	docker run -it --rm -u root $(DARGS) $(OWNER)/$(notdir $@) $(SHELL)
 
 tx-en: ## rebuild en locale strings and push to master (req: GH_TOKEN)
 	@git config --global user.email "travis@travis-ci.org"

all-spark-notebook/Dockerfile

@@ -23,11 +23,11 @@ USER $NB_UID
 
 # R packages
 RUN conda install --quiet --yes \
-    'r-base=3.6.2' \
-    'r-ggplot2=3.2*' \
+    'r-base=3.6.3' \
+    'r-ggplot2=3.3*' \
     'r-irkernel=1.1*' \
     'r-rcurl=1.98*' \
-    'r-sparklyr=1.1*' \
+    'r-sparklyr=1.2*' \
     && \
     conda clean --all -f -y && \
     fix-permissions $CONDA_DIR && \

base-notebook/Dockerfile

@@ -4,7 +4,7 @@
 # Ubuntu 18.04 (bionic)
 # https://hub.docker.com/_/ubuntu/?tab=tags&name=bionic
 # OS/ARCH: linux/amd64
-ARG ROOT_CONTAINER=ubuntu:bionic-20200219@sha256:0925d086715714114c1988f7c947db94064fd385e171a63c07730f1fa014e6f9
+ARG ROOT_CONTAINER=ubuntu:bionic-20200403@sha256:b58746c8a89938b8c9f5b77de3b8cf1fe78210c696ab03a1442e235eea65d84f
 ARG BASE_CONTAINER=$ROOT_CONTAINER
 FROM $BASE_CONTAINER
 
@@ -112,7 +112,7 @@ RUN conda install --quiet --yes 'tini=0.18.0' && \
 RUN conda install --quiet --yes \
     'notebook=6.0.3' \
     'jupyterhub=1.1.0' \
-    'jupyterlab=2.0.1' && \
+    'jupyterlab=2.1.1' && \
     conda clean --all -f -y && \
     npm cache clean --force && \
     jupyter notebook --generate-config && \

datascience-notebook/Dockerfile

@@ -23,12 +23,12 @@ RUN apt-get update && \
 # install Julia packages in /opt/julia instead of $HOME
 ENV JULIA_DEPOT_PATH=/opt/julia
 ENV JULIA_PKGDIR=/opt/julia
-ENV JULIA_VERSION=1.3.1
+ENV JULIA_VERSION=1.4.1
 
 RUN mkdir /opt/julia-${JULIA_VERSION} && \
     cd /tmp && \
     wget -q https://julialang-s3.julialang.org/bin/linux/x64/`echo ${JULIA_VERSION} | cut -d. -f 1,2`/julia-${JULIA_VERSION}-linux-x86_64.tar.gz && \
-    echo "faa707c8343780a6fe5eaf13490355e8190acf8e2c189b9e7ecbddb0fa2643ad *julia-${JULIA_VERSION}-linux-x86_64.tar.gz" | sha256sum -c - && \
+    echo "fd6d8cadaed678174c3caefb92207a3b0e8da9f926af6703fb4d1e4e4f50610a *julia-${JULIA_VERSION}-linux-x86_64.tar.gz" | sha256sum -c - && \
     tar xzf julia-${JULIA_VERSION}-linux-x86_64.tar.gz -C /opt/julia-${JULIA_VERSION} --strip-components=1 && \
     rm /tmp/julia-${JULIA_VERSION}-linux-x86_64.tar.gz
 RUN ln -fs /opt/julia-*/bin/julia /usr/local/bin/julia
@@ -45,11 +45,11 @@ USER $NB_UID
 
 # R packages including IRKernel which gets installed globally.
 RUN conda install --quiet --yes \
-    'r-base=3.6.2' \
+    'r-base=3.6.3' \
     'r-caret=6.0*' \
     'r-crayon=1.3*' \
-    'r-devtools=2.2*' \
-    'r-forecast=8.10*' \
+    'r-devtools=2.3*' \
+    'r-forecast=8.12*' \
     'r-hexbin=1.28*' \
     'r-htmltools=0.4*' \
     'r-htmlwidgets=1.5*' \
@@ -60,8 +60,8 @@ RUN conda install --quiet --yes \
     'r-rcurl=1.98*' \
     'r-reshape2=1.4*' \
     'r-rmarkdown=2.1*' \
-    'r-rsqlite=2.1*' \
-    'r-shiny=1.3*' \
+    'r-rsqlite=2.2*' \
+    'r-shiny=1.4*' \
     'r-tidyverse=1.3*' \
     'rpy2=3.1*' \
     && \

datascience-notebook/test/test_julia.py

+# Copyright (c) Jupyter Development Team.
+# Distributed under the terms of the Modified BSD License.
+import logging
+
+import pytest
+
+LOGGER = logging.getLogger(__name__)
+
+
+def test_julia(container):
+    """Basic julia test"""
+    LOGGER.info(f"Test that julia is correctly installed ...")
+    running_container = container.run(
+        tty=True, command=["start.sh", "bash", "-c", "sleep infinity"]
+    )
+    command = f"julia --version"
+    cmd = running_container.exec_run(command)
+    output = cmd.output.decode("utf-8")
+    assert cmd.exit_code == 0, f"Command {command} failed {output}"
+    LOGGER.debug(output)

docs/using/recipes.md

@@ -112,7 +112,7 @@ ARG conda_env=python36
 ARG py_ver=3.6
 
 # you can add additional libraries you want conda to install by listing them below the first line and ending with "&& \"
-RUN conda env create --quiet --yes -p $CONDA_DIR/envs/$conda_env python=$py_ver ipython ipykernel && \
+RUN conda create --quiet --yes -p $CONDA_DIR/envs/$conda_env python=$py_ver ipython ipykernel && \
     conda clean --all -f -y
 
 # alternatively, you can comment out the lines above and uncomment those below

docs/using/selecting.md

@@ -24,7 +24,7 @@ repository. The following sections describe these images including their content
 
 `jupyter/base-notebook` is a small image supporting the [options common across all core stacks](common.md). It is the basis for all other stacks.
 
-* Minimally-functional Jupyter Notebook server (e.g., no [pandoc](https://pandoc.org/) for saving notebooks as PDFs)
+* Minimally-functional Jupyter Notebook server (e.g., no LaTeX support for saving notebooks as PDFs)
 * [Miniconda](https://conda.io/miniconda.html) Python 3.x in `/opt/conda`
 * No preinstalled scientific computing packages
 * Unprivileged user `jovyan` (`uid=1000`, configurable, see options) in group `users` (`gid=100`) with ownership over the `/home/jovyan` and `/opt/conda` paths
@@ -42,9 +42,8 @@ repository. The following sections describe these images including their content
 `jupyter/minimal-notebook` adds command line tools useful when working in Jupyter applications.
 
 * Everything in `jupyter/base-notebook`
-* [Pandoc](http://pandoc.org) and [TeX Live](https://www.tug.org/texlive/) for notebook document conversion
-* [git](https://git-scm.com/), [emacs](https://www.gnu.org/software/emacs/), [jed](https://www.jedsoft.org/jed/), [nano](https://www.nano-editor.org/), tzdata, and
-unzip
+* [TeX Live](https://www.tug.org/texlive/) for notebook document conversion
+* [git](https://git-scm.com/), [emacs](https://www.gnu.org/software/emacs/) (actually `emacs-nox`), [vi](https://vim.org/) (actually `vim-tiny`), [jed](https://www.jedsoft.org/jed/), [nano](https://www.nano-editor.org/), tzdata, and unzip
 
 ### jupyter/r-notebook
 
@@ -77,13 +76,13 @@ packages from [conda-forge](https://conda-forge.github.io/feedstocks)
 `jupyter/scipy-notebook` includes popular packages from the scientific Python ecosystem.
 
 * Everything in `jupyter/minimal-notebook` and its ancestor images
-* [pandas](https://pandas.pydata.org/), [numexpr](https://github.com/pydata/numexpr), [matplotlib](https://matplotlib.org/), [scipy](https://www.scipy.org/),
+* [dask](https://dask.org/), [pandas](https://pandas.pydata.org/), [numexpr](https://github.com/pydata/numexpr), [matplotlib](https://matplotlib.org/), [scipy](https://www.scipy.org/),
 [seaborn](https://seaborn.pydata.org/), [scikit-learn](http://scikit-learn.org/stable/), [scikit-image](http://scikit-image.org/),
 [sympy](http://www.sympy.org/en/index.html), [cython](http://cython.org/), [patsy](https://patsy.readthedocs.io/en/latest/),
 [statsmodel](http://www.statsmodels.org/stable/index.html), [cloudpickle](https://github.com/cloudpipe/cloudpickle), [dill](https://pypi.python.org/pypi/dill),
 [numba](https://numba.pydata.org/), [bokeh](https://bokeh.pydata.org/en/latest/), [sqlalchemy](https://www.sqlalchemy.org/), [hdf5](http://www.h5py.org/),
 [vincent](http://vincent.readthedocs.io/en/latest/), [beautifulsoup](https://www.crummy.com/software/BeautifulSoup/),
-[protobuf](https://developers.google.com/protocol-buffers/docs/pythontutorial), and [xlrd](http://www.python-excel.org/) packages
+[protobuf](https://developers.google.com/protocol-buffers/docs/pythontutorial), [xlrd](http://www.python-excel.org/), [bottleneck](https://bottleneck.readthedocs.io/en/latest/), and [pytables](https://www.pytables.org/) packages
 * [ipywidgets](https://ipywidgets.readthedocs.io/en/stable/) and [ipympl](https://github.com/matplotlib/jupyter-matplotlib) for interactive visualizations and plots in Python notebooks
 * [Facets](https://github.com/PAIR-code/facets) for visualizing machine learning datasets
 
@@ -183,8 +182,7 @@ the datascience-notebook image. Click here to launch it on
 
   `crosscompass/ihaskell-notebook` is based on [IHaskell](https://github.com/gibiansky/IHaskell). Includes popular packages and example notebooks.
 
-  Try it on binder: [![launch Learn You a Haskell for Great
-Good!](https://mybinder.org/badge_logo.svg)](https://mybinder.org/v2/gh/jamesdbrock/learn-you-a-haskell-notebook/master?urlpath=lab/tree/learn_you_a_haskell/00-preface.ipynb)
+  Try it on [![Binder](https://mybinder.org/badge_logo.svg)](https://mybinder.org/v2/gh/jamesdbrock/learn-you-a-haskell-notebook/master?urlpath=lab/tree/ihaskell_examples/ihaskell/IHaskell.ipynb)
 
 * [java-notebook is a community Jupyter Docker Stack image](https://github.com/jbindinga/java-notebook). The image includes
 [IJava](https://github.com/SpencerPark/IJava) kernel on top of the minimal-notebook image. Click here to launch it on

minimal-notebook/Dockerfile

@@ -10,7 +10,8 @@ USER root
 # Install all OS dependencies for fully functional notebook server
 RUN apt-get update && apt-get install -yq --no-install-recommends \
     build-essential \
-    emacs \
+    emacs-nox \
+    vim-tiny \
     git \
     inkscape \
     jed \

minimal-notebook/test/test_nbconvert.py

@@ -16,7 +16,7 @@ def test_nbconvert(container, format):
     cont_data_dir = "/home/jovyan/data"
     test_file = "notebook1"
     output_dir = "/tmp"
-    LOGGER.info(f"Converting example notebook to {format.upper()} ...")
+    LOGGER.info(f"Test that an example notebook can be converted to {format.upper()} ...")
     command = f"jupyter nbconvert {cont_data_dir}/{test_file}.ipynb --output-dir {output_dir} --to {format}"
     c = container.run(
         volumes={host_data_dir: {"bind": cont_data_dir, "mode": "ro"}},
@@ -24,8 +24,8 @@ def test_nbconvert(container, format):
         command=["start.sh", "bash", "-c", command],
     )
     rv = c.wait(timeout=30)
-    assert rv == 0 or rv["StatusCode"] == 0
+    assert rv == 0 or rv["StatusCode"] == 0, f"Command {command} failed"
     logs = c.logs(stdout=True).decode("utf-8")
     LOGGER.debug(logs)
-    assert f"{output_dir}/{test_file}.{format}" in logs
-
+    expected_file = f"{output_dir}/{test_file}.{format}"
+    assert expected_file in logs, f"Expected file {expected_file} not generated"

r-notebook/Dockerfile

@@ -25,11 +25,11 @@ USER $NB_UID
 
 # R packages
 RUN conda install --quiet --yes \
-    'r-base=3.6.2' \
+    'r-base=3.6.3' \
     'r-caret=6.0*' \
     'r-crayon=1.3*' \
-    'r-devtools=2.2*' \
-    'r-forecast=8.11*' \
+    'r-devtools=2.3*' \
+    'r-forecast=8.12*' \
     'r-hexbin=1.28*' \
     'r-htmltools=0.4*' \
     'r-htmlwidgets=1.5*' \
@@ -41,7 +41,7 @@ RUN conda install --quiet --yes \
     'r-reshape2=1.4*' \
     'r-rmarkdown=2.1*' \
     'r-rodbc=1.3*' \
-    'r-rsqlite=2.1*' \
+    'r-rsqlite=2.2*' \
     'r-shiny=1.4*' \
     'r-tidyverse=1.3*' \
     'unixodbc=2.3.*' \

scipy-notebook/Dockerfile

@@ -16,23 +16,26 @@ USER $NB_UID
 
 # Install Python 3 packages
 RUN conda install --quiet --yes \
-    'beautifulsoup4=4.8.*' \
+    'beautifulsoup4=4.9.*' \
     'conda-forge::blas=*=openblas' \
-    'bokeh=1.4.*' \
-    'cloudpickle=1.3.*' \
+    'bokeh=2.0.*' \
+    'bottleneck=1.3.*' \
+    'cloudpickle=1.4.*' \
     'cython=0.29.*' \
-    'dask=2.11.*' \
+    'dask=2.15.*' \
     'dill=0.3.*' \
     'h5py=2.10.*' \
     'hdf5=1.10.*' \
     'ipywidgets=7.5.*' \
     'ipympl=0.5.*'\
-    'matplotlib-base=3.1.*' \
+    'matplotlib-base=3.2.*' \
+    # numba update to 0.49 fails resolving deps.
     'numba=0.48.*' \
     'numexpr=2.7.*' \
     'pandas=1.0.*' \
     'patsy=0.5.*' \
     'protobuf=3.11.*' \
+    'pytables=3.6.*' \
     'scikit-image=0.16.*' \
     'scikit-learn=0.22.*' \
     'scipy=1.4.*' \
@@ -42,7 +45,7 @@ RUN conda install --quiet --yes \
     'sympy=1.5.*' \
     'vincent=0.4.*' \
     'widgetsnbextension=3.5.*'\
-    'xlrd' \
+    'xlrd=1.2.*' \
     && \
     conda clean --all -f -y && \
     # Activate ipywidgets extension in the environment that runs the notebook server
@@ -53,8 +56,8 @@ RUN conda install --quiet --yes \
     jupyter labextension install @jupyter-widgets/jupyterlab-manager@^2.0.0 --no-build && \
     jupyter labextension install @bokeh/jupyter_bokeh@^2.0.0 --no-build && \
     jupyter labextension install jupyter-matplotlib@^0.7.2 --no-build && \
-    jupyter lab build && \
-    jupyter lab clean && \
+    jupyter lab build -y && \
+    jupyter lab clean -y && \
     npm cache clean --force && \
     rm -rf /home/$NB_USER/.cache/yarn && \
     rm -rf /home/$NB_USER/.node-gyp && \

scipy-notebook/test/data/matplotlib_1.py

+# Matplotlit: Create a simple plot example.
+# Refs: https://matplotlib.org/3.1.1/gallery/lines_bars_and_markers/simple_plot.html
+
+# Optional test with [Matplotlib Jupyter Integration](https://github.com/matplotlib/ipympl)
+# %matplotlib widget
+import matplotlib
+import matplotlib.pyplot as plt
+import numpy as np
+import os
+
+# Data for plotting
+t = np.arange(0.0, 2.0, 0.01)
+s = 1 + np.sin(2 * np.pi * t)
+
+fig, ax = plt.subplots()
+ax.plot(t, s)
+
+ax.set(xlabel='time (s)', ylabel='voltage (mV)',
+       title='About as simple as it gets, folks')
+ax.grid()
+# Note that the test can be run headless by checking if an image is produced
+file_path = os.path.join("/tmp", "test.png")
+fig.savefig(file_path)
+print(f"File {file_path} saved")
\ No newline at end of file

scipy-notebook/test/test_matplotlib.py

+# Copyright (c) Jupyter Development Team.
+# Distributed under the terms of the Modified BSD License.
+
+import logging
+
+import pytest
+import os
+
+LOGGER = logging.getLogger(__name__)
+
+
+def test_matplotlib(container):
+    """Test that matplotlib is able to plot a graph and write it as an image"""
+    host_data_dir = os.path.join(os.path.dirname(os.path.realpath(__file__)), "data")
+    cont_data_dir = "/home/jovyan/data"
+    test_file = "matplotlib_1.py"
+    output_dir = "/tmp"
+    LOGGER.info(f"Test that matplotlib is able to plot a graph and write it as an image ...")
+    command = "sleep infinity"
+    running_container = container.run(
+        volumes={host_data_dir: {"bind": cont_data_dir, "mode": "ro"}},
+        tty=True,
+        command=["start.sh", "bash", "-c", command],
+    )
+    command = f"python {cont_data_dir}/{test_file}"
+    cmd = running_container.exec_run(command)
+    assert cmd.exit_code == 0, f"Command {command} failed"
+    LOGGER.debug(cmd.output.decode("utf-8"))
+    # Checking if the file is generated
+    # https://stackoverflow.com/a/15895594/4413446
+    expected_file = f"{output_dir}/test.png"
+    command = f"test -s {expected_file}"
+    cmd = running_container.exec_run(command)
+    assert cmd.exit_code == 0, f"Command {command} failed"
+    LOGGER.debug(cmd.output.decode("utf-8"))

scipy-notebook/test/test_pandas.py

+# Copyright (c) Jupyter Development Team.
+# Distributed under the terms of the Modified BSD License.
+import logging
+
+import pytest
+
+LOGGER = logging.getLogger(__name__)
+
+
+@pytest.mark.parametrize(
+    "name,command",
+    [
+        (
+            "Sum series",
+            "import pandas as pd; import numpy as np; np.random.seed(0); print(pd.Series(np.random.randint(0, 7, size=10)).sum())",
+        ),
+    ],
+)
+def test_pandas(container, name, command):
+    """Basic pandas tests"""
+    LOGGER.info(f"Testing pandas: {name} ...")
+    c = container.run(tty=True, command=["start.sh", "python", "-c", command])
+    rv = c.wait(timeout=30)
+    assert rv == 0 or rv["StatusCode"] == 0, f"Command {command} failed"
+    logs = c.logs(stdout=True).decode("utf-8")
+    LOGGER.debug(logs)

tensorflow-notebook/test/test_tensorflow.py

+# Copyright (c) Jupyter Development Team.
+# Distributed under the terms of the Modified BSD License.
+import logging
+
+import pytest
+
+LOGGER = logging.getLogger(__name__)
+
+
+@pytest.mark.parametrize(
+    "name,command",
+    [
+        (
+            "Hello world",
+            "import tensorflow as tf;print(tf.constant('Hello, TensorFlow'))",
+        ),
+        (
+            "Sum",
+            "import tensorflow as tf;print(tf.reduce_sum(tf.random.normal([1000, 1000])))",
+        ),
+    ],
+)
+def test_tensorflow(container, name, command):
+    """Basic tensorflow tests"""
+    LOGGER.info(f"Testing tensorflow: {name} ...")
+    c = container.run(tty=True, command=["start.sh", "python", "-c", command])
+    rv = c.wait(timeout=30)
+    assert rv == 0 or rv["StatusCode"] == 0, f"Command {command} failed"
+    logs = c.logs(stdout=True).decode("utf-8")
+    LOGGER.debug(logs)

test/helpers.py

@@ -87,7 +87,11 @@ class CondaPackageHelper:
     @staticmethod
     def _packages_from_json(env_export):
         """Extract packages and versions from the lines returned by the list of specifications"""
+        #dependencies = filter(lambda x:  isinstance(x, str), json.loads(env_export).get("dependencies"))
         dependencies = json.loads(env_export).get("dependencies")
+        # Filtering packages installed through pip in this case it's a dict {'pip': ['toree==0.3.0']}
+        # Since we only manage packages installed through conda here
+        dependencies = filter(lambda x:  isinstance(x, str), dependencies)
         packages_dict = dict()
         for split in map(lambda x: x.split("=", 1), dependencies):
             # default values

test/test_notebook.py

 # Copyright (c) Jupyter Development Team.
 # Distributed under the terms of the Modified BSD License.
 
+
 def test_secured_server(container, http_client):
     """Notebook server should eventually request user login."""
     container.run()
-    resp = http_client.get('http://localhost:8888')
+    resp = http_client.get("http://localhost:8888")
     resp.raise_for_status()
-    assert 'login_submit' in resp.text
+    assert "login_submit" in resp.text, "User login not requested"

test/test_packages.py

@@ -21,7 +21,7 @@ Example:
     $ make test/datascience-notebook
 
     # [...]
-    # test/test_packages.py::test_python_packages 
+    # test/test_packages.py::test_python_packages
     # --------------------------------------------------------------------------------------------- live log setup ----------------------------------------------------------------------------------------------
     # 2020-03-08 09:56:04 [    INFO] Starting container jupyter/datascience-notebook ... (helpers.py:51)
     # 2020-03-08 09:56:04 [    INFO] Running jupyter/datascience-notebook with args {'detach': True, 'ports': {'8888/tcp': 8888}, 'tty': True, 'command': ['start.sh', 'bash', '-c', 'sleep infinity']} ... (conftest.py:78)
@@ -51,6 +51,7 @@ PACKAGE_MAPPING = {
     "scikit-learn": "sklearn",
     "scikit-image": "skimage",
     "spylon-kernel": "spylon_kernel",
+    "pytables": "tables",
     # R
     "randomforest": "randomForest",
     "rsqlite": "DBI",