Merge branch 'master' into patch-2

Makefile

@@ -68,6 +68,12 @@ n-docs-diff: ## number of docs/ files changed since branch from master
 n-other-diff: ## number of files outside docs/ changed since branch from master
 	@git diff --name-only $(DIFF_RANGE) -- ':!docs/' | wc -l | awk '{print $$1}'
 
+run/%: ## run a bash in interactive mode in a stack
+	docker run -it --rm $(OWNER)/$(notdir $@) $(SHELL)
+
+run-sudo/%: ## run a bash in interactive mode as root in a stack
+	docker run -it --rm -u root $(OWNER)/$(notdir $@) $(SHELL)
+
 tx-en: ## rebuild en locale strings and push to master (req: GH_TOKEN)
 	@git config --global user.email "travis@travis-ci.org"
 	@git config --global user.name "Travis CI"
@@ -82,9 +88,5 @@ tx-en: ## rebuild en locale strings and push to master (req: GH_TOKEN)
 	@git remote add origin-tx https://$${GH_TOKEN}@github.com/jupyter/docker-stacks.git
 	@git push -u origin-tx master
 
-
-test/%: ## run tests against a stack
-	@TEST_IMAGE="$(OWNER)/$(notdir $@)" pytest test
-
-test/base-notebook: ## test supported options in the base notebook
-	@TEST_IMAGE="$(OWNER)/$(notdir $@)" pytest test base-notebook/test
+test/%: ## run tests against a stack (only common tests or common tests + specific tests)
+	@if [ ! -d "$(notdir $@)/test" ]; then TEST_IMAGE="$(OWNER)/$(notdir $@)" pytest test; else TEST_IMAGE="$(OWNER)/$(notdir $@)" pytest test $(notdir $@)/test; fi

all-spark-notebook/Dockerfile

@@ -23,11 +23,11 @@ USER $NB_UID
 
 # R packages
 RUN conda install --quiet --yes \
-    'r-base=3.6.1' \
+    'r-base=3.6.2' \
     'r-ggplot2=3.2*' \
-    'r-irkernel=1.0*' \
-    'r-rcurl=1.95*' \
-    'r-sparklyr=1.0*' \
+    'r-irkernel=1.1*' \
+    'r-rcurl=1.98*' \
+    'r-sparklyr=1.1*' \
     && \
     conda clean --all -f -y && \
     fix-permissions $CONDA_DIR && \

base-notebook/test/test_container_options.py

@@ -78,6 +78,20 @@ def test_chown_extra(container):
     assert '/opt/conda/LICENSE.txt:1010:101' in c.logs(stdout=True).decode('utf-8')
 
 
+def test_chown_home(container):
+    """Container should change the NB_USER home directory owner and 
+    group to the current value of NB_UID and NB_GID."""
+    c = container.run(
+        tty=True,
+        user='root',
+        environment=['CHOWN_HOME=yes',
+                     'CHOWN_HOME_OPTS=-R',
+        ],
+        command=['start.sh', 'bash', '-c', 'chown root:root /home/jovyan && ls -alsh /home']
+    )
+    assert "Changing ownership of /home/jovyan to 1000:100 with options '-R'" in c.logs(stdout=True).decode('utf-8')
+
+
 def test_sudo(container):
     """Container should grant passwordless sudo to the default user."""
     c = container.run(

conftest.py

 # Copyright (c) Jupyter Development Team.
 # Distributed under the terms of the Modified BSD License.
 import os
+import logging
 
 import docker
 import pytest
@@ -10,6 +11,8 @@ from requests.packages.urllib3.util.retry import Retry
 from requests.adapters import HTTPAdapter
 
 
+LOGGER = logging.getLogger(__name__)
+
 @pytest.fixture(scope='session')
 def http_client():
     """Requests session with retries and backoff."""
@@ -72,9 +75,10 @@ class TrackedContainer(object):
         all_kwargs = {}
         all_kwargs.update(self.kwargs)
         all_kwargs.update(kwargs)
+        LOGGER.info(f"Running {self.image_name} with args {all_kwargs} ...")
         self.container = self.docker_client.containers.run(self.image_name, **all_kwargs)
         return self.container
-
+    
     def remove(self):
         """Kills and removes the tracked docker container."""
         if self.container:

datascience-notebook/Dockerfile

@@ -45,26 +45,25 @@ USER $NB_UID
 
 # R packages including IRKernel which gets installed globally.
 RUN conda install --quiet --yes \
-    'r-base=3.6.1' \
+    'r-base=3.6.2' \
     'r-caret=6.0*' \
     'r-crayon=1.3*' \
-    'r-devtools=2.1*' \
-    'r-forecast=8.7*' \
-    'r-hexbin=1.27*' \
-    'r-htmltools=0.3*' \
-    'r-htmlwidgets=1.3*' \
-    'r-irkernel=1.0*' \
+    'r-devtools=2.2*' \
+    'r-forecast=8.10*' \
+    'r-hexbin=1.28*' \
+    'r-htmltools=0.4*' \
+    'r-htmlwidgets=1.5*' \
+    'r-irkernel=1.1*' \
     'r-nycflights13=1.0*' \
     'r-plyr=1.8*' \
     'r-randomforest=4.6*' \
-    'r-rcurl=1.95*' \
+    'r-rcurl=1.98*' \
     'r-reshape2=1.4*' \
-    'r-rmarkdown=1.14*' \
+    'r-rmarkdown=2.1*' \
     'r-rsqlite=2.1*' \
     'r-shiny=1.3*' \
-    'r-sparklyr=1.0*' \
-    'r-tidyverse=1.2*' \
-    'rpy2=2.9*' \
+    'r-tidyverse=1.3*' \
+    'rpy2=3.1*' \
     && \
     conda clean --all -f -y && \
     fix-permissions $CONDA_DIR && \

datascience-notebook/hooks/apply_tags

+#!/bin/bash
+set -e
+
+# Tag the latest build with the short git sha as well as version of key runtimes
+# and packages. 
+GIT_SHA_TAG=${SOURCE_COMMIT:0:12}
+docker tag $IMAGE_NAME "$DOCKER_REPO:$GIT_SHA_TAG"
+PY_VERSION_TAG="python-$(docker run --rm ${IMAGE_NAME} python --version 2>&1 | awk '{print $2}')"
+docker tag $IMAGE_NAME "$DOCKER_REPO:$PY_VERSION_TAG"
+R_VERSION_TAG="r-$(docker run --rm -a STDOUT ${IMAGE_NAME} R --version | sed -n 1p | awk '{print $3}')"
+docker tag $IMAGE_NAME "$DOCKER_REPO:${R_VERSION_TAG%%\r}"
+JULIA_VERSION_TAG="julia-$(docker run --rm -a STDOUT ${IMAGE_NAME} julia --version | awk '{print $3}')"
+docker tag $IMAGE_NAME "$DOCKER_REPO:${JULIA_VERSION_TAG%%\r}"
+NB_VERSION_TAG="notebook-$(docker run --rm -a STDOUT ${IMAGE_NAME} jupyter-notebook --version | tr -d '\r')"
+docker tag $IMAGE_NAME "$DOCKER_REPO:${NB_VERSION_TAG%% }"
+LAB_VERSION_TAG="lab-$(docker run --rm -a STDOUT ${IMAGE_NAME} jupyter-lab --version | tr -d '\r')"
+docker tag $IMAGE_NAME "$DOCKER_REPO:${LAB_VERSION_TAG%%\r}"
+HUB_VERSION_TAG="hub-$(docker run --rm -a STDOUT ${IMAGE_NAME} jupyterhub --version | tr -d '\r')"
+docker tag $IMAGE_NAME "$DOCKER_REPO:${HUB_VERSION_TAG%%\r}"

datascience-notebook/hooks/index.tmpl

-INDEX_ROW="|\`${BUILD_TIMESTAMP}\`|\`jupyter/${IMAGE_SHORT_NAME}:${GIT_SHA_TAG}\`|[Git diff](https://github.com/jupyter/docker-stacks/commit/${SOURCE_COMMIT})<br />[Dockerfile](https://github.com/jupyter/docker-stacks/blob/${SOURCE_COMMIT}/${IMAGE_SHORT_NAME}/Dockerfile)<br />[Build manifest](./${IMAGE_SHORT_NAME}-${GIT_SHA_TAG})|"
+INDEX_ROW="|\`${BUILD_TIMESTAMP}\`|\`jupyter/${IMAGE_SHORT_NAME}:${GIT_SHA_TAG}\`<br />\`jupyter/${IMAGE_SHORT_NAME}:${PY_VERSION_TAG}\`<br />\`jupyter/${IMAGE_SHORT_NAME}:${JULIA_VERSION_TAG}\`<br />\`jupyter/${IMAGE_SHORT_NAME}:${R_VERSION_TAG}\`<br />\`jupyter/${IMAGE_SHORT_NAME}:${NB_VERSION_TAG}\`<br />\`jupyter/${IMAGE_SHORT_NAME}:${LAB_VERSION_TAG}\`<br />\`jupyter/${IMAGE_SHORT_NAME}:${HUB_VERSION_TAG}\`|[Git diff](https://github.com/jupyter/docker-stacks/commit/${SOURCE_COMMIT})<br />[Dockerfile](https://github.com/jupyter/docker-stacks/blob/${SOURCE_COMMIT}/${IMAGE_SHORT_NAME}/Dockerfile)<br />[Build manifest](./${IMAGE_SHORT_NAME}-${GIT_SHA_TAG})|"
 sed "/|-|/a ${INDEX_ROW}" -i "$INDEX_FILE"
\ No newline at end of file

datascience-notebook/hooks/post_push

 #!/bin/bash
 set -e
 
+# Apply and push all tags
+source hooks/apply_tags
+docker push $DOCKER_REPO
+
 # Tag the latest build with the short git sha. Push the tag in addition
 # to the "latest" tag already pushed.
 GIT_SHA_TAG=${SOURCE_COMMIT:0:12}
@@ -48,4 +52,4 @@ set +e
 for url in $(echo $NEXT_BUILD_TRIGGERS | sed "s/,/ /g")
 do
     curl -X POST $url
-done
\ No newline at end of file
+done

docs/locale/pt_BR/LC_MESSAGES/index.po

 # SOME DESCRIPTIVE TITLE.
 # Copyright (C) 2018- Project Jupyter
-# This file is distributed under the same license as the docker-stacks package.
-# FIRST AUTHOR <EMAIL@ADDRESS>, YEAR.
+# This file is distributed under the same license as the docker-stacks
+# package.
+# FIRST AUTHOR <EMAIL@ADDRESS>, 2019.
 # 
 # Translators:
-# Allan Neri Fernandes da Silva <allan.nerifs@gmail.com>, 2019
+# Allan Neri Fernandes da Silva <allan.nerifs@gmail.com>, 2020
 # 
 #, fuzzy
 msgid ""
 msgstr ""
 "Project-Id-Version: docker-stacks latest\n"
 "Report-Msgid-Bugs-To: \n"
-"POT-Creation-Date: 2019-04-21 17:25-0400\n"
+"POT-Creation-Date: 2020-01-20 04:43+0000\n"
 "PO-Revision-Date: 2019-04-22 13:45+0000\n"
-"Last-Translator: Allan Neri Fernandes da Silva <allan.nerifs@gmail.com>, 2019\n"
+"Last-Translator: Allan Neri Fernandes da Silva <allan.nerifs@gmail.com>, 2020\n"
 "Language-Team: Portuguese (Brazil) (https://www.transifex.com/project-jupyter/teams/97886/pt_BR/)\n"
 "MIME-Version: 1.0\n"
 "Content-Type: text/plain; charset=UTF-8\n"
 "Content-Transfer-Encoding: 8bit\n"
+"Generated-By: Babel 2.8.0\n"
 "Language: pt_BR\n"
 "Plural-Forms: nplurals=2; plural=(n > 1);\n"
 
-# 85291a2c92d440a089cf156fba58b86a
-#: ../../index.rst:32 ../../index.rst:32
+# 22f1bd46933144e092bf92e3af4c6f4f
+#: /home/travis/build/jupyter/docker-stacks/docs/index.rst:32
+#: 79072cbf86294c09b9313ee07735fb65
 msgid "User Guide"
 msgstr "Guia de uso"
 
-# e4a3fd32059d4486b6b0f3a6fb5065b5
-#: ../../index.rst:42 ../../index.rst:42
+# f35d75046f8c42ae8cab58d826154823
+#: /home/travis/build/jupyter/docker-stacks/docs/index.rst:42
+#: c9e3b347063f4b528690011606e8d5ea
 msgid "Contributor Guide"
 msgstr "Guia de contribuição"
 
-# 45de48c8617d4ebe83b7ff7141295836
-#: ../../index.rst:53 ../../index.rst:53
+# a737afe726cd49c4986d75b7d74eeed3
+#: /home/travis/build/jupyter/docker-stacks/docs/index.rst:54
+#: c7b53fa9956546d691788706b3ef5dfc
 msgid "Maintainer Guide"
 msgstr "Guia de manutenção"
 
-# aa99c4562d8d4cb3abfea375470e8af3
-#: ../../index.rst:59 ../../index.rst:59
+#: /home/travis/build/jupyter/docker-stacks/docs/index.rst:60
+msgid "Jupyter Discourse Forum"
+msgstr "Fórum do Jupyter"
+
+#: /home/travis/build/jupyter/docker-stacks/docs/index.rst:60
+msgid "Jupyter Docker Stacks Issue Tracker"
+msgstr "Rastreamento de problemas nos containers Docker"
+
+#: /home/travis/build/jupyter/docker-stacks/docs/index.rst:60
+msgid "Jupyter Website"
+msgstr "Website do Jupyter"
+
+# 9cd216fa91ef40bbb957373faaf93732
+#: /home/travis/build/jupyter/docker-stacks/docs/index.rst:60
+#: 2789eaad173a43a495ff17fd0e1a1a38
 msgid "Getting Help"
 msgstr "Obtendo ajuda"
 
-# 76df09037a6147c8a85be8ef14c15532
-#: ../../index.rst:2
+# a0aa0bcd999c4c5e96cc57fd77780f96
+#: /home/travis/build/jupyter/docker-stacks/docs/index.rst:2
+#: 121c8abde123400bbdb190b01441a180
 msgid "Jupyter Docker Stacks"
 msgstr "Jupyter Docker Stacks"
 
-# b3fb253184e64174b26a2da73f221cc1
-#: ../../index.rst:4
+# 5d06f458dc524214b2c97e865dd2dc81
+#: /home/travis/build/jupyter/docker-stacks/docs/index.rst:4
+#: 6463d955c7724db682f6fa42da6b25a7
 msgid ""
 "Jupyter Docker Stacks are a set of ready-to-run Docker images containing "
 "Jupyter applications and interactive computing tools. You can use a stack "
 "image to do any of the following (and more):"
 msgstr ""
-"Jupyter Docker Stacks são um conjunto de imagens Docker prontas para uso "
+"Jupyter Docker Stacks é um conjunto de imagens Docker prontas para uso "
 "contendo aplicações Jupyter e ferramentas interativas. Você pode usar uma "
 "pilha de imagens para fazer qualquer uma dessas coisas (e muito mais):"
 
-# 8cd55fb113a049e7be445b21084df7ea
-#: ../../index.rst:6
+# c69f151c806e4cdf9bebda05b06c760e
+#: /home/travis/build/jupyter/docker-stacks/docs/index.rst:6
+#: 417a2a71d6bd4afdba0c10d1824afa36
 msgid "Start a personal Jupyter Notebook server in a local Docker container"
 msgstr "Rodar um servidor Jupyter Notebook em um container Docker local"
 
-# 285fa13bed8547b4af0cfc480adf4861
-#: ../../index.rst:7
+# b26271409ab743b2a349b3a8ca95233e
+#: /home/travis/build/jupyter/docker-stacks/docs/index.rst:7
+#: 318b7b2a1f4644048ce7deb74fc8a2cf
 msgid "Run JupyterLab servers for a team using JupyterHub"
-msgstr "Rode um servidor JupyterLab para uma equipe usando JupyterHub"
+msgstr "Rodar um servidor JupyterLab para uma equipe usando JupyterHub"
 
-# a49d31cedd4946ea98d7a4c720340d84
-#: ../../index.rst:8
+# 4d60f4325fff4ffcad12703a4b9d6781
+#: /home/travis/build/jupyter/docker-stacks/docs/index.rst:8
+#: faebaa8b57f24f52b0873a12b4da2a62
 msgid "Write your own project Dockerfile"
 msgstr "Escrever seu próprio Dockerfile"
 
-# 0aff0df2e46a44b4bc8070545e83c42c
-#: ../../index.rst:11
+# 78b0d31eb6e9462888eef92e6a84cdb7
+#: /home/travis/build/jupyter/docker-stacks/docs/index.rst:11
+#: 549f043c0b734a61817b2c737ac59d7c
 msgid "Quick Start"
 msgstr "Guia rápido"
 
-# 3262accd283140058db73005072be200
-#: ../../index.rst:13
+# d4c0e237dbe74e0d9afbf2b2f0e219c8
+#: /home/travis/build/jupyter/docker-stacks/docs/index.rst:13
+#: bc586127ae4b4cbba1d9709841f2135c
 msgid ""
 "You can try a `recent build of the jupyter/base-notebook image on "
 "mybinder.org <https://mybinder.org/v2/gh/jupyter/docker-"
@@ -96,8 +121,9 @@ msgstr ""
 ":doc:`qual imagem <using/selecting>` usar, e quer iniciar um servidor "
 "Jupyter Notebook em um contêiner."
 
-# 9e4094e8de264e20bca0af8f6cf888b7
-#: ../../index.rst:15
+# 051ed23ef62e41058a7c889604f96035
+#: /home/travis/build/jupyter/docker-stacks/docs/index.rst:15
+#: 51538eb1f8d442acaae41b8e69a8704e
 msgid ""
 "The other pages in this documentation describe additional uses and features "
 "in detail."
@@ -105,8 +131,9 @@ msgstr ""
 "As próximas paginas desta documentação descrevem os usos e as "
 "funcionalidades adicionais com mais detalhes"
 
-# 8b75310f2eaa4aac95d9387e0a4db9e8
-#: ../../index.rst:17
+# e91f3b62a1b54166b966be6d7a4f061e
+#: /home/travis/build/jupyter/docker-stacks/docs/index.rst:17
+#: 0c8148b23d704a1699d2812744b20c7c
 msgid ""
 "**Example 1:** This command pulls the ``jupyter/scipy-notebook`` image "
 "tagged ``17aba6048f44`` from Docker Hub if it is not already present on the "
@@ -128,8 +155,9 @@ msgstr ""
 " token secreto exibido no console. O contêiner permanece intacto para "
 "reinicialização após o termino do servidor.::"
 
-# 9e0229c99f404aceb7e517387ae88365
-#: ../../index.rst:21
+# e04140e6cd8442f7a6f347d88224f591
+#: /home/travis/build/jupyter/docker-stacks/docs/index.rst:21
+#: dcbbce6e5e67473aa32e264e422f334f
 msgid ""
 "**Example 2:** This command performs the same operations as **Example 1**, "
 "but it exposes the server on host port 10000 instead of port 8888. Visiting "
@@ -143,8 +171,9 @@ msgstr ""
 " o JupyterLab, onde ``hostname`` é o nome do computador executando o docker "
 "e ``token`` é o token secreto exibido no console.::"
 
-# 02c25d3f0abe48348100dd878c243a75
-#: ../../index.rst:25
+# 1c3229680cf44a5bb2d8450602bfcf7d
+#: /home/travis/build/jupyter/docker-stacks/docs/index.rst:25
+#: 449c01c1808b427381502b0d33f4efcb
 msgid ""
 "**Example 3:** This command pulls the ``jupyter/datascience-notebook`` image"
 " tagged ``9b06df75e445`` from Docker Hub if it is not already present on the"
@@ -171,7 +200,8 @@ msgstr ""
 "mas os arquivos escritos em ``~work`` do contêiner permanecem intactos no "
 "host.::"
 
-# 1bc0c7d54cd343d689362047c0b00122
-#: ../../index.rst:30
+# 3ac1a41d185844b1b43315a4cc74efc8
+#: /home/travis/build/jupyter/docker-stacks/docs/index.rst:30
+#: 3e1e8e2674784f5caad20d9c110707c5
 msgid "Table of Contents"
 msgstr "Índice"

pyspark-notebook/Dockerfile

@@ -8,16 +8,18 @@ LABEL maintainer="Jupyter Project <jupyter@googlegroups.com>"
 USER root
 
 # Spark dependencies
-ENV APACHE_SPARK_VERSION 2.4.4
-ENV HADOOP_VERSION 2.7
+ENV APACHE_SPARK_VERSION=2.4.5 \
+    HADOOP_VERSION=2.7
 
 RUN apt-get -y update && \
     apt-get install --no-install-recommends -y openjdk-8-jre-headless ca-certificates-java && \
     rm -rf /var/lib/apt/lists/*
 
+# Using the preferred mirror to download the file
 RUN cd /tmp && \
-    wget -q http://mirrors.ukfast.co.uk/sites/ftp.apache.org/spark/spark-${APACHE_SPARK_VERSION}/spark-${APACHE_SPARK_VERSION}-bin-hadoop${HADOOP_VERSION}.tgz && \
-    echo "2E3A5C853B9F28C7D4525C0ADCB0D971B73AD47D5CCE138C85335B9F53A6519540D3923CB0B5CEE41E386E49AE8A409A51AB7194BA11A254E037A848D0C4A9E5 *spark-${APACHE_SPARK_VERSION}-bin-hadoop${HADOOP_VERSION}.tgz" | sha512sum -c - && \
+    wget -q $(wget -qO- https://www.apache.org/dyn/closer.lua/spark/spark-${APACHE_SPARK_VERSION}/spark-${APACHE_SPARK_VERSION}-bin-hadoop${HADOOP_VERSION}.tgz\?as_json | \
+    python -c "import sys, json; content=json.load(sys.stdin); print(content['preferred']+content['path_info'])") && \
+    echo "2426a20c548bdfc07df288cd1d18d1da6b3189d0b78dee76fa034c52a4e02895f0ad460720c526f163ba63a17efae4764c46a1cd8f9b04c60f9937a554db85d2 *spark-${APACHE_SPARK_VERSION}-bin-hadoop${HADOOP_VERSION}.tgz" | sha512sum -c - && \
     tar xzf spark-${APACHE_SPARK_VERSION}-bin-hadoop${HADOOP_VERSION}.tgz -C /usr/local --owner root --group root --no-same-owner && \
     rm spark-${APACHE_SPARK_VERSION}-bin-hadoop${HADOOP_VERSION}.tgz
 RUN cd /usr/local && ln -s spark-${APACHE_SPARK_VERSION}-bin-hadoop${HADOOP_VERSION} spark
@@ -36,10 +38,11 @@ RUN apt-get -y update && \
     rm -rf /var/lib/apt/lists/*
 
 # Spark and Mesos config
-ENV SPARK_HOME /usr/local/spark
-ENV PYTHONPATH $SPARK_HOME/python:$SPARK_HOME/python/lib/py4j-0.10.7-src.zip
-ENV MESOS_NATIVE_LIBRARY /usr/local/lib/libmesos.so
-ENV SPARK_OPTS --driver-java-options=-Xms1024M --driver-java-options=-Xmx4096M --driver-java-options=-Dlog4j.logLevel=info
+ENV SPARK_HOME=/usr/local/spark \
+    PYTHONPATH=$SPARK_HOME/python:$SPARK_HOME/python/lib/py4j-0.10.7-src.zip \
+    MESOS_NATIVE_LIBRARY=/usr/local/lib/libmesos.so \
+    SPARK_OPTS="--driver-java-options=-Xms1024M --driver-java-options=-Xmx4096M --driver-java-options=-Dlog4j.logLevel=info" \
+    PATH=$PATH:/usr/local/spark/bin
 
 USER $NB_UID
 

pyspark-notebook/test/test_spark.py

+# Copyright (c) Jupyter Development Team.
+# Distributed under the terms of the Modified BSD License.
+import time
+import logging
+
+import pytest
+
+LOGGER = logging.getLogger(__name__)
+
+def test_spark_shell(container):
+    """Checking if Spark (spark-shell) is running properly"""
+    c = container.run(
+        tty=True,
+        command=['start.sh', 'bash', '-c', 'spark-shell <<< "1+1"']
+    )
+    c.wait(timeout=30)
+    logs = c.logs(stdout=True).decode('utf-8')
+    LOGGER.debug(logs)
+    assert 'res0: Int = 2' in logs
\ No newline at end of file

pytest.ini

+[pytest]
+log_cli = 1
+log_cli_level = INFO
+log_cli_format = %(asctime)s [%(levelname)8s] %(message)s (%(filename)s:%(lineno)s)
+log_cli_date_format=%Y-%m-%d %H:%M:%S
\ No newline at end of file