Commit 3748d0f8 authored by ttimbers's avatar ttimbers

merging upstream changes

parents 07fd9af5 dd2087c7
---
ignored:
- DL3006
- DL3008
---
# .readthedocs.yml
# Read the Docs configuration file
# See https://docs.readthedocs.io/en/stable/config-file/v2.html for details
......
---
dist: bionic
language: python
python:
- 3.7
- 3.7
sudo: required
services:
- docker
- docker
jobs:
include:
include:
- stage: diff-test
install:
- pip install --upgrade pip
- make dev-env
- pip install --upgrade pip
- make dev-env
- make lint-install
script:
- set -e
- if [ $(make n-docs-diff) -ne 0 ]; then make docs; fi;
- if [ $(make n-other-diff) -ne 0 ]; then make build-test-all DARGS="--build-arg TEST_ONLY_BUILD=1"; fi;
- set -e
- if [ $(make n-docs-diff) -ne 0 ]; then make docs; fi;
- if [ $(make n-other-diff) -ne 0 ]; then make lint-build-test-all DARGS="--build-arg TEST_ONLY_BUILD=1"; fi;
- stage: push-tx
install:
- pip install --upgrade pip
- make dev-env
script:
- if [ $(make n-docs-diff DIFF_RANGE=$TRAVIS_COMMIT_RANGE) -ne 0 ]; then make tx-en; fi;
- pip install --upgrade pip
- make dev-env
script:
- if [ $(make n-docs-diff DIFF_RANGE=$TRAVIS_COMMIT_RANGE) -ne 0 ]; then make tx-en; fi;
- stage: full-test
install:
- pip install --upgrade pip
- make dev-env
- pip install --upgrade pip
- make dev-env
- make lint-install
script:
- set -e
- make docs
- make build-test-all DARGS="--build-arg TEST_ONLY_BUILD=1"
- set -e
- make docs
- make lint-build-test-all DARGS="--build-arg TEST_ONLY_BUILD=1"
stages:
- name: diff-test
if: type = pull_request
- name: push-tx
if: type = push AND branch = master
- name: full-test
if: type = cron AND branch = master
- name: diff-test
if: type = pull_request
- name: push-tx
if: type = push AND branch = master
- name: full-test
if: type = cron AND branch = master
env:
global:
# yamllint disable-line
secure: JDQKgB1laaenzmEGI9gduwK/iS0030qsl62njJg3jib0R8wxBi2OeetEdoGl9m4NFsqqnp0OO7nm4rzGfuwjL1A38flSlXOTxhjm/hvo3vmnqVu5lDcdk9+IRkafnfd3Dyd86tLxRVETOqZwCLmdNkB2fmQII8du5IIqbJuUGp8DrG7kVMf3NBr9rjkZRfByQrgG4s1NXuT61VvpWMPJAOhcrImuHBheVJDEV0U3n6Xavd7Wo+pAHrHU8exvYTZ1IzZMbHc6K0iC/NpCHcH9+9DAeLDk/q1aDNqbTExnQevOHZzNqgHC2qFOlN4jfy/TLYLpLXtUismneBBqVSK3iZso3Vqy2BRXWgouI+Tt+08ffocy9XPwEzSwkgPgDlFVUikPOy5imwjpDb13RMIyMY4CKlSOdQx2rH2kPkZ0MJJPcki3KGuGl3qRvqyblMn+lZvjAu6WVLZfo7EtcxsQ0ZZxbAbGoUVl27FHg+UvIfC0I3wEcZIp7oED47Q8s0MdCijD3AwkRPvx/iyp3J0A42su7kkOooFcmUItEIqegQJ4Aki1FBv2i5vHmBobClktytZceLsKvzCeLjMpL9HcUVfUaJDKRwtUYIozpYeBnac+E6J1s6glcqLrXIHWez8N6SzokBa6SPqdtODdzzk5OJupByub6CYWsRXvxIQ7/wI=
......@@ -24,6 +24,9 @@ endif
ALL_IMAGES:=$(ALL_STACKS)
# Linter
HADOLINT="${HOME}/hadolint"
help:
# http://marmelab.com/blog/2016/02/29/auto-documented-makefile.html
@echo "jupyter/docker-stacks"
......@@ -73,6 +76,23 @@ dev/%: ## run a foreground container for a stack
dev-env: ## install libraries required to build docs and run tests
pip install -r requirements-dev.txt
lint/%: ARGS?=
lint/%: ## lint the dockerfile(s) for a stack
@echo "Linting Dockerfiles in $(notdir $@)..."
@git ls-files --exclude='Dockerfile*' --ignored $(notdir $@) | grep -v ppc64 | xargs -L 1 $(HADOLINT) $(ARGS)
@echo "Linting done!"
lint-all: $(foreach I,$(ALL_IMAGES),lint/$(I) ) ## lint all stacks
lint-build-test-all: $(foreach I,$(ALL_IMAGES),lint/$(I) arch_patch/$(I) build/$(I) test/$(I) ) ## lint, build and test all stacks
lint-install: ## install hadolint
@echo "Installing hadolint at $(HADOLINT) ..."
@curl -sL -o $(HADOLINT) "https://github.com/hadolint/hadolint/releases/download/v1.17.6/hadolint-$(shell uname -s)-$(shell uname -m)"
@chmod 700 $(HADOLINT)
@echo "Installation done!"
@$(HADOLINT) --version
img-clean: img-rm-dang img-rm ## clean dangling and jupyter images
img-list: ## list jupyter images
......
......@@ -30,22 +30,23 @@ RUN conda install --quiet --yes \
'r-sparklyr=1.2*' \
&& \
conda clean --all -f -y && \
fix-permissions $CONDA_DIR && \
fix-permissions /home/$NB_USER
fix-permissions "${CONDA_DIR}" && \
fix-permissions "/home/${NB_USER}"
# Apache Toree kernel
# hadolint ignore=DL3013
RUN pip install --no-cache-dir \
https://dist.apache.org/repos/dist/release/incubator/toree/0.3.0-incubating/toree-pip/toree-0.3.0.tar.gz \
&& \
jupyter toree install --sys-prefix && \
rm -rf /home/$NB_USER/.local && \
fix-permissions $CONDA_DIR && \
fix-permissions /home/$NB_USER
rm -rf "/home/${NB_USER}/.local" && \
fix-permissions "${CONDA_DIR}" && \
fix-permissions "/home/${NB_USER}"
# Spylon-kernel
RUN conda install --quiet --yes 'spylon-kernel=0.4*' && \
conda clean --all -f -y && \
python -m spylon_kernel install --sys-prefix && \
rm -rf /home/$NB_USER/.local && \
fix-permissions $CONDA_DIR && \
fix-permissions /home/$NB_USER
\ No newline at end of file
rm -rf "/home/${NB_USER}/.local" && \
fix-permissions "${CONDA_DIR}" && \
fix-permissions "/home/${NB_USER}"
\ No newline at end of file
......@@ -13,6 +13,9 @@ ARG NB_USER="jovyan"
ARG NB_UID="1000"
ARG NB_GID="100"
# Fix DL4006
SHELL ["/bin/bash", "-o", "pipefail", "-c"]
USER root
# Install all OS dependencies for notebook server that starts but lacks all
......@@ -76,8 +79,8 @@ ENV MINICONDA_VERSION=4.8.2 \
MINICONDA_MD5=87e77f097f6ebb5127c77662dfc3165e \
CONDA_VERSION=4.8.2
RUN cd /tmp && \
wget --quiet https://repo.continuum.io/miniconda/Miniconda3-py37_${MINICONDA_VERSION}-Linux-x86_64.sh && \
WORKDIR /tmp
RUN wget --quiet https://repo.continuum.io/miniconda/Miniconda3-py37_${MINICONDA_VERSION}-Linux-x86_64.sh && \
echo "${MINICONDA_MD5} *Miniconda3-py37_${MINICONDA_VERSION}-Linux-x86_64.sh" | md5sum -c - && \
/bin/bash Miniconda3-py37_${MINICONDA_VERSION}-Linux-x86_64.sh -f -b -p $CONDA_DIR && \
rm Miniconda3-py37_${MINICONDA_VERSION}-Linux-x86_64.sh && \
......@@ -137,3 +140,5 @@ RUN fix-permissions /etc/jupyter/
# Switch back to jovyan to avoid accidental container runs as root
USER $NB_UID
WORKDIR $HOME
......@@ -6,14 +6,14 @@ set -e
wrapper=""
if [[ "${RESTARTABLE}" == "yes" ]]; then
wrapper="run-one-constantly"
wrapper="run-one-constantly"
fi
if [[ ! -z "${JUPYTERHUB_API_TOKEN}" ]]; then
# launched by JupyterHub, use single-user entrypoint
exec /usr/local/bin/start-singleuser.sh "$@"
# launched by JupyterHub, use single-user entrypoint
exec /usr/local/bin/start-singleuser.sh "$@"
elif [[ ! -z "${JUPYTER_ENABLE_LAB}" ]]; then
. /usr/local/bin/start.sh $wrapper jupyter lab "$@"
. /usr/local/bin/start.sh $wrapper jupyter lab "$@"
else
. /usr/local/bin/start.sh $wrapper jupyter notebook "$@"
. /usr/local/bin/start.sh $wrapper jupyter notebook "$@"
fi
......@@ -6,7 +6,7 @@ set -e
# set default ip to 0.0.0.0
if [[ "$NOTEBOOK_ARGS $@" != *"--ip="* ]]; then
NOTEBOOK_ARGS="--ip=0.0.0.0 $NOTEBOOK_ARGS"
NOTEBOOK_ARGS="--ip=0.0.0.0 $NOTEBOOK_ARGS"
fi
# handle some deprecated environment variables
......@@ -14,25 +14,25 @@ fi
# These won't be passed from DockerSpawner 0.9,
# so avoid specifying --arg=empty-string
if [ ! -z "$NOTEBOOK_DIR" ]; then
NOTEBOOK_ARGS="--notebook-dir='$NOTEBOOK_DIR' $NOTEBOOK_ARGS"
NOTEBOOK_ARGS="--notebook-dir='$NOTEBOOK_DIR' $NOTEBOOK_ARGS"
fi
if [ ! -z "$JPY_PORT" ]; then
NOTEBOOK_ARGS="--port=$JPY_PORT $NOTEBOOK_ARGS"
NOTEBOOK_ARGS="--port=$JPY_PORT $NOTEBOOK_ARGS"
fi
if [ ! -z "$JPY_USER" ]; then
NOTEBOOK_ARGS="--user=$JPY_USER $NOTEBOOK_ARGS"
NOTEBOOK_ARGS="--user=$JPY_USER $NOTEBOOK_ARGS"
fi
if [ ! -z "$JPY_COOKIE_NAME" ]; then
NOTEBOOK_ARGS="--cookie-name=$JPY_COOKIE_NAME $NOTEBOOK_ARGS"
NOTEBOOK_ARGS="--cookie-name=$JPY_COOKIE_NAME $NOTEBOOK_ARGS"
fi
if [ ! -z "$JPY_BASE_URL" ]; then
NOTEBOOK_ARGS="--base-url=$JPY_BASE_URL $NOTEBOOK_ARGS"
NOTEBOOK_ARGS="--base-url=$JPY_BASE_URL $NOTEBOOK_ARGS"
fi
if [ ! -z "$JPY_HUB_PREFIX" ]; then
NOTEBOOK_ARGS="--hub-prefix=$JPY_HUB_PREFIX $NOTEBOOK_ARGS"
NOTEBOOK_ARGS="--hub-prefix=$JPY_HUB_PREFIX $NOTEBOOK_ARGS"
fi
if [ ! -z "$JPY_HUB_API_URL" ]; then
NOTEBOOK_ARGS="--hub-api-url=$JPY_HUB_API_URL $NOTEBOOK_ARGS"
NOTEBOOK_ARGS="--hub-api-url=$JPY_HUB_API_URL $NOTEBOOK_ARGS"
fi
NOTEBOOK_BIN="jupyterhub-singleuser"
......
......@@ -9,6 +9,9 @@ LABEL maintainer="Jupyter Project <jupyter@googlegroups.com>"
# be skipped to shorten build time.
ARG TEST_ONLY_BUILD
# Fix DL4006
SHELL ["/bin/bash", "-o", "pipefail", "-c"]
USER root
# R pre-requisites
......@@ -25,21 +28,23 @@ ENV JULIA_DEPOT_PATH=/opt/julia
ENV JULIA_PKGDIR=/opt/julia
ENV JULIA_VERSION=1.4.1
RUN mkdir /opt/julia-${JULIA_VERSION} && \
cd /tmp && \
wget -q https://julialang-s3.julialang.org/bin/linux/x64/`echo ${JULIA_VERSION} | cut -d. -f 1,2`/julia-${JULIA_VERSION}-linux-x86_64.tar.gz && \
WORKDIR /tmp
# hadolint ignore=SC2046
RUN mkdir "/opt/julia-${JULIA_VERSION}" && \
wget -q https://julialang-s3.julialang.org/bin/linux/x64/$(echo "${JULIA_VERSION}" | cut -d. -f 1,2)"/julia-${JULIA_VERSION}-linux-x86_64.tar.gz" && \
echo "fd6d8cadaed678174c3caefb92207a3b0e8da9f926af6703fb4d1e4e4f50610a *julia-${JULIA_VERSION}-linux-x86_64.tar.gz" | sha256sum -c - && \
tar xzf julia-${JULIA_VERSION}-linux-x86_64.tar.gz -C /opt/julia-${JULIA_VERSION} --strip-components=1 && \
rm /tmp/julia-${JULIA_VERSION}-linux-x86_64.tar.gz
tar xzf "julia-${JULIA_VERSION}-linux-x86_64.tar.gz" -C "/opt/julia-${JULIA_VERSION}" --strip-components=1 && \
rm "/tmp/julia-${JULIA_VERSION}-linux-x86_64.tar.gz"
RUN ln -fs /opt/julia-*/bin/julia /usr/local/bin/julia
# Show Julia where conda libraries are \
RUN mkdir /etc/julia && \
echo "push!(Libdl.DL_LOAD_PATH, \"$CONDA_DIR/lib\")" >> /etc/julia/juliarc.jl && \
# Create JULIA_PKGDIR \
mkdir $JULIA_PKGDIR && \
chown $NB_USER $JULIA_PKGDIR && \
fix-permissions $JULIA_PKGDIR
mkdir "${JULIA_PKGDIR}" && \
chown "${NB_USER}" "${JULIA_PKGDIR}" && \
fix-permissions "${JULIA_PKGDIR}"
USER $NB_UID
......@@ -66,8 +71,8 @@ RUN conda install --quiet --yes \
'r-tidymodels=0.1*' \
&& \
conda clean --all -f -y && \
fix-permissions $CONDA_DIR && \
fix-permissions /home/$NB_USER
fix-permissions "${CONDA_DIR}" && \
fix-permissions "/home/${NB_USER}"
# Install e1071 R package (dependency of the caret R package)
RUN conda install --quiet --yes r-e1071
......@@ -83,7 +88,9 @@ RUN julia -e 'import Pkg; Pkg.update()' && \
(test $TEST_ONLY_BUILD || julia -e 'import Pkg; Pkg.add("HDF5")') && \
julia -e "using Pkg; pkg\"add IJulia\"; pkg\"precompile\"" && \
# move kernelspec out of home \
mv $HOME/.local/share/jupyter/kernels/julia* $CONDA_DIR/share/jupyter/kernels/ && \
chmod -R go+rx $CONDA_DIR/share/jupyter && \
rm -rf $HOME/.local && \
fix-permissions $JULIA_PKGDIR $CONDA_DIR/share/jupyter
mv "${HOME}/.local/share/jupyter/kernels/julia"* "${CONDA_DIR}/share/jupyter/kernels/" && \
chmod -R go+rx "${CONDA_DIR}/share/jupyter" && \
rm -rf "${HOME}/.local" && \
fix-permissions "${JULIA_PKGDIR}" "${CONDA_DIR}/share/jupyter"
WORKDIR $HOME
# Image Lint
To comply with [Docker best practices][dbp], we are using the [Hadolint][hadolint] tool to analyse each `Dockerfile` .
## Installation
There is a specific `make` target to install the linter.
By default `hadolint` will be installed in `${HOME}/hadolint`.
```bash
$ make lint-install
# Installing hadolint at /Users/romain/hadolint ...
# Installation done!
# Haskell Dockerfile Linter v1.17.6-0-gc918759
```
## Lint
### Per Stack
The linter can be run per stack.
```bash
$ make lint/scipy-notebook
# Linting Dockerfiles in scipy-notebook...
# scipy-notebook/Dockerfile:4 DL3006 Always tag the version of an image explicitly
# scipy-notebook/Dockerfile:11 DL3008 Pin versions in apt get install. Instead of `apt-get install <package>` use `apt-get install <package>=<version>`
# scipy-notebook/Dockerfile:18 SC2086 Double quote to prevent globbing and word splitting.
# scipy-notebook/Dockerfile:68 SC2086 Double quote to prevent globbing and word splitting.
# scipy-notebook/Dockerfile:68 DL3003 Use WORKDIR to switch to a directory
# scipy-notebook/Dockerfile:79 SC2086 Double quote to prevent globbing and word splitting.
# make: *** [lint/scipy-notebook] Error 1
```
Optionally you can pass arguments to the linter.
```bash
# Use a different export format
$ make lint/scipy-notebook ARGS="--format codeclimate"
```
### All the Stacks
The linter can be run against all the stacks.
```bash
$ make lint-all
```
## Ignoring Rules
Sometimes it is necessary to ignore [some rules][rules].
The following rules are ignored by default and sor for all images in the `.hadolint.yaml` file.
- [`DL3006`][DL3006]: We use a specific policy to manage image tags.
- `base-notebook` `FROM` clause is fixed but based on an argument (`ARG`).
- Building downstream images from (`FROM`) the latest is done on purpose.
- [`DL3008`][DL3008]: System packages are always updated (`apt-get`) to the latest version.
For other rules, the preferred way to do it is to flag ignored rules in the `Dockerfile`.
> It is also possible to ignore rules by using a special comment directly above the Dockerfile instruction you want to make an exception for. Ignore rule comments look like `# hadolint ignore=DL3001,SC1081`. For example:
```dockerfile
FROM ubuntu
# hadolint ignore=DL3003,SC1035
RUN cd /tmp && echo "hello!"
```
[hadolint]: https://github.com/hadolint/hadolint
[dbp]: https://docs.docker.com/develop/develop-images/dockerfile_best-practices
[rules]: https://github.com/hadolint/hadolint#rules
[DL3006]: https://github.com/hadolint/hadolint/wiki/DL3006
[DL3008]: https://github.com/hadolint/hadolint/wiki/DL3008
\ No newline at end of file
......@@ -47,6 +47,7 @@ Table of Contents
contributing/packages
contributing/recipes
contributing/translations
contributing/lint
contributing/tests
contributing/features
contributing/stacks
......
This diff is collapsed.
......@@ -9,12 +9,12 @@ set -e
# Get domain and email from environment
[ -z "$FQDN" ] && \
echo "ERROR: Must set FQDN environment varable" && \
exit 1
echo "ERROR: Must set FQDN environment varable" && \
exit 1
[ -z "$EMAIL" ] && \
echo "ERROR: Must set EMAIL environment varable" && \
exit 1
echo "ERROR: Must set EMAIL environment varable" && \
exit 1
# letsencrypt certificate server type (default is production).
# Set `CERT_SERVER=--staging` for staging.
......@@ -25,24 +25,24 @@ set -e
docker volume create --name $SECRETS_VOLUME 1>/dev/null
# Generate the cert and save it to the Docker volume
docker run --rm -it \
-p 80:80 \
-v $SECRETS_VOLUME:/etc/letsencrypt \
quay.io/letsencrypt/letsencrypt:latest \
certonly \
--non-interactive \
--keep-until-expiring \
--standalone \
--standalone-supported-challenges http-01 \
--agree-tos \
--domain "$FQDN" \
--email "$EMAIL" \
$CERT_SERVER
-p 80:80 \
-v $SECRETS_VOLUME:/etc/letsencrypt \
quay.io/letsencrypt/letsencrypt:latest \
certonly \
--non-interactive \
--keep-until-expiring \
--standalone \
--standalone-supported-challenges http-01 \
--agree-tos \
--domain "$FQDN" \
--email "$EMAIL" \
$CERT_SERVER
# Set permissions so nobody can read the cert and key.
# Also symlink the certs into the root of the /etc/letsencrypt
# directory so that the FQDN doesn't have to be known later.
docker run --rm -it \
-v $SECRETS_VOLUME:/etc/letsencrypt \
ubuntu:18.04 \
bash -c "ln -s /etc/letsencrypt/live/$FQDN/* /etc/letsencrypt/ && \
find /etc/letsencrypt -type d -exec chmod 755 {} +"
-v $SECRETS_VOLUME:/etc/letsencrypt \
ubuntu:18.04 \
bash -c "ln -s /etc/letsencrypt/live/$FQDN/* /etc/letsencrypt/ && \
find /etc/letsencrypt -type d -exec chmod 755 {} +"
......@@ -15,8 +15,8 @@ MACHINE_NAME=$1 && [ -z "$MACHINE_NAME" ] && echo "$USAGE" && exit $E_BADARGS
# Use SOFTLAYER_DOMAIN env var if domain name not set as second arg
DOMAIN="${2:-$SOFTLAYER_DOMAIN}" && [ -z "$DOMAIN" ] && \
echo "Must specify domain or set SOFTLAYER_DOMAIN environment varable" && \
echo "$USAGE" && exit $E_BADARGS
echo "Must specify domain or set SOFTLAYER_DOMAIN environment varable" && \
echo "$USAGE" && exit $E_BADARGS
IP=$(docker-machine ip "$MACHINE_NAME")
......
---
# Copyright (c) Jupyter Development Team.
# Distributed under the terms of the Modified BSD License.
......@@ -9,8 +10,8 @@ services:
image: my-notebook
container_name: ${NAME}
volumes:
- "work:/home/jovyan/work"
- "secrets:/etc/letsencrypt"
- "work:/home/jovyan/work"
- "secrets:/etc/letsencrypt"
ports:
- "${PORT}:8888"
environment:
......
---
# Copyright (c) Jupyter Development Team.
# Distributed under the terms of the Modified BSD License.
......@@ -9,7 +10,7 @@ services:
image: my-notebook
container_name: ${NAME}
volumes:
- "work:/home/jovyan/work"
- "work:/home/jovyan/work"
ports:
- "${PORT}:8888"
......
---
# Copyright (c) Jupyter Development Team.
# Distributed under the terms of the Modified BSD License.
......@@ -9,7 +10,7 @@ services:
image: my-notebook
container_name: ${NAME}
volumes:
- "work:/home/jovyan/work"
- "work:/home/jovyan/work"
ports:
- "${PORT}:8888"
environment:
......
......@@ -11,8 +11,7 @@ USAGE="Usage: `basename $0` [--secure | --letsencrypt] [--password PASSWORD] [--
# Parse args to determine security settings
SECURE=${SECURE:=no}
LETSENCRYPT=${LETSENCRYPT:=no}
while [[ $# > 0 ]]
do
while [[ $# > 0 ]]; do
key="$1"
case $key in
--secure)
......@@ -37,25 +36,25 @@ shift # past argument or value
done
if [[ "$LETSENCRYPT" == yes || "$SECURE" == yes ]]; then
if [ -z "${PASSWORD:+x}" ]; then
echo "ERROR: Must set PASSWORD if running in secure mode"
echo "$USAGE"
exit 1
fi
if [ "$LETSENCRYPT" == yes ]; then
CONFIG=letsencrypt-notebook.yml
if [ -z "${SECRETS_VOLUME:+x}" ]; then
echo "ERROR: Must set SECRETS_VOLUME if running in letsencrypt mode"
echo "$USAGE"
exit 1
if [ -z "${PASSWORD:+x}" ]; then
echo "ERROR: Must set PASSWORD if running in secure mode"
echo "$USAGE"
exit 1
fi
else
CONFIG=secure-notebook.yml
fi
export PORT=${PORT:=443}
if [ "$LETSENCRYPT" == yes ]; then
CONFIG=letsencrypt-notebook.yml
if [ -z "${SECRETS_VOLUME:+x}" ]; then
echo "ERROR: Must set SECRETS_VOLUME if running in letsencrypt mode"
echo "$USAGE"
exit 1
fi
else
CONFIG=secure-notebook.yml
fi
export PORT=${PORT:=443}
else
CONFIG=notebook.yml
export PORT=${PORT:=80}
CONFIG=notebook.yml
export PORT=${PORT:=80}
fi
# Setup environment
......
......@@ -5,6 +5,9 @@ FROM $BASE_CONTAINER
LABEL maintainer="Jupyter Project <jupyter@googlegroups.com>"
# Fix DL4006
SHELL ["/bin/bash", "-o", "pipefail", "-c"]
USER root
# Spark dependencies
......@@ -16,13 +19,16 @@ RUN apt-get -y update && \
rm -rf /var/lib/apt/lists/*
# Using the preferred mirror to download Spark
RUN cd /tmp && \
wget -q $(wget -qO- https://www.apache.org/dyn/closer.lua/spark/spark-${APACHE_SPARK_VERSION}/spark-${APACHE_SPARK_VERSION}-bin-hadoop${HADOOP_VERSION}.tgz\?as_json | \
WORKDIR /tmp
# hadolint ignore=SC2046
RUN wget -q $(wget -qO- https://www.apache.org/dyn/closer.lua/spark/spark-${APACHE_SPARK_VERSION}/spark-${APACHE_SPARK_VERSION}-bin-hadoop${HADOOP_VERSION}.tgz\?as_json | \
python -c "import sys, json; content=json.load(sys.stdin); print(content['preferred']+content['path_info'])") && \
echo "2426a20c548bdfc07df288cd1d18d1da6b3189d0b78dee76fa034c52a4e02895f0ad460720c526f163ba63a17efae4764c46a1cd8f9b04c60f9937a554db85d2 *spark-${APACHE_SPARK_VERSION}-bin-hadoop${HADOOP_VERSION}.tgz" | sha512sum -c - && \
tar xzf spark-${APACHE_SPARK_VERSION}-bin-hadoop${HADOOP_VERSION}.tgz -C /usr/local --owner root --group root --no-same-owner && \
rm spark-${APACHE_SPARK_VERSION}-bin-hadoop${HADOOP_VERSION}.tgz
RUN cd /usr/local && ln -s spark-${APACHE_SPARK_VERSION}-bin-hadoop${HADOOP_VERSION} spark
tar xzf "spark-${APACHE_SPARK_VERSION}-bin-hadoop${HADOOP_VERSION}.tgz" -C /usr/local --owner root --group root --no-same-owner && \
rm "spark-${APACHE_SPARK_VERSION}-bin-hadoop${HADOOP_VERSION}.tgz"
WORKDIR /usr/local
RUN ln -s "spark-${APACHE_SPARK_VERSION}-bin-hadoop${HADOOP_VERSION}" spark
# Configure Spark
ENV SPARK_HOME=/usr/local/spark
......@@ -35,5 +41,7 @@ USER $NB_UID
# Install pyarrow
RUN conda install --quiet -y 'pyarrow' && \
conda clean --all -f -y && \
fix-permissions $CONDA_DIR && \
fix-permissions /home/$NB_USER
fix-permissions "${CONDA_DIR}" && \
fix-permissions "/home/${NB_USER}"
WORKDIR $HOME
......@@ -46,7 +46,7 @@ RUN conda install --quiet --yes \
'r-tidymodels=0.1*' \
&& \
conda clean --all -f -y && \
fix-permissions $CONDA_DIR
fix-permissions "${CONDA_DIR}"
# Install e1071 R package (dependency of the caret R package)
RUN conda install --quiet --yes r-e1071
......@@ -59,24 +59,25 @@ RUN conda install --quiet --yes \
jupyter lab build -y && \
jupyter lab clean -y && \
npm cache clean --force && \
rm -rf /home/$NB_USER/.cache/yarn && \
rm -rf /home/$NB_USER/.node-gyp && \
fix-permissions $CONDA_DIR && \
fix-permissions /home/$NB_USER
rm -rf "/home/${NB_USER}/.cache/yarn" && \
rm -rf "/home/${NB_USER}/.node-gyp" && \
fix-permissions "${CONDA_DIR}" && \
fix-permissions "/home/${NB_USER}"
# Install facets which does not have a pip or conda package at the moment
RUN cd /tmp && \
git clone https://github.com/PAIR-code/facets.git && \
cd facets && \
jupyter nbextension install facets-dist/ --sys-prefix && \
cd && \
WORKDIR /tmp
RUN git clone https://github.com/PAIR-code/facets.git && \
jupyter nbextension install facets/facets-dist/ --sys-prefix && \
rm -rf /tmp/facets && \
fix-permissions $CONDA_DIR && \
fix-permissions /home/$NB_USER
fix-permissions "${CONDA_DIR}" && \
fix-permissions "/home/${NB_USER}"
# Import matplotlib the first time to build the font cache.
ENV XDG_CACHE_HOME /home/$NB_USER/.cache/
ENV XDG_CACHE_HOME="/home/${NB_USER}/.cache/"
RUN MPLBACKEND=Agg python -c "import matplotlib.pyplot" && \
fix-permissions /home/$NB_USER
fix-permissions "/home/${NB_USER}"
USER $NB_UID
WORKDIR $HOME
......@@ -8,5 +8,5 @@ LABEL maintainer="Jupyter Project <jupyter@googlegroups.com>"
# Install Tensorflow
RUN pip install --quiet --no-cache-dir \
'tensorflow==2.2.0' && \
fix-permissions $CONDA_DIR && \
fix-permissions /home/$NB_USER
fix-permissions "${CONDA_DIR}" && \
fix-permissions "/home/${NB_USER}"
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment