Commit 455e10b6 authored by Peter Parente's avatar Peter Parente Committed by GitHub

Merge branch 'master' into pre-commit

parents bf39616d a4d48eaf
name: Build Docker Images
on:
pull_request:
paths-ignore:
- "*.md"
- "binder/**"
- "docs/**"
- "examples/**"
push:
branches:
- master
paths-ignore:
- "*.md"
- "binder/**"
- "docs/**"
- "examples/**"
jobs:
build:
name: Build Docker Images
runs-on: ubuntu-latest
if: >
!contains(github.event.head_commit.message, 'ci skip') &&
!contains(github.event.pull_request.title, 'ci skip')
steps:
- name: Clone Main Repo
uses: actions/checkout@v2
with:
path: main
- name: Clone Wiki
uses: actions/checkout@v2
with:
repository: ${{github.repository}}.wiki
path: wiki
- name: Set Up Python
uses: actions/setup-python@v2
with:
python-version: 3.8
- name: Install Dev Dependencies
run: |
python -m pip install --upgrade pip
make -C main dev-env lint-install
- name: Lint Dockerfiles
run: make -C main lint-all
- name: Build Docker Images
run: make -C main build-test-all
- name: Run Post-Build Hooks
run: make -C main hook-all
env:
COMMIT_MSG: "${{github.event.head_commit.message}}"
WIKI_PATH: ../wiki
- name: Login to Docker Hub
if: github.ref == 'refs/heads/master'
run: >
echo '${{secrets.DOCKERHUB_PASSWORD}}' | docker login --username
'${{secrets.DOCKERHUB_USERNAME}}' --password-stdin
- name: Push Images to DockerHub
if: github.ref == 'refs/heads/master'
run: make -C main push-all
- name: Push Wiki to GitHub
if: github.ref == 'refs/heads/master'
run: make -C main git-commit
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
GITHUB_REPOSITORY: ${{github.repository}}.wiki
LOCAL_PATH: ../wiki
name: Build Sphinx Documentation
on:
pull_request:
paths:
- "docs/**"
- ".github/workflows/sphinx.yml"
push:
branches:
- master
paths:
- "docs/**"
- ".github/workflows/sphinx.yml"
jobs:
build:
name: Build Sphinx Documentation
runs-on: ubuntu-latest
if: >
!contains(github.event.head_commit.message , 'ci skip') &&
!contains(github.event.pull_request.title, 'ci skip')
steps:
- name: Checkout Repo
uses: actions/checkout@v2
- name: Set Up Python
uses: actions/setup-python@v2
with:
python-version: 3.x
- name: Install Dev Dependencies
run: |
python -m pip install --upgrade pip
make dev-env
- name: Build Documentation
run: make docs
- name: Extract Source Strings
if: github.ref == 'refs/heads/master'
working-directory: docs
run: |
make gettext
sphinx-intl update -p _build/gettext -l en
- name: Push Strings to Master
if: github.ref == 'refs/heads/master'
run: make git-commit
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
GITHUB_REPOSITORY: ${{ github.repository }}
LOCAL_PATH: ./docs/locale/en
---
dist: bionic
language: python
python:
- 3.7
sudo: required
services:
- docker
jobs:
include:
- stage: diff-test
install:
- pip install --upgrade pip
- make dev-env
- make lint-install
script:
- set -e
- if [ $(make n-docs-diff) -ne 0 ]; then make docs; fi;
- if [ $(make n-other-diff) -ne 0 ]; then make lint-build-test-all DARGS="--build-arg TEST_ONLY_BUILD=1"; fi;
- stage: push-tx
install:
- pip install --upgrade pip
- make dev-env
script:
- if [ $(make n-docs-diff DIFF_RANGE=$TRAVIS_COMMIT_RANGE) -ne 0 ]; then make tx-en; fi;
- stage: full-test
install:
- pip install --upgrade pip
- make dev-env
- make lint-install
script:
- set -e
- make docs
- make lint-build-test-all DARGS="--build-arg TEST_ONLY_BUILD=1"
stages:
- name: diff-test
if: type = pull_request
- name: push-tx
if: type = push AND branch = master
- name: full-test
if: type = cron AND branch = master
env:
global:
# yamllint disable-line
secure: JDQKgB1laaenzmEGI9gduwK/iS0030qsl62njJg3jib0R8wxBi2OeetEdoGl9m4NFsqqnp0OO7nm4rzGfuwjL1A38flSlXOTxhjm/hvo3vmnqVu5lDcdk9+IRkafnfd3Dyd86tLxRVETOqZwCLmdNkB2fmQII8du5IIqbJuUGp8DrG7kVMf3NBr9rjkZRfByQrgG4s1NXuT61VvpWMPJAOhcrImuHBheVJDEV0U3n6Xavd7Wo+pAHrHU8exvYTZ1IzZMbHc6K0iC/NpCHcH9+9DAeLDk/q1aDNqbTExnQevOHZzNqgHC2qFOlN4jfy/TLYLpLXtUismneBBqVSK3iZso3Vqy2BRXWgouI+Tt+08ffocy9XPwEzSwkgPgDlFVUikPOy5imwjpDb13RMIyMY4CKlSOdQx2rH2kPkZ0MJJPcki3KGuGl3qRvqyblMn+lZvjAu6WVLZfo7EtcxsQ0ZZxbAbGoUVl27FHg+UvIfC0I3wEcZIp7oED47Q8s0MdCijD3AwkRPvx/iyp3J0A42su7kkOooFcmUItEIqegQJ4Aki1FBv2i5vHmBobClktytZceLsKvzCeLjMpL9HcUVfUaJDKRwtUYIozpYeBnac+E6J1s6glcqLrXIHWez8N6SzokBa6SPqdtODdzzk5OJupByub6CYWsRXvxIQ7/wI=
...@@ -4,9 +4,8 @@ ...@@ -4,9 +4,8 @@
# Use bash for inline if-statements in arch_patch target # Use bash for inline if-statements in arch_patch target
SHELL:=bash SHELL:=bash
OWNER:=jupyter
ARCH:=$(shell uname -m) ARCH:=$(shell uname -m)
DIFF_RANGE?=master...HEAD OWNER?=jupyter
# Need to list the images in build dependency order # Need to list the images in build dependency order
ifeq ($(ARCH),ppc64le) ifeq ($(ARCH),ppc64le)
...@@ -77,22 +76,35 @@ dev/%: ## run a foreground container for a stack ...@@ -77,22 +76,35 @@ dev/%: ## run a foreground container for a stack
dev-env: ## install libraries required to build docs and run tests dev-env: ## install libraries required to build docs and run tests
@pip install -r requirements-dev.txt @pip install -r requirements-dev.txt
lint/%: ARGS?= docs: ## build HTML documentation
lint/%: ## lint the dockerfile(s) for a stack make -C docs html
@echo "Linting Dockerfiles in $(notdir $@)..."
@git ls-files --exclude='Dockerfile*' --ignored $(notdir $@) | grep -v ppc64 | xargs -L 1 $(HADOLINT) $(ARGS)
@echo "Linting done!"
lint-all: $(foreach I,$(ALL_IMAGES),lint/$(I) ) ## lint all stacks
lint-build-test-all: $(foreach I,$(ALL_IMAGES),lint/$(I) arch_patch/$(I) build/$(I) test/$(I) ) ## lint, build and test all stacks
lint-install: ## install hadolint git-commit: LOCAL_PATH?=.
@echo "Installing hadolint at $(HADOLINT) ..." git-commit: GITHUB_SHA?=$(shell git rev-parse HEAD)
@curl -sL -o $(HADOLINT) "https://github.com/hadolint/hadolint/releases/download/$(HADOLINT_VERSION)/hadolint-$(shell uname -s)-$(shell uname -m)" git-commit: GITHUB_REPOSITORY?=jupyter/docker-stacks
@chmod 700 $(HADOLINT) git-commit: GITHUB_TOKEN?=
@echo "Installation done!" git-commit: ## commit outstading git changes and push to remote
@$(HADOLINT) --version @git config --global user.name "GitHub Actions"
@git config --global user.email "actions@users.noreply.github.com"
@git remote add publisher https://$(GITHUB_TOKEN)@github.com/$(GITHUB_REPOSITORY).git
@cd $(LOCAL_PATH) && \
git checkout master && \
git add -A -- . && \
git commit -m "[ci skip] Automated publish for $(GITHUB_SHA)" || exit 0 && \
git push -u publisher master
hook/%: export COMMIT_MSG?=$(shell git log -1 --pretty=%B)
hook/%: export GITHUB_SHA?=$(shell git rev-parse HEAD)
hook/%: export WIKI_PATH?=../wiki
hook/%: ## run post-build hooks for an image
BUILD_TIMESTAMP="$$(date -u +%FT%TZ)" \
DOCKER_REPO="$(OWNER)/$(notdir $@)" \
IMAGE_NAME="$(OWNER)/$(notdir $@):latest" \
IMAGE_SHORT_NAME="$(notdir $@)" \
$(SHELL) $(notdir $@)/hooks/run_hook
hook-all: $(foreach I,$(ALL_IMAGES),hook/$(I) ) ## run post-build hooks for all images
img-clean: img-rm-dang img-rm ## clean dangling and jupyter images img-clean: img-rm-dang img-rm ## clean dangling and jupyter images
...@@ -108,15 +120,22 @@ img-rm-dang: ## remove dangling images (tagged None) ...@@ -108,15 +120,22 @@ img-rm-dang: ## remove dangling images (tagged None)
@echo "Removing dangling images ..." @echo "Removing dangling images ..."
-docker rmi --force $(shell docker images -f "dangling=true" -q) 2> /dev/null -docker rmi --force $(shell docker images -f "dangling=true" -q) 2> /dev/null
docs: ## build HTML documentation lint/%: ARGS?=
make -C docs html lint/%: ## lint the dockerfile(s) for a stack
@echo "Linting Dockerfiles in $(notdir $@)..."
@git ls-files --exclude='Dockerfile*' --ignored $(notdir $@) | grep -v ppc64 | xargs -L 1 $(HADOLINT) $(ARGS)
@echo "Linting done!"
n-docs-diff: ## number of docs/ files changed since branch from master lint-all: $(foreach I,$(ALL_IMAGES),lint/$(I) ) ## lint all stacks
@git diff --name-only $(DIFF_RANGE) -- docs/ ':!docs/locale' | wc -l | awk '{print $$1}'
lint-build-test-all: $(foreach I,$(ALL_IMAGES),lint/$(I) arch_patch/$(I) build/$(I) test/$(I) ) ## lint, build and test all stacks
n-other-diff: ## number of files outside docs/ changed since branch from master lint-install: ## install hadolint
@git diff --name-only $(DIFF_RANGE) -- ':!docs/' | wc -l | awk '{print $$1}' @echo "Installing hadolint at $(HADOLINT) ..."
@curl -sL -o $(HADOLINT) "https://github.com/hadolint/hadolint/releases/download/$(HADOLINT_VERSION)/hadolint-$(shell uname -s)-$(shell uname -m)"
@chmod 700 $(HADOLINT)
@echo "Installation done!"
@$(HADOLINT) --version
pre-commit-all: ## run pre-commit hook on all files pre-commit-all: ## run pre-commit hook on all files
@pre-commit run --all @pre-commit run --all
...@@ -129,6 +148,12 @@ pull/%: DARGS?= ...@@ -129,6 +148,12 @@ pull/%: DARGS?=
pull/%: ## pull a jupyter image pull/%: ## pull a jupyter image
docker pull $(DARGS) $(OWNER)/$(notdir $@) docker pull $(DARGS) $(OWNER)/$(notdir $@)
push/%: DARGS?=
push/%: ## push all tags for a jupyter image
docker push $(DARGS) $(OWNER)/$(notdir $@)
push-all: $(foreach I,$(ALL_IMAGES),push/$(I) ) ## push all tagged images
run/%: DARGS?= run/%: DARGS?=
run/%: ## run a bash in interactive mode in a stack run/%: ## run a bash in interactive mode in a stack
docker run -it --rm $(DARGS) $(OWNER)/$(notdir $@) $(SHELL) docker run -it --rm $(DARGS) $(OWNER)/$(notdir $@) $(SHELL)
...@@ -137,20 +162,6 @@ run-sudo/%: DARGS?= ...@@ -137,20 +162,6 @@ run-sudo/%: DARGS?=
run-sudo/%: ## run a bash in interactive mode as root in a stack run-sudo/%: ## run a bash in interactive mode as root in a stack
docker run -it --rm -u root $(DARGS) $(OWNER)/$(notdir $@) $(SHELL) docker run -it --rm -u root $(DARGS) $(OWNER)/$(notdir $@) $(SHELL)
tx-en: ## rebuild en locale strings and push to master (req: GH_TOKEN)
@git config --global user.email "travis@travis-ci.org"
@git config --global user.name "Travis CI"
@git checkout master
@make -C docs clean gettext
@cd docs && sphinx-intl update -p _build/gettext -l en
@git add docs/locale/en
@git commit -m "[ci skip] Update en source strings (build: $$TRAVIS_JOB_NUMBER)"
@git remote add origin-tx https://$${GH_TOKEN}@github.com/jupyter/docker-stacks.git
@git push -u origin-tx master
test/%: ## run tests against a stack (only common tests or common tests + specific tests) test/%: ## run tests against a stack (only common tests or common tests + specific tests)
@if [ ! -d "$(notdir $@)/test" ]; then TEST_IMAGE="$(OWNER)/$(notdir $@)" pytest -m "not info" test; \ @if [ ! -d "$(notdir $@)/test" ]; then TEST_IMAGE="$(OWNER)/$(notdir $@)" pytest -m "not info" test; \
else TEST_IMAGE="$(OWNER)/$(notdir $@)" pytest -m "not info" test $(notdir $@)/test; fi else TEST_IMAGE="$(OWNER)/$(notdir $@)" pytest -m "not info" test $(notdir $@)/test; fi
......
INDEX_ROW="|\`${BUILD_TIMESTAMP}\`|\`jupyter/${IMAGE_SHORT_NAME}:${GIT_SHA_TAG}\`|[Git diff](https://github.com/jupyter/docker-stacks/commit/${SOURCE_COMMIT})<br />[Dockerfile](https://github.com/jupyter/docker-stacks/blob/${SOURCE_COMMIT}/${IMAGE_SHORT_NAME}/Dockerfile)<br />[Build manifest](./${IMAGE_SHORT_NAME}-${GIT_SHA_TAG})|"
sed "/|-|/a ${INDEX_ROW}" -i "$INDEX_FILE"
\ No newline at end of file
#!/bin/bash
set -e
# Tag the latest build with the short git sha. Push the tag in addition
# to the "latest" tag already pushed.
GIT_SHA_TAG=${SOURCE_COMMIT:0:12}
docker tag $IMAGE_NAME $DOCKER_REPO:$GIT_SHA_TAG
docker push $DOCKER_REPO:$GIT_SHA_TAG
# Create a working directory.
BUILD_TIMESTAMP=$(date -u +%FT%TZ)
WORKDIR=$(mktemp -d)
GIT_URI="git@github.com:jupyter/docker-stacks.wiki.git"
GIT_SANDBOX="${WORKDIR}/docker-stacks.wiki"
IMAGE_SHORT_NAME=$(basename $DOCKER_REPO)
MANIFEST_FILE="${GIT_SANDBOX}/manifests/${IMAGE_SHORT_NAME}-${GIT_SHA_TAG}.md"
INDEX_FILE="${GIT_SANDBOX}/Home.md"
# Configure git so it can push back to GitHub.
eval $(ssh-agent -s)
ssh-add <(base64 -d <(echo "$DEPLOY_KEY"))
ssh-add -l
git config --global user.email "jupyter@googlegroups.com"
git config --global user.name "Jupyter Docker Stacks"
# Glone the GitHub project wiki.
pushd "$WORKDIR"
git clone "$GIT_URI"
popd
# Render the build manifest template.
mkdir -p $(dirname "$MANIFEST_FILE")
source hooks/manifest.tmpl
source hooks/index.tmpl
# Push the wiki update back to GitHub.
pushd "$GIT_SANDBOX"
git add .
git commit -m "DOC: Build ${MANIFEST_FILE}"
git push -u origin master
popd
# Shutdown the ssh agent for good measure.
ssh-agent -k
# Invoke all downstream build triggers.
set +e
for url in $(echo $NEXT_BUILD_TRIGGERS | sed "s/,/ /g")
do
curl -X POST $url
done
\ No newline at end of file
#!/bin/bash
set -e
# Apply tags
GIT_SHA_TAG=${GITHUB_SHA:0:12}
docker tag $IMAGE_NAME "$DOCKER_REPO:$GIT_SHA_TAG"
# Update index
INDEX_ROW="|\`${BUILD_TIMESTAMP}\`|\`jupyter/${IMAGE_SHORT_NAME}:${GIT_SHA_TAG}\`|[Git diff](https://github.com/jupyter/docker-stacks/commit/${GITHUB_SHA})<br />[Dockerfile](https://github.com/jupyter/docker-stacks/blob/${GITHUB_SHA}/${IMAGE_SHORT_NAME}/Dockerfile)<br />[Build manifest](./${IMAGE_SHORT_NAME}-${GIT_SHA_TAG})|"
sed "/|-|/a ${INDEX_ROW}" -i "${WIKI_PATH}/Home.md"
# Build manifest
MANIFEST_FILE="${WIKI_PATH}/manifests/${IMAGE_SHORT_NAME}-${GIT_SHA_TAG}.md"
mkdir -p $(dirname "$MANIFEST_FILE")
cat << EOF > "$MANIFEST_FILE" cat << EOF > "$MANIFEST_FILE"
* Build datetime: ${BUILD_TIMESTAMP} * Build datetime: ${BUILD_TIMESTAMP}
* DockerHub build code: ${BUILD_CODE}
* Docker image: ${DOCKER_REPO}:${GIT_SHA_TAG} * Docker image: ${DOCKER_REPO}:${GIT_SHA_TAG}
* Docker image size: $(docker images ${IMAGE_NAME} --format "{{.Size}}") * Docker image size: $(docker images ${IMAGE_NAME} --format "{{.Size}}")
* Git commit SHA: [${SOURCE_COMMIT}](https://github.com/jupyter/docker-stacks/commit/${SOURCE_COMMIT}) * Git commit SHA: [${GITHUB_SHA}](https://github.com/jupyter/docker-stacks/commit/${GITHUB_SHA})
* Git commit message: * Git commit message:
\`\`\` \`\`\`
${COMMIT_MSG} ${COMMIT_MSG}
......
INDEX_ROW="|\`${BUILD_TIMESTAMP}\`|\`jupyter/${IMAGE_SHORT_NAME}:${GIT_SHA_TAG}\`<br />\`jupyter/${IMAGE_SHORT_NAME}:${PY_VERSION_TAG}\`<br />\`jupyter/${IMAGE_SHORT_NAME}:${NB_VERSION_TAG}\`<br />\`jupyter/${IMAGE_SHORT_NAME}:${LAB_VERSION_TAG}\`<br />\`jupyter/${IMAGE_SHORT_NAME}:${HUB_VERSION_TAG}\`|[Git diff](https://github.com/jupyter/docker-stacks/commit/${SOURCE_COMMIT})<br />[Dockerfile](https://github.com/jupyter/docker-stacks/blob/${SOURCE_COMMIT}/${IMAGE_SHORT_NAME}/Dockerfile)<br />[Build manifest](./${IMAGE_SHORT_NAME}-${GIT_SHA_TAG})|"
sed "/|-|/a ${INDEX_ROW}" -i "$INDEX_FILE"
\ No newline at end of file
cat << EOF > "$MANIFEST_FILE"
* Build datetime: ${BUILD_TIMESTAMP}
* DockerHub build code: ${BUILD_CODE}
* Docker image: ${DOCKER_REPO}:${GIT_SHA_TAG}
* Docker image size: $(docker images ${IMAGE_NAME} --format "{{.Size}}")
* Git commit SHA: [${SOURCE_COMMIT}](https://github.com/jupyter/docker-stacks/commit/${SOURCE_COMMIT})
* Git commit message:
\`\`\`
${COMMIT_MSG}
\`\`\`
## Python Packages
\`\`\`
$(docker run --rm ${IMAGE_NAME} python --version)
\`\`\`
\`\`\`
$(docker run --rm ${IMAGE_NAME} conda info)
\`\`\`
\`\`\`
$(docker run --rm ${IMAGE_NAME} conda list)
\`\`\`
## Apt Packages
\`\`\`
$(docker run --rm ${IMAGE_NAME} apt list --installed)
\`\`\`
EOF
\ No newline at end of file
#!/bin/bash
if [[ "$COMMIT_MSG" = *"skip ci"* || "$COMMIT_MSG" = *"ci skip"* ]]; then
exit 1;
fi
\ No newline at end of file
#!/bin/bash
set -e
# Apply and push all tags
source hooks/apply_tags
docker push $DOCKER_REPO
# Create a working directory.
BUILD_TIMESTAMP=$(date -u +%FT%TZ)
WORKDIR=$(mktemp -d)
GIT_URI="git@github.com:jupyter/docker-stacks.wiki.git"
GIT_SANDBOX="${WORKDIR}/docker-stacks.wiki"
IMAGE_SHORT_NAME=$(basename $DOCKER_REPO)
MANIFEST_FILE="${GIT_SANDBOX}/manifests/${IMAGE_SHORT_NAME}-${GIT_SHA_TAG}.md"
INDEX_FILE="${GIT_SANDBOX}/Home.md"
# Configure git so it can push back to GitHub.
eval $(ssh-agent -s)
ssh-add <(base64 -d <(echo "$DEPLOY_KEY"))
ssh-add -l
git config --global user.email "jupyter@googlegroups.com"
git config --global user.name "Jupyter Docker Stacks"
# Glone the GitHub project wiki.
pushd "$WORKDIR"
git clone "$GIT_URI"
popd
# Render the build manifest template.
mkdir -p $(dirname "$MANIFEST_FILE")
source hooks/manifest.tmpl
source hooks/index.tmpl
# Push the wiki update back to GitHub.
pushd "$GIT_SANDBOX"
git add .
git commit -m "DOC: Build ${MANIFEST_FILE}"
git push -u origin master
popd
# Shutdown the ssh agent for good measure.
ssh-agent -k
# Invoke all downstream build triggers.
set +e
for url in $(echo $NEXT_BUILD_TRIGGERS | sed "s/,/ /g")
do
curl -X POST $url
done
\ No newline at end of file
#!/bin/bash #!/bin/bash
set -e set -e
# Tag the latest build with the short git sha as well as version of key runtimes # Apply tags
# and packages. GIT_SHA_TAG=${GITHUB_SHA:0:12}
GIT_SHA_TAG=${SOURCE_COMMIT:0:12}
docker tag $IMAGE_NAME "$DOCKER_REPO:$GIT_SHA_TAG" docker tag $IMAGE_NAME "$DOCKER_REPO:$GIT_SHA_TAG"
PY_VERSION_TAG="python-$(docker run --rm ${IMAGE_NAME} python --version 2>&1 | awk '{print $2}')" PY_VERSION_TAG="python-$(docker run --rm ${IMAGE_NAME} python --version 2>&1 | awk '{print $2}')"
docker tag $IMAGE_NAME "$DOCKER_REPO:$PY_VERSION_TAG" docker tag $IMAGE_NAME "$DOCKER_REPO:$PY_VERSION_TAG"
...@@ -12,4 +11,43 @@ docker tag $IMAGE_NAME "$DOCKER_REPO:${NB_VERSION_TAG%% }" ...@@ -12,4 +11,43 @@ docker tag $IMAGE_NAME "$DOCKER_REPO:${NB_VERSION_TAG%% }"
LAB_VERSION_TAG="lab-$(docker run --rm -a STDOUT ${IMAGE_NAME} jupyter-lab --version | tr -d '\r')" LAB_VERSION_TAG="lab-$(docker run --rm -a STDOUT ${IMAGE_NAME} jupyter-lab --version | tr -d '\r')"
docker tag $IMAGE_NAME "$DOCKER_REPO:${LAB_VERSION_TAG%%\r}" docker tag $IMAGE_NAME "$DOCKER_REPO:${LAB_VERSION_TAG%%\r}"
HUB_VERSION_TAG="hub-$(docker run --rm -a STDOUT ${IMAGE_NAME} jupyterhub --version | tr -d '\r')" HUB_VERSION_TAG="hub-$(docker run --rm -a STDOUT ${IMAGE_NAME} jupyterhub --version | tr -d '\r')"
docker tag $IMAGE_NAME "$DOCKER_REPO:${HUB_VERSION_TAG%%\r}" docker tag $IMAGE_NAME "$DOCKER_REPO:${HUB_VERSION_TAG%%\r}"
\ No newline at end of file
# Update index
INDEX_FILE="${WIKI_PATH}/Home.md"
INDEX_ROW="|\`${BUILD_TIMESTAMP}\`|\`jupyter/${IMAGE_SHORT_NAME}:${GIT_SHA_TAG}\`<br />\`jupyter/${IMAGE_SHORT_NAME}:${PY_VERSION_TAG}\`<br />\`jupyter/${IMAGE_SHORT_NAME}:${NB_VERSION_TAG}\`<br />\`jupyter/${IMAGE_SHORT_NAME}:${LAB_VERSION_TAG}\`<br />\`jupyter/${IMAGE_SHORT_NAME}:${HUB_VERSION_TAG}\`|[Git diff](https://github.com/jupyter/docker-stacks/commit/${GITHUB_SHA})<br />[Dockerfile](https://github.com/jupyter/docker-stacks/blob/${GITHUB_SHA}/${IMAGE_SHORT_NAME}/Dockerfile)<br />[Build manifest](./${IMAGE_SHORT_NAME}-${GIT_SHA_TAG})|"
sed "/|-|/a ${INDEX_ROW}" -i "$INDEX_FILE"
# Build manifest
MANIFEST_FILE="${WIKI_PATH}/manifests/${IMAGE_SHORT_NAME}-${GIT_SHA_TAG}.md"
mkdir -p $(dirname "$MANIFEST_FILE")
cat << EOF > "$MANIFEST_FILE"
* Build datetime: ${BUILD_TIMESTAMP}
* Docker image: ${DOCKER_REPO}:${GIT_SHA_TAG}
* Docker image size: $(docker images ${IMAGE_NAME} --format "{{.Size}}")
* Git commit SHA: [${GITHUB_SHA}](https://github.com/jupyter/docker-stacks/commit/${GITHUB_SHA})
* Git commit message:
\`\`\`
${COMMIT_MSG}
\`\`\`
## Python Packages
\`\`\`
$(docker run --rm ${IMAGE_NAME} python --version)
\`\`\`
\`\`\`
$(docker run --rm ${IMAGE_NAME} conda info)
\`\`\`
\`\`\`
$(docker run --rm ${IMAGE_NAME} conda list)
\`\`\`
## Apt Packages
\`\`\`
$(docker run --rm ${IMAGE_NAME} apt list --installed)
\`\`\`
EOF
...@@ -76,6 +76,9 @@ def test_nb_user_change(container): ...@@ -76,6 +76,9 @@ def test_nb_user_change(container):
command=['start.sh', 'bash', '-c', 'sleep infinity'] command=['start.sh', 'bash', '-c', 'sleep infinity']
) )
# Give the chown time to complete. Use sleep, not wait, because the
# container sleeps forever.
time.sleep(10)
LOGGER.info(f"Checking if the user is changed to {nb_user} by the start script ...") LOGGER.info(f"Checking if the user is changed to {nb_user} by the start script ...")
output = running_container.logs(stdout=True).decode("utf-8") output = running_container.logs(stdout=True).decode("utf-8")
assert f"Set username to: {nb_user}" in output, f"User is not changed to {nb_user}" assert f"Set username to: {nb_user}" in output, f"User is not changed to {nb_user}"
...@@ -123,6 +126,7 @@ def test_chown_home(container): ...@@ -123,6 +126,7 @@ def test_chown_home(container):
], ],
command=['start.sh', 'bash', '-c', 'chown root:root /home/jovyan && ls -alsh /home'] command=['start.sh', 'bash', '-c', 'chown root:root /home/jovyan && ls -alsh /home']
) )
c.wait(timeout=120)
assert "Changing ownership of /home/jovyan to 1000:100 with options '-R'" in c.logs(stdout=True).decode('utf-8') assert "Changing ownership of /home/jovyan to 1000:100 with options '-R'" in c.logs(stdout=True).decode('utf-8')
......
INDEX_ROW="|\`${BUILD_TIMESTAMP}\`|\`jupyter/${IMAGE_SHORT_NAME}:${GIT_SHA_TAG}\`<br />\`jupyter/${IMAGE_SHORT_NAME}:${PY_VERSION_TAG}\`<br />\`jupyter/${IMAGE_SHORT_NAME}:${JULIA_VERSION_TAG}\`<br />\`jupyter/${IMAGE_SHORT_NAME}:${R_VERSION_TAG}\`<br />\`jupyter/${IMAGE_SHORT_NAME}:${NB_VERSION_TAG}\`<br />\`jupyter/${IMAGE_SHORT_NAME}:${LAB_VERSION_TAG}\`<br />\`jupyter/${IMAGE_SHORT_NAME}:${HUB_VERSION_TAG}\`|[Git diff](https://github.com/jupyter/docker-stacks/commit/${SOURCE_COMMIT})<br />[Dockerfile](https://github.com/jupyter/docker-stacks/blob/${SOURCE_COMMIT}/${IMAGE_SHORT_NAME}/Dockerfile)<br />[Build manifest](./${IMAGE_SHORT_NAME}-${GIT_SHA_TAG})|"
sed "/|-|/a ${INDEX_ROW}" -i "$INDEX_FILE"
\ No newline at end of file
#!/bin/bash
set -e
# Apply and push all tags
source hooks/apply_tags
docker push $DOCKER_REPO
# Tag the latest build with the short git sha. Push the tag in addition
# to the "latest" tag already pushed.
GIT_SHA_TAG=${SOURCE_COMMIT:0:12}
docker tag $IMAGE_NAME $DOCKER_REPO:$GIT_SHA_TAG
docker push $DOCKER_REPO:$GIT_SHA_TAG
# Create a working directory.
BUILD_TIMESTAMP=$(date -u +%FT%TZ)
WORKDIR=$(mktemp -d)
GIT_URI="git@github.com:jupyter/docker-stacks.wiki.git"
GIT_SANDBOX="${WORKDIR}/docker-stacks.wiki"
IMAGE_SHORT_NAME=$(basename $DOCKER_REPO)
MANIFEST_FILE="${GIT_SANDBOX}/manifests/${IMAGE_SHORT_NAME}-${GIT_SHA_TAG}.md"
INDEX_FILE="${GIT_SANDBOX}/Home.md"
# Configure git so it can push back to GitHub.
eval $(ssh-agent -s)
ssh-add <(base64 -d <(echo "$DEPLOY_KEY"))
ssh-add -l
git config --global user.email "jupyter@googlegroups.com"
git config --global user.name "Jupyter Docker Stacks"
# Glone the GitHub project wiki.
pushd "$WORKDIR"
git clone "$GIT_URI"
popd
# Render the build manifest template.
mkdir -p $(dirname "$MANIFEST_FILE")
source hooks/manifest.tmpl
source hooks/index.tmpl
# Push the wiki update back to GitHub.
pushd "$GIT_SANDBOX"
git add .
git commit -m "DOC: Build ${MANIFEST_FILE}"
git push -u origin master
popd
# Shutdown the ssh agent for good measure.
ssh-agent -k
# Invoke all downstream build triggers.
set +e
for url in $(echo $NEXT_BUILD_TRIGGERS | sed "s/,/ /g")
do
curl -X POST $url
done
#!/bin/bash #!/bin/bash
set -e set -e
# Tag the latest build with the short git sha as well as version of key runtimes # Apply tags
# and packages. GIT_SHA_TAG=${GITHUB_SHA:0:12}
GIT_SHA_TAG=${SOURCE_COMMIT:0:12}
docker tag $IMAGE_NAME "$DOCKER_REPO:$GIT_SHA_TAG" docker tag $IMAGE_NAME "$DOCKER_REPO:$GIT_SHA_TAG"
PY_VERSION_TAG="python-$(docker run --rm ${IMAGE_NAME} python --version 2>&1 | awk '{print $2}')" PY_VERSION_TAG="python-$(docker run --rm ${IMAGE_NAME} python --version 2>&1 | awk '{print $2}')"
docker tag $IMAGE_NAME "$DOCKER_REPO:$PY_VERSION_TAG" docker tag $IMAGE_NAME "$DOCKER_REPO:$PY_VERSION_TAG"
...@@ -17,3 +16,63 @@ LAB_VERSION_TAG="lab-$(docker run --rm -a STDOUT ${IMAGE_NAME} jupyter-lab --ver ...@@ -17,3 +16,63 @@ LAB_VERSION_TAG="lab-$(docker run --rm -a STDOUT ${IMAGE_NAME} jupyter-lab --ver
docker tag $IMAGE_NAME "$DOCKER_REPO:${LAB_VERSION_TAG%%\r}" docker tag $IMAGE_NAME "$DOCKER_REPO:${LAB_VERSION_TAG%%\r}"
HUB_VERSION_TAG="hub-$(docker run --rm -a STDOUT ${IMAGE_NAME} jupyterhub --version | tr -d '\r')" HUB_VERSION_TAG="hub-$(docker run --rm -a STDOUT ${IMAGE_NAME} jupyterhub --version | tr -d '\r')"
docker tag $IMAGE_NAME "$DOCKER_REPO:${HUB_VERSION_TAG%%\r}" docker tag $IMAGE_NAME "$DOCKER_REPO:${HUB_VERSION_TAG%%\r}"
# Update index
INDEX_ROW="|\`${BUILD_TIMESTAMP}\`|\`jupyter/${IMAGE_SHORT_NAME}:${GIT_SHA_TAG}\`<br />\`jupyter/${IMAGE_SHORT_NAME}:${PY_VERSION_TAG}\`<br />\`jupyter/${IMAGE_SHORT_NAME}:${JULIA_VERSION_TAG}\`<br />\`jupyter/${IMAGE_SHORT_NAME}:${R_VERSION_TAG}\`<br />\`jupyter/${IMAGE_SHORT_NAME}:${NB_VERSION_TAG}\`<br />\`jupyter/${IMAGE_SHORT_NAME}:${LAB_VERSION_TAG}\`<br />\`jupyter/${IMAGE_SHORT_NAME}:${HUB_VERSION_TAG}\`|[Git diff](https://github.com/jupyter/docker-stacks/commit/${GITHUB_SHA})<br />[Dockerfile](https://github.com/jupyter/docker-stacks/blob/${GITHUB_SHA}/${IMAGE_SHORT_NAME}/Dockerfile)<br />[Build manifest](./${IMAGE_SHORT_NAME}-${GIT_SHA_TAG})|"
sed "/|-|/a ${INDEX_ROW}" -i "${WIKI_PATH}/Home.md"
# Build manifest
MANIFEST_FILE="${WIKI_PATH}/manifests/${IMAGE_SHORT_NAME}-${GIT_SHA_TAG}.md"
mkdir -p $(dirname "$MANIFEST_FILE")
cat << EOF > "$MANIFEST_FILE"
* Build datetime: ${BUILD_TIMESTAMP}
* Docker image: ${DOCKER_REPO}:${GIT_SHA_TAG}
* Docker image size: $(docker images ${IMAGE_NAME} --format "{{.Size}}")
* Git commit SHA: [${GITHUB_SHA}](https://github.com/jupyter/docker-stacks/commit/${GITHUB_SHA})
* Git commit message:
\`\`\`
${COMMIT_MSG}
\`\`\`
## Julia Packages
\`\`\`
$(docker run --rm ${IMAGE_NAME} julia -E 'using InteractiveUtils; versioninfo()')
\`\`\`
\`\`\`
$(docker run --rm ${IMAGE_NAME} julia -E 'import Pkg; Pkg.status()')
\`\`\`
## Python Packages
\`\`\`
$(docker run --rm ${IMAGE_NAME} python --version)
\`\`\`
\`\`\`
$(docker run --rm ${IMAGE_NAME} conda info)
\`\`\`
\`\`\`
$(docker run --rm ${IMAGE_NAME} conda list)
\`\`\`
## R Packages
\`\`\`
$(docker run --rm ${IMAGE_NAME} R --version)
\`\`\`
\`\`\`
$(docker run --rm ${IMAGE_NAME} R --silent -e 'installed.packages(.Library)[, c(1,3)]')
\`\`\`
## Apt Packages
\`\`\`
$(docker run --rm ${IMAGE_NAME} apt list --installed)
\`\`\`
EOF
\ No newline at end of file
# New Features # New Features
Thank you for contributing to the Jupyter Docker Stacks! We review pull requests of new features (e.g., new packages, new scripts, new flags) to balance the value of the images to the Jupyter community with the cost of maintaining the images over time. Thank you for contributing to the Jupyter Docker Stacks! We review pull requests of new features
(e.g., new packages, new scripts, new flags) to balance the value of the images to the Jupyter
community with the cost of maintaining the images over time.
## Suggesting a New Feature ## Suggesting a New Feature
Please follow the process below to suggest a new feature for inclusion in one of the core stacks: Please follow the process below to suggest a new feature for inclusion in one of the core stacks:
1. [Open a GitHub issue](https://github.com/jupyter/docker-stacks/issues) describing the feature you'd like to contribute. 1. [Open a GitHub issue](https://github.com/jupyter/docker-stacks/issues) describing the feature
2. Discuss with the maintainers whether the addition makes sense in [one of the core stacks](../using/selecting.md#Core-Stacks), as a [recipe in the documentation](recipes.md), as a [community stack](stacks.md), or as something else entirely. you'd like to contribute.
2. Discuss with the maintainers whether the addition makes sense in
[one of the core stacks](../using/selecting.md#Core-Stacks), as a
[recipe in the documentation](recipes.md), as a [community stack](stacks.md), or as something
else entirely.
## Selection Criteria ## Selection Criteria
Roughly speaking, we evaluate new features based on the following criteria: Roughly speaking, we evaluate new features based on the following criteria:
* **Usefulness to Jupyter users**: Is the feature generally applicable across domains? Does it work with Jupyter Notebook, JupyterLab, JupyterHub, etc.? - **Usefulness to Jupyter users**: Is the feature generally applicable across domains? Does it work
* **Fit with the image purpose**: Does the feature match the theme of the stack in which it will be added? Would it fit better in a new, community stack? with Jupyter Notebook, JupyterLab, JupyterHub, etc.?
* **Complexity of build / runtime configuration**: How many lines of code does the feature require in one of the Dockerfiles or startup scripts? Does it require new scripts entirely? Do users need to adjust how they use the images? - **Fit with the image purpose**: Does the feature match the theme of the stack in which it will be
* **Impact on image metrics**: How many bytes does the feature and its dependencies add to the image(s)? How many minutes do they add to the build time? added? Would it fit better in a new, community stack?
* **Ability to support the addition**: Can existing maintainers answer user questions and address future build issues? Are the contributors interested in helping with long-term maintenance? Can we write tests to ensure the feature continues to work over time? - **Complexity of build / runtime configuration**: How many lines of code does the feature require
in one of the Dockerfiles or startup scripts? Does it require new scripts entirely? Do users need
to adjust how they use the images?
- **Impact on image metrics**: How many bytes does the feature and its dependencies add to the
image(s)? How many minutes do they add to the build time?
- **Ability to support the addition**: Can existing maintainers answer user questions and address
future build issues? Are the contributors interested in helping with long-term maintenance? Can we
write tests to ensure the feature continues to work over time?
## Submitting a Pull Request ## Submitting a Pull Request
If there's agreement that the feature belongs in one or more of the core stacks: If there's agreement that the feature belongs in one or more of the core stacks:
1. Implement the feature in a local clone of the `jupyter/docker-stacks` project. 1. Implement the feature in a local clone of the `jupyter/docker-stacks` project.
2. Please build the image locally before submitting a pull request. Building the image locally shortens the debugging cycle by taking some load off [Travis CI](http://travis-ci.org/), which graciously provides free build services for open source projects like this one. If you use `make`, call: 2. Please build the image locally before submitting a pull request. Building the image locally
```bash shortens the debugging cycle by taking some load off GitHub Actions, which graciously provide
make build/somestack-notebook free build services for open source projects like this one. If you use `make`, call:
``` ```bash
3. [Submit a pull request](https://github.com/PointCloudLibrary/pcl/wiki/A-step-by-step-guide-on-preparing-and-submitting-a-pull-request) (PR) with your changes. make build/somestack-notebook
4. Watch for Travis to report a build success or failure for your PR on GitHub. ```
3. [Submit a pull request](https://github.com/PointCloudLibrary/pcl/wiki/A-step-by-step-guide-on-preparing-and-submitting-a-pull-request)
(PR) with your changes.
4. Watch for GitHub to report a build success or failure for your PR on GitHub.
5. Discuss changes with the maintainers and address any build issues. 5. Discuss changes with the maintainers and address any build issues.
# Package Updates # Package Updates
We actively seek pull requests which update packages already included in the project Dockerfiles. This is a great way for first-time contributors to participate in developing the Jupyter Docker Stacks. We actively seek pull requests which update packages already included in the project Dockerfiles.
This is a great way for first-time contributors to participate in developing the Jupyter Docker
Stacks.
Please follow the process below to update a package version: Please follow the process below to update a package version:
1. Locate the Dockerfile containing the library you wish to update (e.g., [base-notebook/Dockerfile](https://github.com/jupyter/docker-stacks/blob/master/base-notebook/Dockerfile), [scipy-notebook/Dockerfile](https://github.com/jupyter/docker-stacks/blob/master/scipy-notebook/Dockerfile)) 1. Locate the Dockerfile containing the library you wish to update (e.g.,
2. Adjust the version number for the package. We prefer to pin the major and minor version number of packages so as to minimize rebuild side-effects when users submit pull requests (PRs). For example, you'll find the Jupyter Notebook package, `notebook`, installed using conda with `notebook=5.4.*`. [base-notebook/Dockerfile](https://github.com/jupyter/docker-stacks/blob/master/base-notebook/Dockerfile),
3. Please build the image locally before submitting a pull request. Building the image locally shortens the debugging cycle by taking some load off [Travis CI](http://travis-ci.org/), which graciously provides free build services for open source projects like this one. If you use `make`, call: [scipy-notebook/Dockerfile](https://github.com/jupyter/docker-stacks/blob/master/scipy-notebook/Dockerfile))
```bash 2. Adjust the version number for the package. We prefer to pin the major and minor version number of
make build/somestack-notebook packages so as to minimize rebuild side-effects when users submit pull requests (PRs). For
``` example, you'll find the Jupyter Notebook package, `notebook`, installed using conda with
4. [Submit a pull request](https://github.com/PointCloudLibrary/pcl/wiki/A-step-by-step-guide-on-preparing-and-submitting-a-pull-request) (PR) with your changes. `notebook=5.4.*`.
5. Watch for Travis to report a build success or failure for your PR on GitHub. 3. Please build the image locally before submitting a pull request. Building the image locally
6. Discuss changes with the maintainers and address any build issues. Version conflicts are the most common problem. You may need to upgrade additional packages to fix build failures. shortens the debugging cycle by taking some load off GitHub Actions, which graciously provide
free build services for open source projects like this one. If you use `make`, call:
```bash
make build/somestack-notebook
```
4. [Submit a pull request](https://github.com/PointCloudLibrary/pcl/wiki/A-step-by-step-guide-on-preparing-and-submitting-a-pull-request)
(PR) with your changes.
5. Watch for GitHub to report a build success or failure for your PR on GitHub.
6. Discuss changes with the maintainers and address any build issues. Version conflicts are the most
common problem. You may need to upgrade additional packages to fix build failures.
## Notes ## Notes
In order to help identifying packages that can be updated you can use the following helper tool. In order to help identifying packages that can be updated you can use the following helper tool. It
It will list all the packages installed in the `Dockerfile` that can be updated -- dependencies are filtered to focus only on requested packages. will list all the packages installed in the `Dockerfile` that can be updated -- dependencies are
filtered to focus only on requested packages.
```bash ```bash
$ make check-outdated/base-notebook $ make check-outdated/base-notebook
# INFO test_outdated:test_outdated.py:80 3/8 (38%) packages could be updated # INFO test_outdated:test_outdated.py:80 3/8 (38%) packages could be updated
# INFO test_outdated:test_outdated.py:82 # INFO test_outdated:test_outdated.py:82
# Package Current Newest # Package Current Newest
# ---------- --------- -------- # ---------- --------- --------
# conda 4.7.12 4.8.2 # conda 4.7.12 4.8.2
......
This diff is collapsed.
# Image Tests # Image Tests
We greatly appreciate pull requests that extend the automated tests that vet the basic functionality of the Docker images. We greatly appreciate pull requests that extend the automated tests that vet the basic functionality
of the Docker images.
## How the Tests Work ## How the Tests Work
Travis executes `make build-test-all` against pull requests submitted to the `jupyter/docker-stacks` repository. This `make` command builds every docker image. After building each image, the `make` command executes `pytest` to run both image-specific tests like those in [base-notebook/test/](https://github.com/jupyter/docker-stacks/tree/master/base-notebook/test) and common tests defined in [test/](https://github.com/jupyter/docker-stacks/tree/master/test). Both kinds of tests make use of global [pytest fixtures](https://docs.pytest.org/en/latest/fixture.html) defined in the [conftest.py](https://github.com/jupyter/docker-stacks/blob/master/conftest.py) file at the root of the projects. GitHub executes `make build-test-all` against pull requests submitted to the `jupyter/docker-stacks`
repository. This `make` command builds every docker image. After building each image, the `make`
command executes `pytest` to run both image-specific tests like those in
[base-notebook/test/](https://github.com/jupyter/docker-stacks/tree/master/base-notebook/test) and
common tests defined in [test/](https://github.com/jupyter/docker-stacks/tree/master/test). Both
kinds of tests make use of global [pytest fixtures](https://docs.pytest.org/en/latest/fixture.html)
defined in the [conftest.py](https://github.com/jupyter/docker-stacks/blob/master/conftest.py) file
at the root of the projects.
## Contributing New Tests ## Contributing New Tests
Please follow the process below to add new tests: Please follow the process below to add new tests:
1. If the test should run against every image built, add your test code to one of the modules in [test/](https://github.com/jupyter/docker-stacks/tree/master/test) or create a new module. 1. If the test should run against every image built, add your test code to one of the modules in
2. If your test should run against a single image, add your test code to one of the modules in `some-notebook/test/` or create a new module. [test/](https://github.com/jupyter/docker-stacks/tree/master/test) or create a new module.
2. If your test should run against a single image, add your test code to one of the modules in
`some-notebook/test/` or create a new module.
3. Build one or more images you intend to test and run the tests locally. If you use `make`, call: 3. Build one or more images you intend to test and run the tests locally. If you use `make`, call:
```bash ```bash
make build/somestack-notebook make build/somestack-notebook
make test/somestack-notebook make test/somestack-notebook
``` ```
4. [Submit a pull request](https://github.com/PointCloudLibrary/pcl/wiki/A-step-by-step-guide-on-preparing-and-submitting-a-pull-request) (PR) with your changes. 4. [Submit a pull request](https://github.com/PointCloudLibrary/pcl/wiki/A-step-by-step-guide-on-preparing-and-submitting-a-pull-request)
5. Watch for Travis to report a build success or failure for your PR on GitHub. (PR) with your changes.
6. Discuss changes with the maintainers and address any issues running the tests on Travis. 5. Watch for GitHub to report a build success or failure for your PR on GitHub.
6. Discuss changes with the maintainers and address any issues running the tests on GitHub.
...@@ -4,41 +4,61 @@ ...@@ -4,41 +4,61 @@
To build new images on Docker Cloud and publish them to the Docker Hub registry, do the following: To build new images on Docker Cloud and publish them to the Docker Hub registry, do the following:
1. Make sure Travis is green for a PR. 1. Make sure GitHub Actions status checks pas for the PR.
2. Merge the PR. 2. Merge the PR.
3. Monitor the Docker Cloud build status for each of the stacks, starting with [jupyter/base-notebook](https://cloud.docker.com/app/jupyter/repository/docker/jupyter/base-notebook/general) and ending with [jupyter/all-spark-notebook](https://cloud.docker.com/app/jupyter/repository/docker/jupyter/all-spark-notebook/general). See the [stack hierarchy diagram](../using/selecting.html#image-relationships) for the current, complete build order. 3. Monitor the Docker Cloud build status for each of the stacks, starting with
4. Manually click the retry button next to any build that fails to resume that build and any dependent builds. [jupyter/base-notebook](https://cloud.docker.com/app/jupyter/repository/docker/jupyter/base-notebook/general)
5. Try to avoid merging another PR to master until all outstanding builds complete. There's no way at present to propagate the git SHA to build through the Docker Cloud build trigger API. Every build trigger works off of master HEAD. and ending with
[jupyter/all-spark-notebook](https://cloud.docker.com/app/jupyter/repository/docker/jupyter/all-spark-notebook/general).
See the [stack hierarchy diagram](../using/selecting.html#image-relationships) for the current,
complete build order.
4. Manually click the retry button next to any build that fails to resume that build and any
dependent builds.
5. Try to avoid merging another PR to master until all outstanding builds complete. There's no way
at present to propagate the git SHA to build through the Docker Cloud build trigger API. Every
build trigger works off of master HEAD.
## Updating the Ubuntu Base Image ## Updating the Ubuntu Base Image
When there's a security fix in the Ubuntu base image or after some time passes, it's a good idea to update the pinned SHA in the [jupyter/base-notebook Dockerfile](https://github.com/jupyter/docker-stacks/blob/master/base-notebook/Dockerfile). Submit it as a regular PR and go through the build process. Expect the build to take a while to complete: every image layer will rebuild. When there's a security fix in the Ubuntu base image or after some time passes, it's a good idea to
update the pinned SHA in the
[jupyter/base-notebook Dockerfile](https://github.com/jupyter/docker-stacks/blob/master/base-notebook/Dockerfile).
Submit it as a regular PR and go through the build process. Expect the build to take a while to
complete: every image layer will rebuild.
## Adding a New Core Image to Docker Cloud ## Adding a New Core Image to Docker Cloud
When there's a new stack definition, do the following before merging the PR with the new stack: When there's a new stack definition, do the following before merging the PR with the new stack:
1. Ensure the PR includes an update to the stack overview diagram [in the documentation](https://github.com/jupyter/docker-stacks/blob/master/docs/using/selecting.md#image-relationships). The image links to the [blockdiag source](http://interactive.blockdiag.com/) used to create it. 1. Ensure the PR includes an update to the stack overview diagram
2. Ensure the PR updates the Makefile which is used to build the stacks in order on Travis CI. [in the documentation](https://github.com/jupyter/docker-stacks/blob/master/docs/using/selecting.md#image-relationships).
3. Create a new repository in the `jupyter` org on Docker Cloud named after the stack folder in the git repo. The image links to the [blockdiag source](http://interactive.blockdiag.com/) used to create it.
2. Ensure the PR updates the Makefile which is used to build the stacks in order on GitHub Actions.
3. Create a new repository in the `jupyter` org on Docker Cloud named after the stack folder in the
git repo.
4. Grant the `stacks` team permission to write to the repo. 4. Grant the `stacks` team permission to write to the repo.
5. Click *Builds* and then *Configure Automated Builds* for the repository. 5. Click _Builds_ and then _Configure Automated Builds_ for the repository.
6. Select `jupyter/docker-stacks` as the source repository. 6. Select `jupyter/docker-stacks` as the source repository.
7. Choose *Build on Docker Cloud's infrastructure using a Small node* unless you have reason to believe a bigger host is required. 7. Choose _Build on Docker Cloud's infrastructure using a Small node_ unless you have reason to
8. Update the *Build Context* in the default build rule to be `/<name-of-the-stack>`. believe a bigger host is required.
9. Toggle *Autobuild* to disabled unless the stack is a new root stack (e.g., like `jupyter/base-notebook`). 8. Update the _Build Context_ in the default build rule to be `/<name-of-the-stack>`.
9. Toggle _Autobuild_ to disabled unless the stack is a new root stack (e.g., like
`jupyter/base-notebook`).
10. If the new stack depends on the build of another stack in the hierarchy: 10. If the new stack depends on the build of another stack in the hierarchy:
1. Hit *Save* and then click *Configure Automated Builds*. 1. Hit _Save_ and then click _Configure Automated Builds_.
2. At the very bottom, add a build trigger named *Stack hierarchy trigger*. 2. At the very bottom, add a build trigger named _Stack hierarchy trigger_.
3. Copy the build trigger URL. 3. Copy the build trigger URL.
4. Visit the parent repository *Builds* page and click *Configure Automated Builds*. 4. Visit the parent repository _Builds_ page and click _Configure Automated Builds_.
5. Add the URL you copied to the *NEXT_BUILD_TRIGGERS* environment variable comma separated list of URLs, creating that environment variable if it does not already exist. 5. Add the URL you copied to the _NEXT_BUILD_TRIGGERS_ environment variable comma separated list
6. Hit *Save*. of URLs, creating that environment variable if it does not already exist.
6. Hit _Save_.
11. If the new stack should trigger other dependent builds: 11. If the new stack should trigger other dependent builds:
1. Add an environment variable named *NEXT_BUILD_TRIGGERS*. 1. Add an environment variable named _NEXT_BUILD_TRIGGERS_.
2. Copy the build trigger URLs from the dependent builds into the *NEXT_BUILD_TRIGGERS* comma separated list of URLs. 2. Copy the build trigger URLs from the dependent builds into the _NEXT_BUILD_TRIGGERS_ comma
3. Hit *Save*. separated list of URLs.
12. Adjust other *NEXT_BUILD_TRIGGERS* values as needed so that the build order matches that in the stack hierarchy diagram. 3. Hit _Save_.
12. Adjust other _NEXT_BUILD_TRIGGERS_ values as needed so that the build order matches that in the
stack hierarchy diagram.
## Adding a New Maintainer Account ## Adding a New Maintainer Account
...@@ -61,22 +81,22 @@ If automated builds on Docker Cloud have got you down, do the following to push ...@@ -61,22 +81,22 @@ If automated builds on Docker Cloud have got you down, do the following to push
First enable translation on Transifex: First enable translation on Transifex:
1. Visit https://www.transifex.com/project-jupyter/jupyter-docker-stacks-1/languages/ 1. Visit https://www.transifex.com/project-jupyter/jupyter-docker-stacks-1/languages/
2. Click *Edit Languages* in the top right. 2. Click _Edit Languages_ in the top right.
3. Select the language from the dropdown. 3. Select the language from the dropdown.
4. Click *Apply*. 4. Click _Apply_.
Then setup a subproject on ReadTheDocs for the language: Then setup a subproject on ReadTheDocs for the language:
1. Visit https://readthedocs.org/dashboard/import/manual/ 1. Visit https://readthedocs.org/dashboard/import/manual/
2. Enter *jupyter-docker-stacks-language_abbreviation* for the project name. 2. Enter _jupyter-docker-stacks-language_abbreviation_ for the project name.
3. Enter https://github.com/jupyter/docker-stacks for the URL. 3. Enter https://github.com/jupyter/docker-stacks for the URL.
4. Check *Edit advanced project options*. 4. Check _Edit advanced project options_.
5. Click *Next*. 5. Click _Next_.
6. Select the *Language* from the dropdown on the next screen. 6. Select the _Language_ from the dropdown on the next screen.
7. Click *Finish*. 7. Click _Finish_.
Finally link the new language subproject to the top level doc project: Finally link the new language subproject to the top level doc project:
1. Visit https://readthedocs.org/dashboard/jupyter-docker-stacks/translations/ 1. Visit https://readthedocs.org/dashboard/jupyter-docker-stacks/translations/
2. Select the subproject you created from the *Project* dropdown. 2. Select the subproject you created from the _Project_ dropdown.
3. Click *Add*. 3. Click _Add_.
This diff is collapsed.
...@@ -25,8 +25,6 @@ RUN apt-get update && apt-get install -yq --no-install-recommends \ ...@@ -25,8 +25,6 @@ RUN apt-get update && apt-get install -yq --no-install-recommends \
texlive-xetex \ texlive-xetex \
texlive-fonts-recommended \ texlive-fonts-recommended \
texlive-plain-generic \ texlive-plain-generic \
# Optional dependency
texlive-fonts-extra \
# ---- # ----
tzdata \ tzdata \
unzip \ unzip \
......
INDEX_ROW="|\`${BUILD_TIMESTAMP}\`|\`jupyter/${IMAGE_SHORT_NAME}:${GIT_SHA_TAG}\`|[Git diff](https://github.com/jupyter/docker-stacks/commit/${SOURCE_COMMIT})<br />[Dockerfile](https://github.com/jupyter/docker-stacks/blob/${SOURCE_COMMIT}/${IMAGE_SHORT_NAME}/Dockerfile)<br />[Build manifest](./${IMAGE_SHORT_NAME}-${GIT_SHA_TAG})|"
sed "/|-|/a ${INDEX_ROW}" -i "$INDEX_FILE"
\ No newline at end of file
cat << EOF > "$MANIFEST_FILE"
* Build datetime: ${BUILD_TIMESTAMP}
* DockerHub build code: ${BUILD_CODE}
* Docker image: ${DOCKER_REPO}:${GIT_SHA_TAG}
* Docker image size: $(docker images ${IMAGE_NAME} --format "{{.Size}}")
* Git commit SHA: [${SOURCE_COMMIT}](https://github.com/jupyter/docker-stacks/commit/${SOURCE_COMMIT})
* Git commit message:
\`\`\`
${COMMIT_MSG}
\`\`\`
## Python Packages
\`\`\`
$(docker run --rm ${IMAGE_NAME} python --version)
\`\`\`
\`\`\`
$(docker run --rm ${IMAGE_NAME} conda info)
\`\`\`
\`\`\`
$(docker run --rm ${IMAGE_NAME} conda list)
\`\`\`
## Apt Packages
\`\`\`
$(docker run --rm ${IMAGE_NAME} apt list --installed)
\`\`\`
EOF
\ No newline at end of file
#!/bin/bash
set -e
# Tag the latest build with the short git sha. Push the tag in addition
# to the "latest" tag already pushed.
GIT_SHA_TAG=${SOURCE_COMMIT:0:12}
docker tag $IMAGE_NAME $DOCKER_REPO:$GIT_SHA_TAG
docker push $DOCKER_REPO:$GIT_SHA_TAG
# Create a working directory.
BUILD_TIMESTAMP=$(date -u +%FT%TZ)
WORKDIR=$(mktemp -d)
GIT_URI="git@github.com:jupyter/docker-stacks.wiki.git"
GIT_SANDBOX="${WORKDIR}/docker-stacks.wiki"
IMAGE_SHORT_NAME=$(basename $DOCKER_REPO)
MANIFEST_FILE="${GIT_SANDBOX}/manifests/${IMAGE_SHORT_NAME}-${GIT_SHA_TAG}.md"
INDEX_FILE="${GIT_SANDBOX}/Home.md"
# Configure git so it can push back to GitHub.
eval $(ssh-agent -s)
ssh-add <(base64 -d <(echo "$DEPLOY_KEY"))
ssh-add -l
git config --global user.email "jupyter@googlegroups.com"
git config --global user.name "Jupyter Docker Stacks"
# Glone the GitHub project wiki.
pushd "$WORKDIR"
git clone "$GIT_URI"
popd
# Render the build manifest template.
mkdir -p $(dirname "$MANIFEST_FILE")
source hooks/manifest.tmpl
source hooks/index.tmpl
# Push the wiki update back to GitHub.
pushd "$GIT_SANDBOX"
git add .
git commit -m "DOC: Build ${MANIFEST_FILE}"
git push -u origin master
popd
# Shutdown the ssh agent for good measure.
ssh-agent -k
# Invoke all downstream build triggers.
set +e
for url in $(echo $NEXT_BUILD_TRIGGERS | sed "s/,/ /g")
do
curl -X POST $url
done
\ No newline at end of file
#!/bin/bash
set -e
# Apply tags
GIT_SHA_TAG=${GITHUB_SHA:0:12}
docker tag $IMAGE_NAME "$DOCKER_REPO:$GIT_SHA_TAG"
# Update index
INDEX_ROW="|\`${BUILD_TIMESTAMP}\`|\`jupyter/${IMAGE_SHORT_NAME}:${GIT_SHA_TAG}\`|[Git diff](https://github.com/jupyter/docker-stacks/commit/${GITHUB_SHA})<br />[Dockerfile](https://github.com/jupyter/docker-stacks/blob/${GITHUB_SHA}/${IMAGE_SHORT_NAME}/Dockerfile)<br />[Build manifest](./${IMAGE_SHORT_NAME}-${GIT_SHA_TAG})|"
sed "/|-|/a ${INDEX_ROW}" -i "${WIKI_PATH}/Home.md"
# Build manifest
MANIFEST_FILE="${WIKI_PATH}/manifests/${IMAGE_SHORT_NAME}-${GIT_SHA_TAG}.md"
mkdir -p $(dirname "$MANIFEST_FILE")
cat << EOF > "$MANIFEST_FILE" cat << EOF > "$MANIFEST_FILE"
* Build datetime: ${BUILD_TIMESTAMP} * Build datetime: ${BUILD_TIMESTAMP}
* DockerHub build code: ${BUILD_CODE}
* Docker image: ${DOCKER_REPO}:${GIT_SHA_TAG} * Docker image: ${DOCKER_REPO}:${GIT_SHA_TAG}
* Docker image size: $(docker images ${IMAGE_NAME} --format "{{.Size}}") * Docker image size: $(docker images ${IMAGE_NAME} --format "{{.Size}}")
* Git commit SHA: [${SOURCE_COMMIT}](https://github.com/jupyter/docker-stacks/commit/${SOURCE_COMMIT}) * Git commit SHA: [${GITHUB_SHA}](https://github.com/jupyter/docker-stacks/commit/${GITHUB_SHA})
* Git commit message: * Git commit message:
\`\`\` \`\`\`
${COMMIT_MSG} ${COMMIT_MSG}
\`\`\` \`\`\`
## R Packages
\`\`\`
$(docker run --rm ${IMAGE_NAME} R --version)
\`\`\`
\`\`\`
$(docker run --rm ${IMAGE_NAME} R --silent -e 'installed.packages(.Library)[, c(1,3)]')
\`\`\`
## Python Packages ## Python Packages
\`\`\` \`\`\`
......
INDEX_ROW="|\`${BUILD_TIMESTAMP}\`|\`jupyter/${IMAGE_SHORT_NAME}:${GIT_SHA_TAG}\`|[Git diff](https://github.com/jupyter/docker-stacks/commit/${SOURCE_COMMIT})<br />[Dockerfile](https://github.com/jupyter/docker-stacks/blob/${SOURCE_COMMIT}/${IMAGE_SHORT_NAME}/Dockerfile)<br />[Build manifest](./${IMAGE_SHORT_NAME}-${GIT_SHA_TAG})|"
sed "/|-|/a ${INDEX_ROW}" -i "$INDEX_FILE"
\ No newline at end of file
#!/bin/bash
set -e
# Tag the latest build with the short git sha. Push the tag in addition
# to the "latest" tag already pushed.
GIT_SHA_TAG=${SOURCE_COMMIT:0:12}
docker tag $IMAGE_NAME $DOCKER_REPO:$GIT_SHA_TAG
docker push $DOCKER_REPO:$GIT_SHA_TAG
# Create a working directory.
BUILD_TIMESTAMP=$(date -u +%FT%TZ)
WORKDIR=$(mktemp -d)
GIT_URI="git@github.com:jupyter/docker-stacks.wiki.git"
GIT_SANDBOX="${WORKDIR}/docker-stacks.wiki"
IMAGE_SHORT_NAME=$(basename $DOCKER_REPO)
MANIFEST_FILE="${GIT_SANDBOX}/manifests/${IMAGE_SHORT_NAME}-${GIT_SHA_TAG}.md"
INDEX_FILE="${GIT_SANDBOX}/Home.md"
# Configure git so it can push back to GitHub.
eval $(ssh-agent -s)
ssh-add <(base64 -d <(echo "$DEPLOY_KEY"))
ssh-add -l
git config --global user.email "jupyter@googlegroups.com"
git config --global user.name "Jupyter Docker Stacks"
# Glone the GitHub project wiki.
pushd "$WORKDIR"
git clone "$GIT_URI"
popd
# Render the build manifest template.
mkdir -p $(dirname "$MANIFEST_FILE")
source hooks/manifest.tmpl
source hooks/index.tmpl
# Push the wiki update back to GitHub.
pushd "$GIT_SANDBOX"
git add .
git commit -m "DOC: Build ${MANIFEST_FILE}"
git push -u origin master
popd
# Shutdown the ssh agent for good measure.
ssh-agent -k
# Invoke all downstream build triggers.
set +e
for url in $(echo $NEXT_BUILD_TRIGGERS | sed "s/,/ /g")
do
curl -X POST $url
done
\ No newline at end of file
#!/bin/bash
set -e
# Apply tags
GIT_SHA_TAG=${GITHUB_SHA:0:12}
docker tag $IMAGE_NAME "$DOCKER_REPO:$GIT_SHA_TAG"
# Update index
INDEX_ROW="|\`${BUILD_TIMESTAMP}\`|\`jupyter/${IMAGE_SHORT_NAME}:${GIT_SHA_TAG}\`|[Git diff](https://github.com/jupyter/docker-stacks/commit/${GITHUB_SHA})<br />[Dockerfile](https://github.com/jupyter/docker-stacks/blob/${GITHUB_SHA}/${IMAGE_SHORT_NAME}/Dockerfile)<br />[Build manifest](./${IMAGE_SHORT_NAME}-${GIT_SHA_TAG})|"
sed "/|-|/a ${INDEX_ROW}" -i "${WIKI_PATH}/Home.md"
# Build manifest
MANIFEST_FILE="${WIKI_PATH}/manifests/${IMAGE_SHORT_NAME}-${GIT_SHA_TAG}.md"
mkdir -p $(dirname "$MANIFEST_FILE")
cat << EOF > "$MANIFEST_FILE" cat << EOF > "$MANIFEST_FILE"
* Build datetime: ${BUILD_TIMESTAMP} * Build datetime: ${BUILD_TIMESTAMP}
* DockerHub build code: ${BUILD_CODE}
* Docker image: ${DOCKER_REPO}:${GIT_SHA_TAG} * Docker image: ${DOCKER_REPO}:${GIT_SHA_TAG}
* Docker image size: $(docker images ${IMAGE_NAME} --format "{{.Size}}") * Docker image size: $(docker images ${IMAGE_NAME} --format "{{.Size}}")
* Git commit SHA: [${SOURCE_COMMIT}](https://github.com/jupyter/docker-stacks/commit/${SOURCE_COMMIT}) * Git commit SHA: [${GITHUB_SHA}](https://github.com/jupyter/docker-stacks/commit/${GITHUB_SHA})
* Git commit message: * Git commit message:
\`\`\` \`\`\`
${COMMIT_MSG} ${COMMIT_MSG}
......
INDEX_ROW="|\`${BUILD_TIMESTAMP}\`|\`jupyter/${IMAGE_SHORT_NAME}:${GIT_SHA_TAG}\`|[Git diff](https://github.com/jupyter/docker-stacks/commit/${SOURCE_COMMIT})<br />[Dockerfile](https://github.com/jupyter/docker-stacks/blob/${SOURCE_COMMIT}/${IMAGE_SHORT_NAME}/Dockerfile)<br />[Build manifest](./${IMAGE_SHORT_NAME}-${GIT_SHA_TAG})|"
sed "/|-|/a ${INDEX_ROW}" -i "$INDEX_FILE"
\ No newline at end of file
#!/bin/bash
set -e
# Tag the latest build with the short git sha. Push the tag in addition
# to the "latest" tag already pushed.
GIT_SHA_TAG=${SOURCE_COMMIT:0:12}
docker tag $IMAGE_NAME $DOCKER_REPO:$GIT_SHA_TAG
docker push $DOCKER_REPO:$GIT_SHA_TAG
# Create a working directory.
BUILD_TIMESTAMP=$(date -u +%FT%TZ)
WORKDIR=$(mktemp -d)
GIT_URI="git@github.com:jupyter/docker-stacks.wiki.git"
GIT_SANDBOX="${WORKDIR}/docker-stacks.wiki"
IMAGE_SHORT_NAME=$(basename $DOCKER_REPO)
MANIFEST_FILE="${GIT_SANDBOX}/manifests/${IMAGE_SHORT_NAME}-${GIT_SHA_TAG}.md"
INDEX_FILE="${GIT_SANDBOX}/Home.md"
# Configure git so it can push back to GitHub.
eval $(ssh-agent -s)
ssh-add <(base64 -d <(echo "$DEPLOY_KEY"))
ssh-add -l
git config --global user.email "jupyter@googlegroups.com"
git config --global user.name "Jupyter Docker Stacks"
# Glone the GitHub project wiki.
pushd "$WORKDIR"
git clone "$GIT_URI"
popd
# Render the build manifest template.
mkdir -p $(dirname "$MANIFEST_FILE")
source hooks/manifest.tmpl
source hooks/index.tmpl
# Push the wiki update back to GitHub.
pushd "$GIT_SANDBOX"
git add .
git commit -m "DOC: Build ${MANIFEST_FILE}"
git push -u origin master
popd
# Shutdown the ssh agent for good measure.
ssh-agent -k
# Invoke all downstream build triggers.
set +e
for url in $(echo $NEXT_BUILD_TRIGGERS | sed "s/,/ /g")
do
curl -X POST $url
done
\ No newline at end of file
#!/bin/bash
set -e
# Apply tags
GIT_SHA_TAG=${GITHUB_SHA:0:12}
docker tag $IMAGE_NAME "$DOCKER_REPO:$GIT_SHA_TAG"
# Update index
INDEX_ROW="|\`${BUILD_TIMESTAMP}\`|\`jupyter/${IMAGE_SHORT_NAME}:${GIT_SHA_TAG}\`|[Git diff](https://github.com/jupyter/docker-stacks/commit/${GITHUB_SHA})<br />[Dockerfile](https://github.com/jupyter/docker-stacks/blob/${GITHUB_SHA}/${IMAGE_SHORT_NAME}/Dockerfile)<br />[Build manifest](./${IMAGE_SHORT_NAME}-${GIT_SHA_TAG})|"
sed "/|-|/a ${INDEX_ROW}" -i "${WIKI_PATH}/Home.md"
# Build manifest
MANIFEST_FILE="${WIKI_PATH}/manifests/${IMAGE_SHORT_NAME}-${GIT_SHA_TAG}.md"
mkdir -p $(dirname "$MANIFEST_FILE")
cat << EOF > "$MANIFEST_FILE" cat << EOF > "$MANIFEST_FILE"
* Build datetime: ${BUILD_TIMESTAMP} * Build datetime: ${BUILD_TIMESTAMP}
* DockerHub build code: ${BUILD_CODE}
* Docker image: ${DOCKER_REPO}:${GIT_SHA_TAG} * Docker image: ${DOCKER_REPO}:${GIT_SHA_TAG}
* Docker image size: $(docker images ${IMAGE_NAME} --format "{{.Size}}") * Docker image size: $(docker images ${IMAGE_NAME} --format "{{.Size}}")
* Git commit SHA: [${SOURCE_COMMIT}](https://github.com/jupyter/docker-stacks/commit/${SOURCE_COMMIT}) * Git commit SHA: [${GITHUB_SHA}](https://github.com/jupyter/docker-stacks/commit/${GITHUB_SHA})
* Git commit message: * Git commit message:
\`\`\` \`\`\`
${COMMIT_MSG} ${COMMIT_MSG}
\`\`\` \`\`\`
## Julia Packages ## R Packages
\`\`\` \`\`\`
$(docker run --rm ${IMAGE_NAME} julia -E 'using InteractiveUtils; versioninfo()') $(docker run --rm ${IMAGE_NAME} R --version)
\`\`\` \`\`\`
\`\`\` \`\`\`
$(docker run --rm ${IMAGE_NAME} julia -E 'import Pkg; Pkg.status()') $(docker run --rm ${IMAGE_NAME} R --silent -e 'installed.packages(.Library)[, c(1,3)]')
\`\`\` \`\`\`
## Python Packages ## Python Packages
...@@ -33,16 +47,6 @@ $(docker run --rm ${IMAGE_NAME} conda info) ...@@ -33,16 +47,6 @@ $(docker run --rm ${IMAGE_NAME} conda info)
$(docker run --rm ${IMAGE_NAME} conda list) $(docker run --rm ${IMAGE_NAME} conda list)
\`\`\` \`\`\`
## R Packages
\`\`\`
$(docker run --rm ${IMAGE_NAME} R --version)
\`\`\`
\`\`\`
$(docker run --rm ${IMAGE_NAME} R --silent -e 'installed.packages(.Library)[, c(1,3)]')
\`\`\`
## Apt Packages ## Apt Packages
\`\`\` \`\`\`
......
docker docker
pre-commit pre-commit
pytest pytest
recommonmark==0.5.0 recommonmark
requests requests
sphinx>=1.6 sphinx>=1.6
sphinx-intl sphinx-intl
......
...@@ -18,11 +18,11 @@ USER $NB_UID ...@@ -18,11 +18,11 @@ USER $NB_UID
RUN conda install --quiet --yes \ RUN conda install --quiet --yes \
'beautifulsoup4=4.9.*' \ 'beautifulsoup4=4.9.*' \
'conda-forge::blas=*=openblas' \ 'conda-forge::blas=*=openblas' \
'bokeh=2.0.*' \ 'bokeh=2.1.*' \
'bottleneck=1.3.*' \ 'bottleneck=1.3.*' \
'cloudpickle=1.4.*' \ 'cloudpickle=1.5.*' \
'cython=0.29.*' \ 'cython=0.29.*' \
'dask=2.15.*' \ 'dask=2.20.*' \
'dill=0.3.*' \ 'dill=0.3.*' \
'h5py=2.10.*' \ 'h5py=2.10.*' \
'hdf5=1.10.*' \ 'hdf5=1.10.*' \
...@@ -34,15 +34,15 @@ RUN conda install --quiet --yes \ ...@@ -34,15 +34,15 @@ RUN conda install --quiet --yes \
'numexpr=2.7.*' \ 'numexpr=2.7.*' \
'pandas=1.0.*' \ 'pandas=1.0.*' \
'patsy=0.5.*' \ 'patsy=0.5.*' \
'protobuf=3.11.*' \ 'protobuf=3.12.*' \
'pytables=3.6.*' \ 'pytables=3.6.*' \
'scikit-image=0.16.*' \ 'scikit-image=0.17.*' \
'scikit-learn=0.23.*' \ 'scikit-learn=0.23.*' \
'scipy=1.4.*' \ 'scipy=1.5.*' \
'seaborn=0.10.*' \ 'seaborn=0.10.*' \
'sqlalchemy=1.3.*' \ 'sqlalchemy=1.3.*' \
'statsmodels=0.11.*' \ 'statsmodels=0.11.*' \
'sympy=1.5.*' \ 'sympy=1.6.*' \
'vincent=0.4.*' \ 'vincent=0.4.*' \
'widgetsnbextension=3.5.*'\ 'widgetsnbextension=3.5.*'\
'xlrd=1.2.*' \ 'xlrd=1.2.*' \
......
INDEX_ROW="|\`${BUILD_TIMESTAMP}\`|\`jupyter/${IMAGE_SHORT_NAME}:${GIT_SHA_TAG}\`|[Git diff](https://github.com/jupyter/docker-stacks/commit/${SOURCE_COMMIT})<br />[Dockerfile](https://github.com/jupyter/docker-stacks/blob/${SOURCE_COMMIT}/${IMAGE_SHORT_NAME}/Dockerfile)<br />[Build manifest](./${IMAGE_SHORT_NAME}-${GIT_SHA_TAG})|"
sed "/|-|/a ${INDEX_ROW}" -i "$INDEX_FILE"
\ No newline at end of file
cat << EOF > "$MANIFEST_FILE"
* Build datetime: ${BUILD_TIMESTAMP}
* DockerHub build code: ${BUILD_CODE}
* Docker image: ${DOCKER_REPO}:${GIT_SHA_TAG}
* Docker image size: $(docker images ${IMAGE_NAME} --format "{{.Size}}")
* Git commit SHA: [${SOURCE_COMMIT}](https://github.com/jupyter/docker-stacks/commit/${SOURCE_COMMIT})
* Git commit message:
\`\`\`
${COMMIT_MSG}
\`\`\`
## Python Packages
\`\`\`
$(docker run --rm ${IMAGE_NAME} python --version)
\`\`\`
\`\`\`
$(docker run --rm ${IMAGE_NAME} conda info)
\`\`\`
\`\`\`
$(docker run --rm ${IMAGE_NAME} conda list)
\`\`\`
## Apt Packages
\`\`\`
$(docker run --rm ${IMAGE_NAME} apt list --installed)
\`\`\`
EOF
\ No newline at end of file
#!/bin/bash
set -e
# Tag the latest build with the short git sha. Push the tag in addition
# to the "latest" tag already pushed.
GIT_SHA_TAG=${SOURCE_COMMIT:0:12}
docker tag $IMAGE_NAME $DOCKER_REPO:$GIT_SHA_TAG
docker push $DOCKER_REPO:$GIT_SHA_TAG
# Create a working directory.
BUILD_TIMESTAMP=$(date -u +%FT%TZ)
WORKDIR=$(mktemp -d)
GIT_URI="git@github.com:jupyter/docker-stacks.wiki.git"
GIT_SANDBOX="${WORKDIR}/docker-stacks.wiki"
IMAGE_SHORT_NAME=$(basename $DOCKER_REPO)
MANIFEST_FILE="${GIT_SANDBOX}/manifests/${IMAGE_SHORT_NAME}-${GIT_SHA_TAG}.md"
INDEX_FILE="${GIT_SANDBOX}/Home.md"
# Configure git so it can push back to GitHub.
eval $(ssh-agent -s)
ssh-add <(base64 -d <(echo "$DEPLOY_KEY"))
ssh-add -l
git config --global user.email "jupyter@googlegroups.com"
git config --global user.name "Jupyter Docker Stacks"
# Glone the GitHub project wiki.
pushd "$WORKDIR"
git clone "$GIT_URI"
popd
# Render the build manifest template.
mkdir -p $(dirname "$MANIFEST_FILE")
source hooks/manifest.tmpl
source hooks/index.tmpl
# Push the wiki update back to GitHub.
pushd "$GIT_SANDBOX"
git add .
git commit -m "DOC: Build ${MANIFEST_FILE}"
git push -u origin master
popd
# Shutdown the ssh agent for good measure.
ssh-agent -k
# Invoke all downstream build triggers.
set +e
for url in $(echo $NEXT_BUILD_TRIGGERS | sed "s/,/ /g")
do
curl -X POST $url
done
\ No newline at end of file
#!/bin/bash
set -e
# Apply tags
GIT_SHA_TAG=${GITHUB_SHA:0:12}
docker tag $IMAGE_NAME "$DOCKER_REPO:$GIT_SHA_TAG"
# Update index
INDEX_ROW="|\`${BUILD_TIMESTAMP}\`|\`jupyter/${IMAGE_SHORT_NAME}:${GIT_SHA_TAG}\`|[Git diff](https://github.com/jupyter/docker-stacks/commit/${GITHUB_SHA})<br />[Dockerfile](https://github.com/jupyter/docker-stacks/blob/${GITHUB_SHA}/${IMAGE_SHORT_NAME}/Dockerfile)<br />[Build manifest](./${IMAGE_SHORT_NAME}-${GIT_SHA_TAG})|"
sed "/|-|/a ${INDEX_ROW}" -i "${WIKI_PATH}/Home.md"
# Build manifest
MANIFEST_FILE="${WIKI_PATH}/manifests/${IMAGE_SHORT_NAME}-${GIT_SHA_TAG}.md"
mkdir -p $(dirname "$MANIFEST_FILE")
cat << EOF > "$MANIFEST_FILE"
* Build datetime: ${BUILD_TIMESTAMP}
* Docker image: ${DOCKER_REPO}:${GIT_SHA_TAG}
* Docker image size: $(docker images ${IMAGE_NAME} --format "{{.Size}}")
* Git commit SHA: [${GITHUB_SHA}](https://github.com/jupyter/docker-stacks/commit/${GITHUB_SHA})
* Git commit message:
\`\`\`
${COMMIT_MSG}
\`\`\`
## Python Packages
\`\`\`
$(docker run --rm ${IMAGE_NAME} python --version)
\`\`\`
\`\`\`
$(docker run --rm ${IMAGE_NAME} conda info)
\`\`\`
\`\`\`
$(docker run --rm ${IMAGE_NAME} conda list)
\`\`\`
## Apt Packages
\`\`\`
$(docker run --rm ${IMAGE_NAME} apt list --installed)
\`\`\`
EOF
\ No newline at end of file
INDEX_ROW="|\`${BUILD_TIMESTAMP}\`|\`jupyter/${IMAGE_SHORT_NAME}:${GIT_SHA_TAG}\`|[Git diff](https://github.com/jupyter/docker-stacks/commit/${SOURCE_COMMIT})<br />[Dockerfile](https://github.com/jupyter/docker-stacks/blob/${SOURCE_COMMIT}/${IMAGE_SHORT_NAME}/Dockerfile)<br />[Build manifest](./${IMAGE_SHORT_NAME}-${GIT_SHA_TAG})|"
sed "/|-|/a ${INDEX_ROW}" -i "$INDEX_FILE"
\ No newline at end of file
cat << EOF > "$MANIFEST_FILE"
* Build datetime: ${BUILD_TIMESTAMP}
* DockerHub build code: ${BUILD_CODE}
* Docker image: ${DOCKER_REPO}:${GIT_SHA_TAG}
* Docker image size: $(docker images ${IMAGE_NAME} --format "{{.Size}}")
* Git commit SHA: [${SOURCE_COMMIT}](https://github.com/jupyter/docker-stacks/commit/${SOURCE_COMMIT})
* Git commit message:
\`\`\`
${COMMIT_MSG}
\`\`\`
## Python Packages
\`\`\`
$(docker run --rm ${IMAGE_NAME} python --version)
\`\`\`
\`\`\`
$(docker run --rm ${IMAGE_NAME} conda info)
\`\`\`
\`\`\`
$(docker run --rm ${IMAGE_NAME} conda list)
\`\`\`
## Apt Packages
\`\`\`
$(docker run --rm ${IMAGE_NAME} apt list --installed)
\`\`\`
EOF
\ No newline at end of file
#!/bin/bash
set -e
# Tag the latest build with the short git sha. Push the tag in addition
# to the "latest" tag already pushed.
GIT_SHA_TAG=${SOURCE_COMMIT:0:12}
docker tag $IMAGE_NAME $DOCKER_REPO:$GIT_SHA_TAG
docker push $DOCKER_REPO:$GIT_SHA_TAG
# Create a working directory.
BUILD_TIMESTAMP=$(date -u +%FT%TZ)
WORKDIR=$(mktemp -d)
GIT_URI="git@github.com:jupyter/docker-stacks.wiki.git"
GIT_SANDBOX="${WORKDIR}/docker-stacks.wiki"
IMAGE_SHORT_NAME=$(basename $DOCKER_REPO)
MANIFEST_FILE="${GIT_SANDBOX}/manifests/${IMAGE_SHORT_NAME}-${GIT_SHA_TAG}.md"
INDEX_FILE="${GIT_SANDBOX}/Home.md"
# Configure git so it can push back to GitHub.
eval $(ssh-agent -s)
ssh-add <(base64 -d <(echo "$DEPLOY_KEY"))
ssh-add -l
git config --global user.email "jupyter@googlegroups.com"
git config --global user.name "Jupyter Docker Stacks"
# Glone the GitHub project wiki.
pushd "$WORKDIR"
git clone "$GIT_URI"
popd
# Render the build manifest template.
mkdir -p $(dirname "$MANIFEST_FILE")
source hooks/manifest.tmpl
source hooks/index.tmpl
# Push the wiki update back to GitHub.
pushd "$GIT_SANDBOX"
git add .
git commit -m "DOC: Build ${MANIFEST_FILE}"
git push -u origin master
popd
# Shutdown the ssh agent for good measure.
ssh-agent -k
# Invoke all downstream build triggers.
set +e
for url in $(echo $NEXT_BUILD_TRIGGERS | sed "s/,/ /g")
do
curl -X POST $url
done
\ No newline at end of file
#!/bin/bash
set -e
# Apply tags
GIT_SHA_TAG=${GITHUB_SHA:0:12}
docker tag $IMAGE_NAME "$DOCKER_REPO:$GIT_SHA_TAG"
# Update index
INDEX_ROW="|\`${BUILD_TIMESTAMP}\`|\`jupyter/${IMAGE_SHORT_NAME}:${GIT_SHA_TAG}\`|[Git diff](https://github.com/jupyter/docker-stacks/commit/${GITHUB_SHA})<br />[Dockerfile](https://github.com/jupyter/docker-stacks/blob/${GITHUB_SHA}/${IMAGE_SHORT_NAME}/Dockerfile)<br />[Build manifest](./${IMAGE_SHORT_NAME}-${GIT_SHA_TAG})|"
sed "/|-|/a ${INDEX_ROW}" -i "${WIKI_PATH}/Home.md"
# Build manifest
MANIFEST_FILE="${WIKI_PATH}/manifests/${IMAGE_SHORT_NAME}-${GIT_SHA_TAG}.md"
mkdir -p $(dirname "$MANIFEST_FILE")
cat << EOF > "$MANIFEST_FILE"
* Build datetime: ${BUILD_TIMESTAMP}
* Docker image: ${DOCKER_REPO}:${GIT_SHA_TAG}
* Docker image size: $(docker images ${IMAGE_NAME} --format "{{.Size}}")
* Git commit SHA: [${GITHUB_SHA}](https://github.com/jupyter/docker-stacks/commit/${GITHUB_SHA})
* Git commit message:
\`\`\`
${COMMIT_MSG}
\`\`\`
## Python Packages
\`\`\`
$(docker run --rm ${IMAGE_NAME} python --version)
\`\`\`
\`\`\`
$(docker run --rm ${IMAGE_NAME} conda info)
\`\`\`
\`\`\`
$(docker run --rm ${IMAGE_NAME} conda list)
\`\`\`
## Apt Packages
\`\`\`
$(docker run --rm ${IMAGE_NAME} apt list --installed)
\`\`\`
EOF
\ No newline at end of file
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment