Merge pull request #508 from parente/406-integration-tests

pytest integration tests

.travis.yml

-language: generic
-
+language: python
+python:
+    - 3.6
 sudo: required
 services:
     - docker
-
+install:
+    - make test-reqs
 script:
     - make build-test-all

Makefile

 # Copyright (c) Jupyter Development Team.
 # Distributed under the terms of the Modified BSD License.
-
-.PHONY: build-all help environment-check release-all
+.PHONY: help test
 
 # Use bash for inline if-statements in test target
 SHELL:=bash
-
 OWNER:=jupyter
-# need to list these manually because there's a dependency tree
 ARCH:=$(shell uname -m)
 
+# Need to list the images in build dependency order
 ifeq ($(ARCH),ppc64le)
-ALL_STACKS:=base-notebook 
+ALL_STACKS:=base-notebook
 else
 ALL_STACKS:=base-notebook \
 	minimal-notebook \
@@ -25,10 +23,6 @@ endif
 
 ALL_IMAGES:=$(ALL_STACKS)
 
-GIT_MASTER_HEAD_SHA:=$(shell git rev-parse --short=12 --verify HEAD)
-
-RETRIES:=10
-
 help:
 # http://marmelab.com/blog/2016/02/29/auto-documented-makefile.html
 	@echo "jupyter/docker-stacks"
@@ -38,7 +32,7 @@ help:
 	@grep -E '^[a-zA-Z0-9_%/-]+:.*?## .*$$' $(MAKEFILE_LIST) | sort | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}'
 
 arch_patch/%: ## apply hardware architecture specific patches to the Dockerfile
-	if [ -e ./$(notdir $@)/Dockerfile.$(ARCH).patch ]; then \
+	@if [ -e ./$(notdir $@)/Dockerfile.$(ARCH).patch ]; then \
 		if [ -e ./$(notdir $@)/Dockerfile.orig ]; then \
                		cp -f ./$(notdir $@)/Dockerfile.orig ./$(notdir $@)/Dockerfile;\
 		else\
@@ -60,48 +54,11 @@ dev/%: PORT?=8888
 dev/%: ## run a foreground container for a stack
 	docker run -it --rm -p $(PORT):8888 $(DARGS) $(OWNER)/$(notdir $@) $(ARGS)
 
-environment-check:
-	test -e ~/.docker-stacks-builder
-
-push/%: ## push the latest and HEAD git SHA tags for a stack to Docker Hub
-	docker push $(OWNER)/$(notdir $@):latest
-	docker push $(OWNER)/$(notdir $@):$(GIT_MASTER_HEAD_SHA)
-
-push-all: $(ALL_IMAGES:%=push/%) ## push all stacks
-
-refresh/%: ## pull the latest image from Docker Hub for a stack
-# skip if error: a stack might not be on dockerhub yet
-	-docker pull $(OWNER)/$(notdir $@):latest
-
-refresh-all: $(ALL_IMAGES:%=refresh/%) ## refresh all stacks
-
-release-all: environment-check \
-	refresh-all \
-	build-test-all \
-	tag-all \
-	push-all
-release-all: ## build, test, tag, and push all stacks
-
-retry/%:
-	@for i in $$(seq 1 $(RETRIES)); do \
-		make $(notdir $@) ; \
-		if [[ $$? == 0 ]]; then exit 0; fi; \
-		echo "Sleeping for $$((i * 60))s before retry" ; \
-		sleep $$((i * 60)) ; \
-	done ; exit 1
-
-tag/%: ##tag the latest stack image with the HEAD git SHA
-	docker tag -f $(OWNER)/$(notdir $@):latest $(OWNER)/$(notdir $@):$(GIT_MASTER_HEAD_SHA)
-
-tag-all: $(ALL_IMAGES:%=tag/%) ## tag all stacks
+test-reqs: # install libraries required to run the integration tests
+	pip install -r requirements-test.txt
 
-test/%: ## run a stack container, check for jupyter server liveliness
-	@-docker rm -f iut
-	@docker run -d --name iut $(OWNER)/$(notdir $@)
-	@for i in $$(seq 0 9); do \
-		sleep $$i; \
-		docker exec iut bash -c 'wget http://localhost:8888 -O- | grep -i jupyter'; \
-		if [[ $$? == 0 ]]; then exit 0; fi; \
-	done ; exit 1
+test/%:
+	@TEST_IMAGE="$(OWNER)/$(notdir $@)" pytest test
 
-test-all: $(ALL_IMAGES:%=test/%) ## test all stacks
+test/base-notebook: ## test supported options in the base notebook
+	@TEST_IMAGE="$(OWNER)/$(notdir $@)" pytest test base-notebook/test
\ No newline at end of file

base-notebook/Dockerfile

@@ -77,9 +77,10 @@ RUN cd /tmp && \
 RUN conda install --quiet --yes \
     'notebook=5.2.*' \
     'jupyterhub=0.8.*' \
-    'jupyterlab=0.29.*' \
+    'jupyterlab=0.30.*' \
     && conda clean -tipsy && \
-    jupyter labextension install @jupyterlab/hub-extension@^0.6.0 && \
+    jupyter labextension install @jupyterlab/hub-extension@^0.7.0 && \
+    npm cache clean && \
     rm -rf $CONDA_DIR/share/jupyter/lab/staging && \
     fix-permissions $CONDA_DIR
 

base-notebook/start.sh

@@ -33,8 +33,8 @@ if [ $(id -u) == 0 ] ; then
         usermod -u $NB_UID $NB_USER
     fi
 
-    # Change GID of NB_USER to NB_GID if NB_GID is passed as a parameter
-    if [ "$NB_GID" ] ; then
+    # Change GID of NB_USER to NB_GID if it does not match
+    if [ "$NB_GID" != $(id -g $NB_USER) ] ; then
         echo "Set $NB_USER GID to: $NB_GID"
         groupmod -g $NB_GID -o $(id -g -n $NB_USER)
     fi

base-notebook/test/test_container_options.py

+# Copyright (c) Jupyter Development Team.
+# Distributed under the terms of the Modified BSD License.
+import time
+
+import pytest
+
+
+def test_cli_args(container, http_client):
+    """Container should respect notebook server command line args
+    (e.g., disabling token security)"""
+    container.run(
+        command=['start-notebook.sh', '--NotebookApp.token=""']
+    )
+    resp = http_client.get('http://localhost:8888')
+    resp.raise_for_status()
+    assert 'login_submit' not in resp.text
+
+
+@pytest.mark.filterwarnings('ignore:Unverified HTTPS request')
+def test_unsigned_ssl(container, http_client):
+    """Container should generate a self-signed SSL certificate
+    and notebook server should use it to enable HTTPS.
+    """
+    container.run(
+        environment=['GEN_CERT=yes']
+    )
+    # NOTE: The requests.Session backing the http_client fixture  does not retry
+    # properly while the server is booting up. An SSL handshake error seems to
+    # abort the retry logic. Forcing a long sleep for the moment until I have
+    # time to dig more.
+    time.sleep(5)
+    resp = http_client.get('https://localhost:8888', verify=False)
+    resp.raise_for_status()
+    assert 'login_submit' in resp.text
+
+
+def test_uid_change(container):
+    """Container should change the UID of the default user."""
+    c = container.run(
+        tty=True,
+        user='root',
+        environment=['NB_UID=1010'],
+        command=['start.sh', 'id && touch /opt/conda/test-file']
+    )
+    # usermod is slow so give it some time
+    c.wait(timeout=120)
+    assert 'uid=1010(jovyan)' in c.logs(stdout=True).decode('utf-8')
+
+
+def test_gid_change(container):
+    """Container should change the GID of the default user."""
+    c = container.run(
+        tty=True,
+        user='root',
+        environment=['NB_GID=110'],
+        command=['start.sh', 'id']
+    )
+    c.wait(timeout=10)
+    assert 'gid=110(users)' in c.logs(stdout=True).decode('utf-8')
+
+
+def test_sudo(container):
+    """Container should grant passwordless sudo to the default user."""
+    c = container.run(
+        tty=True,
+        user='root',
+        environment=['GRANT_SUDO=yes'],
+        command=['start.sh', 'sudo', 'id']
+    )
+    rv = c.wait(timeout=10)
+    assert rv == 0
+    assert 'uid=0(root)' in c.logs(stdout=True).decode('utf-8')
+
+def test_group_add(container, tmpdir):
+    """Container should run with the specified uid, gid, and secondary
+    group.
+    """
+    c = container.run(
+        user='1010:1010',
+        group_add=['users'],
+        command=['start.sh', 'id']
+    )
+    rv = c.wait(timeout=5)
+    assert rv == 0
+    assert 'uid=1010 gid=1010 groups=1010,100(users)' in c.logs(stdout=True).decode('utf-8')

conftest.py

+# Copyright (c) Jupyter Development Team.
+# Distributed under the terms of the Modified BSD License.
+import os
+
+import docker
+import pytest
+import requests
+
+from requests.packages.urllib3.util.retry import Retry
+from requests.adapters import HTTPAdapter
+
+
+@pytest.fixture(scope='session')
+def http_client():
+    """Requests session with retries and backoff."""
+    s = requests.Session()
+    retries = Retry(total=5, backoff_factor=1)
+    s.mount('http://', HTTPAdapter(max_retries=retries))
+    s.mount('https://', HTTPAdapter(max_retries=retries))
+    return s
+
+
+@pytest.fixture(scope='session')
+def docker_client():
+    """Docker client configured based on the host environment"""
+    return docker.from_env()
+
+
+@pytest.fixture(scope='session')
+def image_name():
+    """Image name to test"""
+    return os.getenv('TEST_IMAGE', 'jupyter/base-notebook')
+
+
+class TrackedContainer(object):
+    """Wrapper that collects docker container configuration and delays
+    container creation/execution.
+
+    Parameters
+    ----------
+    docker_client: docker.DockerClient
+        Docker client instance
+    image_name: str
+        Name of the docker image to launch
+    **kwargs: dict, optional
+        Default keyword arguments to pass to docker.DockerClient.containers.run
+    """
+    def __init__(self, docker_client, image_name, **kwargs):
+        self.container = None
+        self.docker_client = docker_client
+        self.image_name = image_name
+        self.kwargs = kwargs
+
+    def run(self, **kwargs):
+        """Runs a docker container using the preconfigured image name
+        and a mix of the preconfigured container options and those passed
+        to this method.
+
+        Keeps track of the docker.Container instance spawned to kill it
+        later.
+
+        Parameters
+        ----------
+        **kwargs: dict, optional
+            Keyword arguments to pass to docker.DockerClient.containers.run
+            extending and/or overriding key/value pairs passed to the constructor
+
+        Returns
+        -------
+        docker.Container
+        """
+        all_kwargs = {}
+        all_kwargs.update(self.kwargs)
+        all_kwargs.update(kwargs)
+        self.container = self.docker_client.containers.run(self.image_name, **all_kwargs)
+        return self.container
+
+    def remove(self):
+        """Kills and removes the tracked docker container."""
+        if self.container:
+            self.container.remove(force=True)
+
+
+@pytest.fixture(scope='function')
+def container(docker_client, image_name):
+    """Notebook container with initial configuration appropriate for testing
+    (e.g., HTTP port exposed to the host for HTTP calls).
+
+    Yields the container instance and kills it when the caller is done with it.
+    """
+    container = TrackedContainer(
+        docker_client,
+        image_name,
+        detach=True,
+        ports={
+            '8888/tcp': 8888
+        }
+    )
+    yield container
+    container.remove()

requirements-test.txt

+docker
+pytest
+requests
\ No newline at end of file

scipy-notebook/Dockerfile

@@ -48,6 +48,8 @@ RUN conda install --quiet --yes \
     jupyter nbextension enable --py widgetsnbextension --sys-prefix && \
     # Also activate ipywidgets extension for JupyterLab
     jupyter labextension install @jupyter-widgets/jupyterlab-manager && \
+    npm cache clean && \
+    rm -rf $CONDA_DIR/share/jupyter/lab/staging && \
     fix-permissions $CONDA_DIR
 
 # Install facets which does not have a pip or conda package at the moment

test/test_notebook.py

+# Copyright (c) Jupyter Development Team.
+# Distributed under the terms of the Modified BSD License.
+
+def test_secured_server(container, http_client):
+    """Notebook server should eventually request user login."""
+    container.run()
+    resp = http_client.get('http://localhost:8888')
+    resp.raise_for_status()
+    assert 'login_submit' in resp.text