Merge pull request #80 from parente/make-deploy-example

make-deploy example: run stacks on docker-machines

examples/make-deploy/Dockerfile

+# Copyright (c) Jupyter Development Team.
+# Distributed under the terms of the Modified BSD License.
+
+# Pick your favorite docker-stacks image
+FROM jupyter/minimal-notebook:2d125a7161b5
+
+USER jovyan
+
+# Add permanent pip/conda installs, data files, other user libs here
+# e.g., RUN pip install jupyter_dashboards
+
+USER root
+
+# Add permanent apt-get installs and other root commands here
+# e.g., RUN apt-get install npm nodejs

examples/make-deploy/Makefile

+# Copyright (c) Jupyter Development Team.
+# Distributed under the terms of the Modified BSD License.
+
+.PHONY: help check image notebook
+
+IMAGE:=my-notebook
+
+# Common, extensible docker run command
+define RUN_NOTEBOOK
+@docker volume create --name $(WORK_VOLUME) > /dev/null
+-@docker rm -f $(NAME) 2> /dev/null
+@docker run -d -p $(PORT):8888 \
+		--name $(NAME) \
+		-v $(WORK_VOLUME):/home/jovyan/work \
+		$(DOCKER_ARGS) \
+		$(IMAGE) bash -c "$(PRE_CMD) chown jovyan /home/jovyan/work && start-notebook.sh $(ARGS)" > /dev/null
+@echo "DONE: Notebook '$(NAME)' listening on $$(docker-machine ip $$(docker-machine active)):$(PORT)"
+endef
+
+help:
+	@cat README.md
+
+check:
+	@which docker-machine > /dev/null || (echo "ERROR: docker-machine not found (brew install docker-machine)"; exit 1)
+	@which docker > /dev/null || (echo "ERROR: docker not found (brew install docker)"; exit 1)
+	@docker | grep volume > /dev/null || (echo "ERROR: docker 1.9.0+ required"; exit 1)
+
+image: DOCKER_ARGS?=
+image:
+	@docker build --rm $(DOCKER_ARGS) -t $(IMAGE) .
+
+notebook: PORT?=80
+notebook: NAME?=notebook
+notebook: WORK_VOLUME?=$(NAME)-data
+notebook: check
+	$(RUN_NOTEBOOK)
+
+# docker-machine drivers
+include virtualbox.makefile
+include softlayer.makefile
+
+# Preset notebook configurations
+include self-signed.makefile
+include letsencrypt.makefile
\ No newline at end of file

examples/make-deploy/README.md

+This folder contains a Makefile and a set of supporting files demonstrating how to run a docker-stack notebook container on a docker-machine controlled host.
+
+## Prerequisites
+
+* make 3.81+ 
+    * Ubuntu users: Be aware of [make 3.81 defect 483086](https://bugs.launchpad.net/ubuntu/+source/make-dfsg/+bug/483086) which exists in 14.04 LTS but is fixed in 15.04+
+* docker-machine 0.5.0+
+* docker 1.9.0+
+
+## Quickstart
+
+To show what's possible, here's how to run the `jupyter/minimal-notebook` on a brand new local virtualbox.
+
+```
+# create a new VM
+make virtualbox-vm NAME=dev
+# make the new VM the active docker machine
+eval $(docker-machine env dev)
+# pull a docker stack and build a local image from it
+make image
+# start a notebook server in a container
+make notebook
+```
+
+The last command will log the IP address and port to visit in your browser.
+
+## FAQ
+
+### Can I run multiple notebook containers on the same VM?
+
+Yes. Specify a unique name and port on the `make notebook` command.
+
+```
+make notebook NAME=my-notebook PORT=9000
+make notebook NAME=your-notebook PORT=9001
+```
+
+### Can multiple notebook containers share their notebook directory?
+
+Yes.
+
+```
+make notebook NAME=my-notebook PORT=9000 WORK_VOLUME=our-work
+make notebook NAME=your-notebook PORT=9001 WORK_VOLUME=our-work
+```
+
+### How do I run over HTTPS?
+
+Instead of `make notebook`, run `make self-signed-notebook PASSWORD=your_desired_password`. This target gives you a notebook wtih a self-signed certificate.
+
+### That self-signed certificate is a pain. Let's Encrypt?
+
+Yes. Please.
+
+```
+make letsencrypt FQDN=host.mydomain.com EMAIL=myemail@somewhere.com
+make letsencrypt-notebook
+```
+
+The first command creates a Docker volume named after the notebook container with a `-secrets` suffix. It then runs the `letsencrypt` client with a slew of options (one of which has you automatically agreeing to the Let's Encrypt Terms of Service, see the Makefile). The second command mounts the secrets volume and configures Jupyter to use the full-chain certificate and private key.
+
+Be aware: Let's Encrypt has a pretty [low rate limit per domain](https://community.letsencrypt.org/t/public-beta-rate-limits/4772/3) at the moment. Don't exhaust your requests playing around!
+
+Also, keep in mind Let's Encrypt certificates are short lived: 90 days at the moment. You'll need to manually setup a cron job to run the renewal steps at the moment. (You can reuse the first command above.)
+
+### My pip/conda/apt-get installs disappear every time I restart the container. Can I make them permanent?
+
+```
+# add your pip, conda, apt-get, etc. permanent features to the Dockerfile where
+# indicated by the comments in the Dockerfile
+vi Dockerfile
+make image
+make notebook
+```
+
+### How do I upgrade my Docker container?
+
+```
+make image DOCKER_ARGS=--pull
+make notebook
+```
+
+The first line pulls the latest version of the Docker image used in the local Dockerfile. Then it rebuilds the local Docker image containing any customizations you may have added to it. The second line kills your currently running notebook container, and starts a fresh one using the new image.
+
+### Can I run on another VM provider other than VirtualBox?
+
+Yes. As an example, there's a `softlayer.makefile` included in this repo as an example. You would use it like so:
+
+```
+make softlayer-vm NAME=myhost \
+    SOFTLAYER_DOMAIN=your_desired_domain \
+    SOFTLAYER_USER=your_user_id \
+    SOFTLAYER_API_KEY=your_api_key
+eval $(docker-machine env myhost)
+# optional, creates a real DNS entry for the VM using the machine name as the hostname
+make softlayer-dns SOFTLAYER_DOMAIN=your_desired_domain
+make image
+make notebook
+```
+
+If you'd like to add support for another docker-machine driver, use the `softlayer.makefile` as a template.
+
+### Where are my notebooks stored?
+
+`make notebook` creates a Docker volume named after the notebook container with a `-data` suffix.
+
+### Uh ... make?
+
+Yes, sorry Windows users. It got the job done for a simple example. We can certainly accept other deployment mechanism examples in the parent folder or in other repos.
+
+### Are there any other options?
+
+Yes indeed. `cat` the Makefiles and look at the target parameters.

examples/make-deploy/letsencrypt.makefile

+# Copyright (c) Jupyter Development Team.
+# Distributed under the terms of the Modified BSD License.
+
+letsencrypt: NAME?=notebook
+letsencrypt: SECRETS_VOLUME?=$(NAME)-secrets
+letsencrypt:
+	@test -n "$(FQDN)" || \
+		(echo "ERROR: FQDN not defined or blank"; exit 1)
+	@test -n "$(EMAIL)" || \
+		(echo "ERROR: EMAIL not defined or blank"; exit 1)
+	@docker volume create --name $(SECRETS_VOLUME) > /dev/null
+# Specifying an alternative cert path doesn't work with the --duplicate
+# setting which we want to use for renewal.
+	@docker run -it --rm -p 80:80 \
+		-v $(SECRETS_VOLUME):/etc/letsencrypt \
+		quay.io/letsencrypt/letsencrypt:latest \
+		certonly \
+		--standalone \
+		--standalone-supported-challenges http-01 \
+		--agree-tos \
+		--duplicate \
+		--domain '$(FQDN)' \
+		--email '$(EMAIL)'
+# The lets encrypt image has an entrypoint so we use the notebook image
+# instead which we know uses tini as the entry and can run arbitrary commands.
+# Here we need to set the permissions so nobody in the proxy container can read
+# the cert and key. Plus we want to symlink the certs into the root of the 
+# /etc/letsencrypt directory so that the FQDN doesn't have to be known later.
+	@docker run -it --rm \
+		-v $(SECRETS_VOLUME):/etc/letsencrypt \
+		$(NOTEBOOK_IMAGE) \
+		bash -c "ln -s /etc/letsencrypt/live/$(FQDN)/* /etc/letsencrypt/ && \
+			find /etc/letsencrypt -type d -exec chmod 755 {} +"
+
+letsencrypt-notebook: PORT?=443
+letsencrypt-notebook: NAME?=notebook
+letsencrypt-notebook: WORK_VOLUME?=$(NAME)-data
+letsencrypt-notebook: SECRETS_VOLUME?=$(NAME)-secrets
+letsencrypt-notebook: DOCKER_ARGS:=-e USE_HTTPS=yes \
+	-e PASSWORD=$(PASSWORD) \
+	-v $(SECRETS_VOLUME):/etc/letsencrypt
+letsencrypt-notebook: ARGS:=\
+	--NotebookApp.certfile=/etc/letsencrypt/fullchain.pem \
+	--NotebookApp.keyfile=/etc/letsencrypt/privkey.pem
+letsencrypt-notebook: check
+	@test -n "$(PASSWORD)" || \
+		(echo "ERROR: PASSWORD not defined or blank"; exit 1)
+	$(RUN_NOTEBOOK)

examples/make-deploy/self-signed.makefile

+# Copyright (c) Jupyter Development Team.
+# Distributed under the terms of the Modified BSD License.
+
+self-signed-notebook: PORT?=443
+self-signed-notebook: NAME?=notebook
+self-signed-notebook: WORK_VOLUME?=$(NAME)-data
+self-signed-notebook: DOCKER_ARGS:=-e USE_HTTPS=yes \
+	-e PASSWORD=$(PASSWORD)
+self-signed-notebook: check
+	@test -n "$(PASSWORD)" || \
+		(echo "ERROR: PASSWORD not defined or blank"; exit 1)
+	$(RUN_NOTEBOOK)

examples/make-deploy/softlayer.makefile

+# Copyright (c) Jupyter Development Team.
+# Distributed under the terms of the Modified BSD License.
+
+softlayer-vm: export SOFTLAYER_CPU?=4
+softlayer-vm: export SOFTLAYER_DISK_SIZE?=100
+softlayer-vm: export SOFTLAYER_MEMORY?=4096
+softlayer-vm: export SOFTLAYER_REGION?=wdc01
+softlayer-vm: check
+	@test -n "$(NAME)" || \
+		(echo "ERROR: NAME not defined (make help)"; exit 1)
+	@test -n "$(SOFTLAYER_API_KEY)" || \
+		(echo "ERROR: SOFTLAYER_API_KEY not defined (make help)"; exit 1)
+	@test -n "$(SOFTLAYER_USER)" || \
+		(echo "ERROR: SOFTLAYER_USER not defined (make help)"; exit 1)
+	@test -n "$(SOFTLAYER_DOMAIN)" || \
+		(echo "ERROR: SOFTLAYER_DOMAIN not defined (make help)"; exit 1)
+	@docker-machine create -d softlayer $(NAME)
+	@echo "DONE: Docker host '$(NAME)' up at $$(docker-machine ip $(NAME))"
+
+softlayer-dns: HOST_NAME:=$$(docker-machine active)
+softlayer-dns: IP:=$$(docker-machine ip $(HOST_NAME))
+softlayer-dns: check
+	@which slcli > /dev/null || (echo "softlayer cli not found (pip install softlayer)"; exit 1)
+	@test -n "$(SOFTLAYER_DOMAIN)" || \
+		(echo "ERROR: SOFTLAYER_DOMAIN not defined (make help)"; exit 1)
+	@slcli dns record-add $(SOFTLAYER_DOMAIN) $(HOST_NAME) A $(IP)

examples/make-deploy/virtualbox.makefile

+# Copyright (c) Jupyter Development Team.
+# Distributed under the terms of the Modified BSD License.
+
+virtualbox-vm: export VIRTUALBOX_CPU_COUNT?=4
+virtualbox-vm: export VIRTUALBOX_DISK_SIZE?=100000
+virtualbox-vm: export VIRTUALBOX_MEMORY_SIZE?=4096
+virtualbox-vm: check
+	@test -n "$(NAME)" || \
+		(echo "ERROR: NAME not defined (make help)"; exit 1)
+	@docker-machine create -d virtualbox $(NAME)