From 063534d18b5d9b0b1a9049a0d0a07278dcc69add Mon Sep 17 00:00:00 2001 From: Tom Gallacher Date: Wed, 2 Nov 2016 16:13:15 +0000 Subject: [PATCH] Adding nginx as a service to load balance `ui` --- cloudapi-graphql/src/credentials.js | 13 ++-- docker-compose.yml | 24 ++++++- local-compose.yml | 12 ++++ nginx/Dockerfile | 5 ++ nginx/Makefile | 33 +++++++++ nginx/etc/containerpilot.json | 78 +++++++++++++++++++++ nginx/etc/nginx/nginx.conf.ctmpl | 105 ++++++++++++++++++++++++++++ 7 files changed, 265 insertions(+), 5 deletions(-) create mode 100644 nginx/Dockerfile create mode 100644 nginx/Makefile create mode 100644 nginx/etc/containerpilot.json create mode 100644 nginx/etc/nginx/nginx.conf.ctmpl diff --git a/cloudapi-graphql/src/credentials.js b/cloudapi-graphql/src/credentials.js index 2a2cc6bb..f73ece0e 100644 --- a/cloudapi-graphql/src/credentials.js +++ b/cloudapi-graphql/src/credentials.js @@ -1,12 +1,17 @@ const json = (() => { try { - return require('../credentials.json'); - } catch (err) { require('dotenv').config({ - path: '../.env' + path: '../.env', + silent: true }); - return {}; + } catch (err) { + try { + return require('../credentials.json'); + } catch (err) { + return {}; + } } + return {}; })(); module.exports = { diff --git a/docker-compose.yml b/docker-compose.yml index 80762666..71c0d361 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -5,6 +5,7 @@ consul: image: progrium/consul:latest labels: - triton.cns.services=consul + - com.docker.swarm.affinities=["container!=~*"] restart: always mem_limit: 128m expose: @@ -26,8 +27,10 @@ cloudapi: mem_limit: 128m labels: - triton.cns.services=cloudapi + - com.docker.swarm.affinities=["container!=~*cloudapi*"] env_file: .env environment: + - CONSUL_AGENT=1 - PORT=3000 ports: - 3000:3000 @@ -39,8 +42,10 @@ frontend: mem_limit: 128m labels: - triton.cns.services=frontend + - com.docker.swarm.affinities=["container!=~*frontend*"] env_file: .env environment: + - CONSUL_AGENT=1 - PORT=8000 ports: - 8000:8000 @@ -52,8 +57,25 @@ ui: mem_limit: 128m labels: - triton.cns.services=ui + - com.docker.swarm.affinities=["container!=~*ui*"] env_file: .env environment: + - CONSUL_AGENT=1 - PORT=8080 +############################################################################# +# Nginx as a load-balancing tier and reverse proxy +############################################################################# +nginx: + image: quay.io/yldio/joyent-portal-nginx + restart: always + mem_limit: 128m ports: - - 8080:8080 + - 80 + - 443 + - 9090 + env_file: .env + environment: + - CONSUL_AGENT=1 + labels: + - triton.cns.services=nginx + - com.docker.swarm.affinities=["container!=~*nginx*"] diff --git a/local-compose.yml b/local-compose.yml index 673b83c1..d3550172 100644 --- a/local-compose.yml +++ b/local-compose.yml @@ -37,3 +37,15 @@ ui: - PORT=8080 - ROOT_URL=http://localhost:8080 - CONSUL=consul +nginx: + extends: + file: docker-compose.yml + service: nginx + build: ./nginx + restart: never + environment: + - CONSUL=consul + links: + - consul:consul + ports: + - 80 diff --git a/nginx/Dockerfile b/nginx/Dockerfile new file mode 100644 index 00000000..c3efdc94 --- /dev/null +++ b/nginx/Dockerfile @@ -0,0 +1,5 @@ +# a minimal Nginx container including containerpilot and a simple virtulhost config +FROM autopilotpattern/nginx:1-r6.1.0 + +# Add our configuration files +COPY etc /etc diff --git a/nginx/Makefile b/nginx/Makefile new file mode 100644 index 00000000..6505b6ea --- /dev/null +++ b/nginx/Makefile @@ -0,0 +1,33 @@ +NAME := $(lastword $(subst /, ,$(CURDIR))) + +.PHONY: test +test: + +.PHONY: test-ci +test-ci: + +.PHONY: install +install: + +.PHONY: start +start: + +.PHONY: install-production +install-production: + +.PHONY: build +build: + docker build -t quay.io/yldio/joyent-portal-$(NAME) . + +.PHONY: push +push: + docker push quay.io/yldio/joyent-portal-$(NAME) + +.PHONY: clean +clean: + +.PHONY: lint +lint: + +.PHONY: lint-ci +lint-ci: diff --git a/nginx/etc/containerpilot.json b/nginx/etc/containerpilot.json new file mode 100644 index 00000000..20392272 --- /dev/null +++ b/nginx/etc/containerpilot.json @@ -0,0 +1,78 @@ +{ + "consul": "{{ if .CONSUL_AGENT }}localhost{{ else }}{{ .CONSUL }}{{ end }}:8500", + "preStart": "/usr/local/bin/reload.sh preStart", + "logging": {"level": "DEBUG"}, + "services": [ + { + "name": "nginx", + "port": 80, + "health": "/usr/bin/curl --fail --silent --show-error --output /dev/null http://localhost/nginx-health", + "poll": 10, + "ttl": 25, + "interfaces": ["eth0"] + }, + { + "name": "nginx-public", + "port": 80, + "health": "/usr/bin/curl --fail --silent --show-error --output /dev/null http://localhost/nginx-health", + "poll": 10, + "ttl": 25, + "interfaces": ["eth1", "eth0"] + }{{ if .ACME_DOMAIN }}, + { + "name": "nginx-public-ssl", + "port": 443, + "health": "/usr/local/bin/acme init && /usr/bin/curl --insecure --fail --silent --show-error --output /dev/null --header \"HOST: {{ .ACME_DOMAIN }}\" https://localhost/nginx-health", + "poll": 10, + "ttl": 25, + "interfaces": ["eth1", "eth0"] + }{{ end }} + ], + "backends": [ + { + "name": "joyent-portal-ui", + "poll": 7, + "onChange": "/usr/local/bin/reload.sh" + } + ], + "coprocesses": [{{ if .CONSUL_AGENT }} + { + "command": ["/usr/local/bin/consul", "agent", + "-data-dir=/data", + "-config-dir=/config", + "-rejoin", + "-retry-join", "{{ .CONSUL }}", + "-retry-max", "10", + "-retry-interval", "10s"], + "restarts": "unlimited" + }{{ end }} + {{ if and .CONSUL_AGENT .ACME_DOMAIN }},{{ end }} + {{ if .ACME_DOMAIN }} + { + "command": ["/usr/local/bin/consul-template", + "-config", "/etc/acme/watch.hcl", + "-consul", "{{ if .CONSUL_AGENT }}localhost{{ else }}{{ .CONSUL }}{{ end }}:8500"], + "restarts": "unlimited" + }{{ end }}], + "telemetry": { + "port": 9090, + "sensors": [ + { + "name": "nginx_connections_unhandled_total", + "help": "Number of accepted connnections that were not handled", + "type": "gauge", + "poll": 5, + "check": ["/usr/local/bin/sensor.sh", "unhandled"] + }, + { + "name": "nginx_connections_load", + "help": "Ratio of active connections (less waiting) to the maximum worker connections", + "type": "gauge", + "poll": 5, + "check": ["/usr/local/bin/sensor.sh", "connections_load"] + } + ] + }, + "tasks": [ + ] +} diff --git a/nginx/etc/nginx/nginx.conf.ctmpl b/nginx/etc/nginx/nginx.conf.ctmpl new file mode 100644 index 00000000..af797b0a --- /dev/null +++ b/nginx/etc/nginx/nginx.conf.ctmpl @@ -0,0 +1,105 @@ +user nginx; +worker_processes 1; + +error_log /var/log/nginx/error.log warn; +pid /var/run/nginx.pid; + +events { + worker_connections 1024; +} + +http { + include /etc/nginx/mime.types; + default_type application/octet-stream; + + map $status $isError { + ~^2 0; + default 1; + } + + log_format main '$remote_addr - $remote_user [$time_local] "$request" ' + '$status $body_bytes_sent "$http_referer" ' + '"$http_user_agent" "$http_x_forwarded_for"'; + + access_log /var/log/nginx/access.log main; + sendfile on; + keepalive_timeout 65; + + # Environment variables that are provided to us at the time our Nginx + # config is generated from this template + {{ $acme_domain := env "ACME_DOMAIN" }} # Domain that we're requesting certs for if set + {{ $ssl_ready := env "SSL_READY" }} # true if certs have been written to disk + + {{ if service "joyent-portal-ui" }} + upstream joyent-portal-ui { + # write the address:port pairs for each healthy joyent-portal-ui node + {{range service "joyent-portal-ui"}} + server {{.Address}}:{{.Port}}; + {{end}} + least_conn; + }{{ end }} + + # If we're listening on https, define an http listener that redirects everything to https + {{ if eq $ssl_ready "true" }} + server { + server_name _; + listen 80; + + # Respond to health requests defined in containerpilot.json + location /nginx-health { + stub_status; + allow 127.0.0.1; + deny all; + # Don't log these requests unless they fail + access_log /var/log/nginx/access.log main if=$isError; + } + + location / { + return 301 https://$host$request_uri; + } + } + {{ end }} + + server { + server_name _; + # Listen on port 80 unless we have certificates installed, then listen on 443 + listen {{ if ne $ssl_ready "true" }}80{{ else }}443 ssl{{ end }}; + {{ if eq $ssl_ready "true" }} + ssl_certificate /var/www/ssl/fullchain.pem; + ssl_certificate_key /var/www/ssl/privkey.pem; + ssl_protocols TLSv1 TLSv1.1 TLSv1.2; + ssl_prefer_server_ciphers on; + ssl_ciphers 'ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-AES256-GCM-SHA384:DHE-RSA-AES128-GCM-SHA256:DHE-DSS-AES128-GCM-SHA256:kEDH+AESGCM:ECDHE-RSA-AES128-SHA256:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA:ECDHE-ECDSA-AES128-SHA:ECDHE-RSA-AES256-SHA384:ECDHE-ECDSA-AES256-SHA384:ECDHE-RSA-AES256-SHA:ECDHE-ECDSA-AES256-SHA:DHE-RSA-AES128-SHA256:DHE-RSA-AES128-SHA:DHE-DSS-AES128-SHA256:DHE-RSA-AES256-SHA256:DHE-DSS-AES256-SHA:DHE-RSA-AES256-SHA:AES128-GCM-SHA256:AES256-GCM-SHA384:AES128-SHA256:AES256-SHA256:AES128-SHA:AES256-SHA:AES:CAMELLIA:DES-CBC3-SHA:!aNULL:!eNULL:!EXPORT:!DES:!RC4:!MD5:!PSK:!aECDH:!EDH-DSS-DES-CBC3-SHA:!EDH-RSA-DES-CBC3-SHA:!KRB5-DES-CBC3-SHA'; + ssl_session_timeout 1d; + ssl_session_cache shared:SSL:50m; + ssl_stapling on; + ssl_stapling_verify on; + add_header Strict-Transport-Security max-age=15768000; + {{ end }} + + {{ if service "joyent-portal-ui" }} + location ^~ / { + proxy_pass http://joyent-portal-ui; + proxy_set_header Host $http_host; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_redirect off; + } + {{end}} + + # Respond to health requests defined in containerpilot.json + location /nginx-health { + stub_status; + allow 127.0.0.1; + deny all; + # Don't log these requests unless they fail + access_log /var/log/nginx/access.log main if=$isError; + } + + # Respond to ACME certificate request challenges + location /.well-known/acme-challenge { + alias /var/www/acme/challenge; + } + } +}