Adding nginx as a service to load balance ui

cloudapi-graphql/src/credentials.js

@@ -1,12 +1,17 @@
 const json = (() => {
+  try {
+    require('dotenv').config({
+      path: '../.env',
+      silent: true
+    });
+  } catch (err) {
     try {
       return require('../credentials.json');
     } catch (err) {
-    require('dotenv').config({
-      path: '../.env'
-    });
       return {};
     }
+  }
+  return {};
 })();
 
 module.exports = {

docker-compose.yml

@@ -5,6 +5,7 @@ consul:
   image: progrium/consul:latest
   labels:
     - triton.cns.services=consul
+    - com.docker.swarm.affinities=["container!=~*"]
   restart: always
   mem_limit: 128m
   expose:
@@ -26,8 +27,10 @@ cloudapi:
   mem_limit: 128m
   labels:
     - triton.cns.services=cloudapi
+    - com.docker.swarm.affinities=["container!=~*cloudapi*"]
   env_file: .env
   environment:
+    - CONSUL_AGENT=1
     - PORT=3000
   ports:
     - 3000:3000
@@ -39,8 +42,10 @@ frontend:
   mem_limit: 128m
   labels:
     - triton.cns.services=frontend
+    - com.docker.swarm.affinities=["container!=~*frontend*"]
   env_file: .env
   environment:
+    - CONSUL_AGENT=1
     - PORT=8000
   ports:
     - 8000:8000
@@ -52,8 +57,25 @@ ui:
   mem_limit: 128m
   labels:
     - triton.cns.services=ui
+    - com.docker.swarm.affinities=["container!=~*ui*"]
   env_file: .env
   environment:
+    - CONSUL_AGENT=1
     - PORT=8080
+#############################################################################
+#  Nginx as a load-balancing tier and reverse proxy
+#############################################################################
+nginx:
+  image: quay.io/yldio/joyent-portal-nginx
+  restart: always
+  mem_limit: 128m
   ports:
-    - 8080:8080
+    - 80
+    - 443
+    - 9090
+  env_file: .env
+  environment:
+    - CONSUL_AGENT=1
+  labels:
+    - triton.cns.services=nginx
+    - com.docker.swarm.affinities=["container!=~*nginx*"]

local-compose.yml

@@ -37,3 +37,15 @@ ui:
     - PORT=8080
     - ROOT_URL=http://localhost:8080
     - CONSUL=consul
+nginx:
+  extends:
+    file: docker-compose.yml
+    service: nginx
+  build: ./nginx
+  restart: never
+  environment:
+    - CONSUL=consul
+  links:
+    - consul:consul
+  ports:
+    - 80

nginx/Dockerfile

@@ -0,0 +1,5 @@
+# a minimal Nginx container including containerpilot and a simple virtulhost config
+FROM autopilotpattern/nginx:1-r6.1.0
+
+# Add our configuration files
+COPY etc /etc

nginx/Makefile

@@ -0,0 +1,33 @@
+NAME := $(lastword $(subst /, ,$(CURDIR)))
+
+.PHONY: test
+test:
+
+.PHONY: test-ci
+test-ci:
+
+.PHONY: install
+install: 
+
+.PHONY: start
+start:
+
+.PHONY: install-production
+install-production: 
+
+.PHONY: build
+build:
+	docker build -t quay.io/yldio/joyent-portal-$(NAME) .
+
+.PHONY: push
+push:
+	docker push quay.io/yldio/joyent-portal-$(NAME)
+
+.PHONY: clean
+clean:
+
+.PHONY: lint
+lint:
+
+.PHONY: lint-ci
+lint-ci:

nginx/etc/containerpilot.json

@@ -0,0 +1,78 @@
+{
+  "consul": "{{ if .CONSUL_AGENT }}localhost{{ else }}{{ .CONSUL }}{{ end }}:8500",
+  "preStart": "/usr/local/bin/reload.sh preStart",
+  "logging": {"level": "DEBUG"},
+  "services": [
+    {
+      "name": "nginx",
+      "port": 80,
+      "health": "/usr/bin/curl --fail --silent --show-error --output /dev/null http://localhost/nginx-health",
+      "poll": 10,
+      "ttl": 25,
+      "interfaces": ["eth0"]
+    },
+    {
+      "name": "nginx-public",
+      "port": 80,
+      "health": "/usr/bin/curl --fail --silent --show-error --output /dev/null http://localhost/nginx-health",
+      "poll": 10,
+      "ttl": 25,
+      "interfaces": ["eth1", "eth0"]
+    }{{ if .ACME_DOMAIN }},
+    {
+      "name": "nginx-public-ssl",
+      "port": 443,
+      "health": "/usr/local/bin/acme init && /usr/bin/curl --insecure --fail --silent --show-error --output /dev/null --header \"HOST: {{ .ACME_DOMAIN }}\" https://localhost/nginx-health",
+      "poll": 10,
+      "ttl": 25,
+      "interfaces": ["eth1", "eth0"]
+    }{{ end }}
+  ],
+  "backends": [
+    {
+      "name": "joyent-portal-ui",
+      "poll": 7,
+      "onChange": "/usr/local/bin/reload.sh"
+    }
+  ],
+  "coprocesses": [{{ if .CONSUL_AGENT }}
+    {
+      "command": ["/usr/local/bin/consul", "agent",
+                  "-data-dir=/data",
+                  "-config-dir=/config",
+                  "-rejoin",
+                  "-retry-join", "{{ .CONSUL }}",
+                  "-retry-max", "10",
+                  "-retry-interval", "10s"],
+      "restarts": "unlimited"
+    }{{ end }}
+    {{ if and .CONSUL_AGENT .ACME_DOMAIN }},{{ end }}
+    {{ if .ACME_DOMAIN }}
+    {
+      "command": ["/usr/local/bin/consul-template",
+                  "-config", "/etc/acme/watch.hcl",
+                  "-consul", "{{ if .CONSUL_AGENT }}localhost{{ else }}{{ .CONSUL }}{{ end }}:8500"],
+      "restarts": "unlimited"
+    }{{ end }}],
+  "telemetry": {
+    "port": 9090,
+    "sensors": [
+      {
+        "name": "nginx_connections_unhandled_total",
+        "help": "Number of accepted connnections that were not handled",
+        "type": "gauge",
+        "poll": 5,
+        "check": ["/usr/local/bin/sensor.sh", "unhandled"]
+      },
+      {
+        "name": "nginx_connections_load",
+        "help": "Ratio of active connections (less waiting) to the maximum worker connections",
+        "type": "gauge",
+        "poll": 5,
+        "check": ["/usr/local/bin/sensor.sh", "connections_load"]
+      }
+    ]
+  },
+  "tasks": [
+  ]
+}

nginx/etc/nginx/nginx.conf.ctmpl

@@ -0,0 +1,105 @@
+user  nginx;
+worker_processes  1;
+
+error_log  /var/log/nginx/error.log warn;
+pid        /var/run/nginx.pid;
+
+events {
+    worker_connections  1024;
+}
+
+http {
+    include       /etc/nginx/mime.types;
+    default_type  application/octet-stream;
+
+    map $status $isError {
+        ~^2 0;
+        default 1;
+    }
+
+    log_format  main  '$remote_addr - $remote_user [$time_local] "$request" '
+                      '$status $body_bytes_sent "$http_referer" '
+                      '"$http_user_agent" "$http_x_forwarded_for"';
+
+    access_log         /var/log/nginx/access.log  main;
+    sendfile           on;
+    keepalive_timeout  65;
+
+    # Environment variables that are provided to us at the time our Nginx 
+    # config is generated from this template
+    {{ $acme_domain := env "ACME_DOMAIN" }} # Domain that we're requesting certs for if set
+    {{ $ssl_ready := env "SSL_READY" }}     # true if certs have been written to disk
+
+    {{ if service "joyent-portal-ui" }}
+    upstream joyent-portal-ui {
+        # write the address:port pairs for each healthy joyent-portal-ui node
+        {{range service "joyent-portal-ui"}}
+        server {{.Address}}:{{.Port}};
+        {{end}}
+        least_conn;
+    }{{ end }}
+
+    # If we're listening on https, define an http listener that redirects everything to https
+    {{ if eq $ssl_ready "true" }}
+    server { 
+        server_name _;
+        listen      80;
+
+        # Respond to health requests defined in containerpilot.json
+        location /nginx-health {
+            stub_status;
+            allow 127.0.0.1;
+            deny all;
+            # Don't log these requests unless they fail
+            access_log /var/log/nginx/access.log  main if=$isError;
+        }
+
+        location / {
+            return 301 https://$host$request_uri;
+        }
+    }
+    {{ end }}
+
+    server {
+        server_name _;
+        # Listen on port 80 unless we have certificates installed, then listen on 443
+        listen {{ if ne $ssl_ready "true" }}80{{ else }}443 ssl{{ end }};
+        {{ if eq $ssl_ready "true" }}
+        ssl_certificate /var/www/ssl/fullchain.pem;
+        ssl_certificate_key /var/www/ssl/privkey.pem;
+        ssl_protocols TLSv1 TLSv1.1 TLSv1.2;
+        ssl_prefer_server_ciphers on;
+        ssl_ciphers 'ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-AES256-GCM-SHA384:DHE-RSA-AES128-GCM-SHA256:DHE-DSS-AES128-GCM-SHA256:kEDH+AESGCM:ECDHE-RSA-AES128-SHA256:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA:ECDHE-ECDSA-AES128-SHA:ECDHE-RSA-AES256-SHA384:ECDHE-ECDSA-AES256-SHA384:ECDHE-RSA-AES256-SHA:ECDHE-ECDSA-AES256-SHA:DHE-RSA-AES128-SHA256:DHE-RSA-AES128-SHA:DHE-DSS-AES128-SHA256:DHE-RSA-AES256-SHA256:DHE-DSS-AES256-SHA:DHE-RSA-AES256-SHA:AES128-GCM-SHA256:AES256-GCM-SHA384:AES128-SHA256:AES256-SHA256:AES128-SHA:AES256-SHA:AES:CAMELLIA:DES-CBC3-SHA:!aNULL:!eNULL:!EXPORT:!DES:!RC4:!MD5:!PSK:!aECDH:!EDH-DSS-DES-CBC3-SHA:!EDH-RSA-DES-CBC3-SHA:!KRB5-DES-CBC3-SHA';
+        ssl_session_timeout 1d;
+        ssl_session_cache shared:SSL:50m;
+        ssl_stapling on;
+        ssl_stapling_verify on;
+        add_header Strict-Transport-Security max-age=15768000;
+        {{ end }}
+
+        {{ if service "joyent-portal-ui" }}
+        location ^~ / {
+            proxy_pass http://joyent-portal-ui;
+            proxy_set_header Host $http_host;
+            proxy_set_header X-Forwarded-Proto $scheme;
+            proxy_set_header X-Real-IP $remote_addr;
+            proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+            proxy_redirect off;
+        }
+        {{end}}
+
+        # Respond to health requests defined in containerpilot.json
+        location /nginx-health {
+            stub_status;
+            allow 127.0.0.1;
+            deny all;
+            # Don't log these requests unless they fail
+            access_log /var/log/nginx/access.log  main if=$isError;
+        }
+
+        # Respond to ACME certificate request challenges
+        location /.well-known/acme-challenge {
+            alias /var/www/acme/challenge;
+        }
+    }
+}