From 1b5a2bfa9ae4817f758565ceae9e624f15ce93db Mon Sep 17 00:00:00 2001 From: geek Date: Thu, 14 Sep 2017 19:08:35 -0500 Subject: [PATCH] bug: fix bootstrap process --- README.md | 33 +++++---- triton-compose.yml => dev-compose.yml | 66 ++++++++--------- docker-compose.yml | 72 +++++++++---------- docker/api/bootstrap-data.js | 18 ++--- docker/api/etc/containerpilot.json5 | 37 ++++++---- docker/api/package.json | 2 +- local-compose.yml | 16 +++-- packages/portal-api/lib/data/index.js | 2 +- packages/portal-api/lib/index.js | 6 +- .../portal-api/lib/watch/container-pilot.js | 17 ++--- packages/portal-api/package.json | 4 +- 11 files changed, 137 insertions(+), 136 deletions(-) rename triton-compose.yml => dev-compose.yml (70%) diff --git a/README.md b/README.md index cfe934e2..2a49ae61 100644 --- a/README.md +++ b/README.md @@ -51,32 +51,35 @@ $ ./setup.sh ~/path/to/TRITON_PRIVATE_KEY keys-test.com/ca.crt keys-test.com/ser ## Usage -You have 3 options for where to run CoPilot. You can either run it using the published docker images locally, or on Triton. The last option is to build the docker images and run docker containers from these locally built images. +You have 3 options for where to run CoPilot. You can either run it using the published docker images on Triton or locally. The last option is to build the docker images and run docker containers from these locally built images. + + +### Deploy and run CoPilot on Triton + +Optionally use [_triton-docker_](https://github.com/joyent/triton-docker-cli) +```sh +$ triton-compose up -d +``` + +Or use `docker-compose` with preconfigured Triton environment variables +```sh +$ docker-compose up -d +``` + ### Start CoPilot using published docker images locally ```sh -$ docker-compose up -d +$ docker-compose -f local-compose.yml up -d ``` Navigate to [https://localhost]() to load the dashboard. -### Deploy and run CoPilot on Triton +### Build and run CoPilot locally for development ```sh -$ docker-compose -f triton-compose.yml up -d -``` - -Optionally use [_triton-docker_](https://github.com/joyent/triton-docker-cli) -```sh -$ triton-compose -f triton-compose.yml up -d -``` - -### Build and run CoPilot locally - -```sh -$ docker-compose -f local-compose.yml up -d +$ docker-compose -f dev-compose.yml up -d ``` ## Contribute diff --git a/triton-compose.yml b/dev-compose.yml similarity index 70% rename from triton-compose.yml rename to dev-compose.yml index 7378546b..3e73bda3 100644 --- a/triton-compose.yml +++ b/dev-compose.yml @@ -16,8 +16,6 @@ consul: -ui-dir /ui restart: always mem_limit: 128m - labels: - - triton.cns.services=copilot-consul ports: - 8500:8500 @@ -33,82 +31,80 @@ prometheus: mem_limit: 1g ports: - 9090:9090 - labels: - # This label sets the CNS name, Triton's automatic DNS - # Learn more at https://docs.joyent.com/public-cloud/network/cns - - triton.cns.services=prometheus - # This label selects the proper Joyent resource package - # https://www.joyent.com/blog/optimizing-docker-on-triton#ram-cpu-and-disk-resources-for-your-containers - - com.joyent.package=g4-highcpu-1G - env_file: - - _env + links: + - consul:consul environment: + - CONSUL=consul - CONSUL_AGENT=1 - - -rethinkdb: - image: autopilotpattern/rethinkdb:2.3.5r1 - restart: always - mem_limit: 1g - env_file: - - _env - environment: - - CONSUL_AGENT=1 - expose: - - 28015 - - 29015 + dns: + - 127.0.0.1 ############################################################################# # FRONTEND ############################################################################# frontend: - image: joyent/copilot-frontend:1.3.4 + build: docker/frontend mem_limit: 512m - labels: - - triton.cns.services=copilot + links: + - consul:consul env_file: - _env environment: + - CONSUL=consul - PORT=443 ports: - "80:80" - "443:443" - restart: always + dns: + - 127.0.0.1 ############################################################################# # BACKEND ############################################################################# api: - image: joyent/copilot-api:1.8.8 - mem_limit: 1g - expose: - - 3000 + build: docker/api + mem_limit: 512m + links: + - consul:consul env_file: - _env environment: + - CONSUL=consul - PORT=3000 - restart: always + expose: + - 3000 # Docker-compose wrapper # Create _env file from running ./setup.sh compose-api: - image: joyent/copilot-compose:1.1.0 + build: docker/compose-api + links: + - consul:consul expose: - 4242 env_file: - _env + environment: + - CONSUL=consul restart: always rethinkdb: - image: autopilotpattern/rethinkdb:2.3.5r2 + image: autopilotpattern/rethinkdb:2.3.5r1 restart: always mem_limit: 1g + links: + - consul:consul env_file: - _env environment: + - CONSUL=consul - CONSUL_AGENT=1 + ports: + - 8080:8080 expose: - 28015 - 29015 + dns: + - 127.0.0.1 diff --git a/docker-compose.yml b/docker-compose.yml index fdbe5cce..1e0a4898 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -16,10 +16,10 @@ consul: -ui-dir /ui restart: always mem_limit: 128m + labels: + - triton.cns.services=copilot-consul ports: - 8500:8500 - dns: - - 127.0.0.1 ############################################################################# # PROMETHEUS @@ -28,87 +28,87 @@ consul: # it is included here for demo purposes and is not required ############################################################################# prometheus: - image: autopilotpattern/prometheus:1.7.1-r20 + image: autopilotpattern/prometheus:1.7.1-r24 restart: always mem_limit: 1g ports: - 9090:9090 - links: - - consul:consul + labels: + # This label sets the CNS name, Triton's automatic DNS + # Learn more at https://docs.joyent.com/public-cloud/network/cns + - triton.cns.services=prometheus + # This label selects the proper Joyent resource package + # https://www.joyent.com/blog/optimizing-docker-on-triton#ram-cpu-and-disk-resources-for-your-containers + - com.joyent.package=g4-highcpu-1G + env_file: + - _env environment: - - CONSUL=consul - CONSUL_AGENT=1 - dns: - - 127.0.0.1 + + +rethinkdb: + image: autopilotpattern/rethinkdb:2.3.5r1 + restart: always + mem_limit: 1g + env_file: + - _env + environment: + - CONSUL_AGENT=1 + expose: + - 28015 + - 29015 ############################################################################# # FRONTEND ############################################################################# frontend: - image: joyent/copilot-frontend:1.0.0 + image: joyent/copilot-frontend:1.3.4 mem_limit: 512m - links: - - consul:consul + labels: + - triton.cns.services=copilot env_file: - _env environment: - - CONSUL=consul - PORT=443 ports: - "80:80" - "443:443" - dns: - - 127.0.0.1 + restart: always ############################################################################# # BACKEND ############################################################################# api: - image: joyent/copilot-api:1.8.8 - mem_limit: 512m - links: - - consul:consul - - rethinkdb:rethinkdb + image: joyent/copilot-api:1.8.9 + mem_limit: 1g + expose: + - 3000 env_file: - _env environment: - - CONSUL=consul - PORT=3000 - - RETHINK_HOST=rethinkdb - expose: - - 3000 + restart: always # Docker-compose wrapper # Create _env file from running ./setup.sh compose-api: - image: joyent/copilot-compose:1.0.0 - links: - - consul:consul + image: joyent/copilot-compose:1.1.0 expose: - 4242 env_file: - _env - environment: - - CONSUL=consul restart: always rethinkdb: - image: autopilotpattern/rethinkdb:2.3.5r1 + image: autopilotpattern/rethinkdb:2.3.5r2 restart: always mem_limit: 1g - links: - - consul:consul env_file: - _env environment: - - CONSUL=consul - CONSUL_AGENT=1 - ports: - - 8080:8080 expose: - 28015 - 29015 - dns: - - 127.0.0.1 diff --git a/docker/api/bootstrap-data.js b/docker/api/bootstrap-data.js index 4478472d..aa49a3a9 100644 --- a/docker/api/bootstrap-data.js +++ b/docker/api/bootstrap-data.js @@ -73,13 +73,9 @@ const bootstrap = function ({ docker, rethink }, cb) { return cb(err); } - data.getDatacenters((err, datacenters) => { - if (err) { - return cb(err); - } - + data.getPortal({}, (err, portal) => { // Don't continue since data is already bootstrapped - if (datacenters && datacenters.length) { + if (portal) { return cb(); } @@ -98,17 +94,13 @@ const bootstrap = function ({ docker, rethink }, cb) { return cb(err); } - cloudapi.getAccount((err, { - id, - firstName, - lastName, - email, - login - }) => { + cloudapi.getAccount({}, (err, body) => { if (err) { return cb(err); } + const { id, firstName, lastName, email, login } = body; + data.createUser({ tritonId: id, firstName, diff --git a/docker/api/etc/containerpilot.json5 b/docker/api/etc/containerpilot.json5 index c0f14123..1408053c 100644 --- a/docker/api/etc/containerpilot.json5 +++ b/docker/api/etc/containerpilot.json5 @@ -6,13 +6,34 @@ exec: '/bin/prestart.sh' }, { - name: 'bootstrap', - exec: 'node bootstrap-data.js', + name: 'consul-agent', + exec: ['/usr/local/bin/consul', 'agent', + '-data-dir=/data', + '-config-dir=/config', + '-log-level=err', + '-rejoin', + '-retry-join', '{{ .CONSUL | default "consul" }}', + '-retry-max', '20', + '-retry-interval', '5s'], + restarts: 'unlimited', + "health": { + "exec": "curl -so /dev/null http://localhost:8500", + "interval": 10, + "ttl": 25 + }, when: { source: 'setup-config', once: 'exitSuccess' } }, + { + name: 'bootstrap', + exec: 'node bootstrap-data.js', + when: { + source: 'consul-agent', + once: 'healthy' + } + }, { name: 'api', port: {{.PORT}}, @@ -28,18 +49,6 @@ }, restarts: 'unlimited' }, - { - name: 'consul-agent', - exec: ['/usr/local/bin/consul', 'agent', - '-data-dir=/data', - '-config-dir=/config', - '-log-level=err', - '-rejoin', - '-retry-join', '{{ .CONSUL | default "consul" }}', - '-retry-max', '20', - '-retry-interval', '5s'], - restarts: 'unlimited' - }, { name: 'sensor_memory_usage', exec: '/bin/sensors.sh memory', diff --git a/docker/api/package.json b/docker/api/package.json index 5c2a4c80..0ce8185b 100644 --- a/docker/api/package.json +++ b/docker/api/package.json @@ -21,7 +21,7 @@ "joi": "^10.6.0", "joyent-cp-gql-schema": "^1.7.0", "piloted": "^3.1.1", - "portal-api": "^1.8.8", + "portal-api": "^1.8.9", "toppsy": "^1.1.0", "triton": "^5.2.0" } diff --git a/local-compose.yml b/local-compose.yml index a4a8de74..8aa54ea2 100644 --- a/local-compose.yml +++ b/local-compose.yml @@ -18,6 +18,8 @@ consul: mem_limit: 128m ports: - 8500:8500 + dns: + - 127.0.0.1 ############################################################################# # PROMETHEUS @@ -26,7 +28,7 @@ consul: # it is included here for demo purposes and is not required ############################################################################# prometheus: - image: autopilotpattern/prometheus:1.7.1-r24 + image: autopilotpattern/prometheus:1.7.1-r20 restart: always mem_limit: 1g ports: @@ -36,13 +38,15 @@ prometheus: environment: - CONSUL=consul - CONSUL_AGENT=1 + dns: + - 127.0.0.1 ############################################################################# # FRONTEND ############################################################################# frontend: - build: docker/frontend + image: joyent/copilot-frontend:1.3.4 mem_limit: 512m links: - consul:consul @@ -54,28 +58,32 @@ frontend: ports: - "80:80" - "443:443" + dns: + - 127.0.0.1 ############################################################################# # BACKEND ############################################################################# api: - build: docker/api + image: joyent/copilot-api:1.8.9 mem_limit: 512m links: - consul:consul + - rethinkdb:rethinkdb env_file: - _env environment: - CONSUL=consul - PORT=3000 + - RETHINK_HOST=rethinkdb expose: - 3000 # Docker-compose wrapper # Create _env file from running ./setup.sh compose-api: - build: docker/compose-api + image: joyent/copilot-compose:1.0.0 links: - consul:consul expose: diff --git a/packages/portal-api/lib/data/index.js b/packages/portal-api/lib/data/index.js index e636f612..a099f2aa 100644 --- a/packages/portal-api/lib/data/index.js +++ b/packages/portal-api/lib/data/index.js @@ -263,7 +263,7 @@ class Data extends EventEmitter { // datacenters createDatacenter (datacenter, cb) { - this._db.datacenters.insert(datacenter, (err, key) => { + this._db.datacenters.insert(datacenter, { merge: true }, (err, key) => { if (err) { return cb(err); } diff --git a/packages/portal-api/lib/index.js b/packages/portal-api/lib/index.js index e6f49e05..ea7db301 100644 --- a/packages/portal-api/lib/index.js +++ b/packages/portal-api/lib/index.js @@ -49,8 +49,10 @@ module.exports = function (server, options, next) { settings.data.server = server; const data = new Data(settings.data); - const cpWatcher = new ContainerPilotWatcher(Object.assign(settings.watch, { data })); - const machinesWatcher = new MachinesWatcher(Object.assign(settings.watch, { data })); + const watcherOptions = settings.watch; + watcherOptions.data = data; + const cpWatcher = new ContainerPilotWatcher(watcherOptions); + const machinesWatcher = new MachinesWatcher(watcherOptions); // watcher <-> watcher // portal depends on watcher and vice-versa diff --git a/packages/portal-api/lib/watch/container-pilot.js b/packages/portal-api/lib/watch/container-pilot.js index ea7b6c52..ff8cfcb3 100644 --- a/packages/portal-api/lib/watch/container-pilot.js +++ b/packages/portal-api/lib/watch/container-pilot.js @@ -186,24 +186,15 @@ module.exports = class ContainerPilotWatcher extends Events { const fetchStatus = (ip, next) => { Wreck.get(`http://${ip}:9090/status`, { - timeout: 1000 // 1s - }, (err, res, payload) => { + timeout: 2000, // 2 seconds + json: 'force' + }, (err, res, status) => { if (err) { this.emit('error', err); return next(); } - if (Buffer.isBuffer(payload)) { - payload = payload.toString(); - } - - try { - const status = JSON.parse(payload); - next(null, status); - } catch (err) { - this.emit('error', err); - next(); - } + next(null, status); }); }; diff --git a/packages/portal-api/package.json b/packages/portal-api/package.json index e3aa86fb..778c930b 100644 --- a/packages/portal-api/package.json +++ b/packages/portal-api/package.json @@ -1,6 +1,6 @@ { "name": "portal-api", - "version": "1.8.8", + "version": "1.8.9", "description": "", "main": "./lib/index.js", "scripts": { @@ -29,7 +29,7 @@ "hapi": "^16.5.2", "lab": "^14.2.2", "lodash.findindex": "^4.6.0", - "wreck": "^12.4.0" + "wreck": "^12.5.0" }, "dependencies": { "boom": "^5.2.0",