From a58be6be37888bcefd17aadb9475146cf437c840 Mon Sep 17 00:00:00 2001 From: geek Date: Tue, 12 Sep 2017 20:39:39 -0500 Subject: [PATCH] chore: more stable db connect/bootstrap --- docker-compose.yml | 8 ++-- docker/api/Dockerfile | 4 +- docker/api/bootstrap-data.js | 48 ++++++++++--------- docker/api/etc/containerpilot.json5 | 4 +- docker/api/package.json | 4 +- local-compose.yml | 8 +--- packages/portal-api/lib/data/index.js | 21 +++++++- .../portal-api/lib/watch/container-pilot.js | 6 ++- packages/portal-api/package.json | 2 +- triton-compose.yml | 2 +- 10 files changed, 64 insertions(+), 43 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 0711172c..697dcd55 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -28,7 +28,7 @@ consul: # it is included here for demo purposes and is not required ############################################################################# prometheus: - image: autopilotpattern/prometheus:1.7.1-r20 + image: autopilotpattern/prometheus:1.7.1-r24 restart: always mem_limit: 1g ports: @@ -46,7 +46,7 @@ prometheus: # FRONTEND ############################################################################# frontend: - image: joyent/copilot-frontend:1.0.0 + image: joyent/copilot-frontend:1.3.4 mem_limit: 512m links: - consul:consul @@ -66,7 +66,7 @@ frontend: # BACKEND ############################################################################# api: - image: joyent/copilot-api:1.1.0 + image: joyent/copilot-api:1.8.5 mem_limit: 512m links: - consul:consul @@ -83,7 +83,7 @@ api: # Docker-compose wrapper # Create _env file from running ./setup.sh compose-api: - image: joyent/copilot-compose:1.0.0 + image: joyent/copilot-compose:1.1.0 links: - consul:consul expose: diff --git a/docker/api/Dockerfile b/docker/api/Dockerfile index fad65601..862447b7 100644 --- a/docker/api/Dockerfile +++ b/docker/api/Dockerfile @@ -17,8 +17,8 @@ RUN curl --retry 7 --fail -vo /tmp/consul.zip "https://releases.hashicorp.com/co && mkdir /config # Install Containerpilot -ENV CONTAINERPILOT_VERSION 3.4.1 -RUN export CONTAINERPILOT_CHECKSUM=4d13cfb345de86135ab2271b77516c6b6a7bed3a \ +ENV CONTAINERPILOT_VERSION 3.4.2 +RUN export CONTAINERPILOT_CHECKSUM=5c99ae9ede01e8fcb9b027b5b3cb0cfd8c0b8b88 \ && export archive=containerpilot-${CONTAINERPILOT_VERSION}.tar.gz \ && curl -Lso /tmp/${archive} \ "https://github.com/joyent/containerpilot/releases/download/${CONTAINERPILOT_VERSION}/${archive}" \ diff --git a/docker/api/bootstrap-data.js b/docker/api/bootstrap-data.js index 137a8acc..c4073638 100644 --- a/docker/api/bootstrap-data.js +++ b/docker/api/bootstrap-data.js @@ -13,13 +13,24 @@ const loadConfig = function () { const docker = Piloted.service('docker-compose-api'); const rethink = Piloted.service('rethinkdb'); - if (docker && rethink) { - bootstrap({ docker, rethink }); - } else if (!timeoutId) { + const retry = () => { timeoutId = setTimeout(() => { timeoutId = null; Piloted.refresh(); }, 1000); + }; + + if (docker && rethink) { + bootstrap({ docker, rethink }, (err) => { + if (err) { + console.error(err); + return retry(); + } + + process.exit(0); + }); + } else if (!timeoutId) { + retry(); } }; @@ -28,7 +39,7 @@ Piloted.on('refresh', () => { }); -const bootstrap = function ({ docker, rethink }) { +const bootstrap = function ({ docker, rethink }, cb) { const settings = { db: { host: rethink.address @@ -57,27 +68,24 @@ const bootstrap = function ({ docker, rethink }) { const data = new Data(settings); const region = process.env.TRITON_DC || 'us-sw-1'; - data.connect(err => { + data.connect((err) => { if (err) { - console.error(err); - return; + return cb(err); } data.getDatacenters((err, datacenters) => { if (err) { - console.error(err); - return; + return cb(err); } + // Don't continue since data is already bootstrapped if (datacenters && datacenters.length) { - process.exit(0); - return; + return cb(); } data.createDatacenter({ region, name: region }, (err, datacenter) => { if (err) { - console.error(err); - return; + return cb(err); } Triton.createClient( @@ -86,22 +94,19 @@ const bootstrap = function ({ docker, rethink }) { }, (err, { cloudapi }) => { if (err) { - console.error(err); - return; + return cb(err); } cloudapi.getAccount((err, { firstName, lastName, email, login }) => { if (err) { - console.error(err); - return; + return cb(err); } data.createUser( { firstName, lastName, email, login }, (err, user) => { if (err) { - console.error(err); - return; + return cb(err); } data.createPortal( @@ -111,12 +116,11 @@ const bootstrap = function ({ docker, rethink }) { }, (err, portal) => { if (err) { - console.error(err); - return; + return cb(err); } console.log('data bootstrapped'); - process.exit(0); + cb(); } ); } diff --git a/docker/api/etc/containerpilot.json5 b/docker/api/etc/containerpilot.json5 index b3307196..c0f14123 100644 --- a/docker/api/etc/containerpilot.json5 +++ b/docker/api/etc/containerpilot.json5 @@ -36,8 +36,8 @@ '-log-level=err', '-rejoin', '-retry-join', '{{ .CONSUL | default "consul" }}', - '-retry-max', '10', - '-retry-interval', '10s'], + '-retry-max', '20', + '-retry-interval', '5s'], restarts: 'unlimited' }, { diff --git a/docker/api/package.json b/docker/api/package.json index 387e7d70..464b0d50 100644 --- a/docker/api/package.json +++ b/docker/api/package.json @@ -16,12 +16,12 @@ "good-console": "^6.4.0", "good-squeeze": "^5.0.2", "graphi": "^2.3.0", - "hapi": "^16.1.0", + "hapi": "^16.6.0", "hoek": "^4.1.1", "joi": "^10.6.0", "joyent-cp-gql-schema": "^1.7.0", "piloted": "^3.1.1", - "portal-api": "^1.8.2", + "portal-api": "^1.8.5", "toppsy": "^1.1.0", "triton": "^5.2.0" } diff --git a/local-compose.yml b/local-compose.yml index bf7952d8..a4a8de74 100644 --- a/local-compose.yml +++ b/local-compose.yml @@ -18,8 +18,6 @@ consul: mem_limit: 128m ports: - 8500:8500 - dns: - - 127.0.0.1 ############################################################################# # PROMETHEUS @@ -28,7 +26,7 @@ consul: # it is included here for demo purposes and is not required ############################################################################# prometheus: - image: autopilotpattern/prometheus:1.7.1-r20 + image: autopilotpattern/prometheus:1.7.1-r24 restart: always mem_limit: 1g ports: @@ -38,8 +36,6 @@ prometheus: environment: - CONSUL=consul - CONSUL_AGENT=1 - dns: - - 127.0.0.1 ############################################################################# @@ -58,8 +54,6 @@ frontend: ports: - "80:80" - "443:443" - dns: - - 127.0.0.1 ############################################################################# diff --git a/packages/portal-api/lib/data/index.js b/packages/portal-api/lib/data/index.js index 06524d4a..10e704c8 100644 --- a/packages/portal-api/lib/data/index.js +++ b/packages/portal-api/lib/data/index.js @@ -148,7 +148,26 @@ class Data extends EventEmitter { } connect (cb) { - this._db.establish(internals.tables, cb); + if (!this._db._connection) { + return this._db._connect((err) => { + if (err) { + return cb(err); + } + + return this.connect(cb); + }); + } + + this._db._exists((err, exists) => { + if (exists) { + const tables = this._db._normalizeTables(internals.tables); + this._db.table(tables); + + return cb(); + } + + this._db.establish(internals.tables, cb); + }); } reconnectCompose (dockerComposeHost) { diff --git a/packages/portal-api/lib/watch/container-pilot.js b/packages/portal-api/lib/watch/container-pilot.js index dc6a96a2..ea7b6c52 100644 --- a/packages/portal-api/lib/watch/container-pilot.js +++ b/packages/portal-api/lib/watch/container-pilot.js @@ -110,7 +110,7 @@ module.exports = class ContainerPilotWatcher extends Events { }; const getDeploymentGroups = (err, portal) => { - if (err) { + if (err || !portal) { return cb(err); } @@ -519,6 +519,10 @@ module.exports = class ContainerPilotWatcher extends Events { return cb(); } + if (!dgs || !dgs.length) { + return cb(); + } + VAsync.forEachParallel({ inputs: dgs, func: (dg, next) => { diff --git a/packages/portal-api/package.json b/packages/portal-api/package.json index 957bf53e..56a6af3a 100644 --- a/packages/portal-api/package.json +++ b/packages/portal-api/package.json @@ -1,6 +1,6 @@ { "name": "portal-api", - "version": "1.8.2", + "version": "1.8.5", "description": "", "main": "./lib/index.js", "scripts": { diff --git a/triton-compose.yml b/triton-compose.yml index 5642c5d3..e3d68ec0 100644 --- a/triton-compose.yml +++ b/triton-compose.yml @@ -81,7 +81,7 @@ frontend: # BACKEND ############################################################################# api: - image: joyent/copilot-api:1.8.2 + image: joyent/copilot-api:1.8.5 mem_limit: 1g expose: - 3000