diff --git a/docker/api/etc/containerpilot.json5 b/docker/api/etc/containerpilot.json5 index c8aa0029..b3307196 100644 --- a/docker/api/etc/containerpilot.json5 +++ b/docker/api/etc/containerpilot.json5 @@ -22,11 +22,6 @@ interval: 5, ttl: 5 }, - tags: [ - 'traefik.backend=api', - 'traefik.frontend.rule=PathPrefixStrip:/api', - 'traefik.frontend.entryPoints=http' - ], when: { source: 'bootstrap', once: 'exitSuccess' @@ -89,11 +84,23 @@ each: 'changed' } }, + { + name: 'onchange-rethinkdb', + exec: 'pkill -SIGHUP node', + when: { + source: 'watch.rethinkdb', + each: 'changed' + } + } ], watches: [ { name: 'docker-compose-api', interval: 3 + }, + { + name: 'rethinkdb', + interval: 3 } ], telemetry: { diff --git a/local-compose.yml b/local-compose.yml index ba89f3c7..a225b2cf 100644 --- a/local-compose.yml +++ b/local-compose.yml @@ -70,13 +70,11 @@ api: mem_limit: 512m links: - consul:consul - - rethinkdb:rethinkdb env_file: - _env environment: - CONSUL=consul - PORT=3000 - - RETHINK_HOST=rethinkdb expose: - 3000 @@ -95,11 +93,16 @@ compose-api: restart: always rethinkdb: - image: rethinkdb + image: autopilotpattern/rethinkdb:2.3.5r1 restart: always mem_limit: 1g + environment: + - CONSUL=consul + - CONSUL_AGENT=1 ports: - 8080:8080 expose: - 28015 - 29015 + dns: + - 127.0.0.1 diff --git a/packages/portal-api/lib/data/index.js b/packages/portal-api/lib/data/index.js index ba2ca9b5..a0e9be8e 100644 --- a/packages/portal-api/lib/data/index.js +++ b/packages/portal-api/lib/data/index.js @@ -3,6 +3,9 @@ // core modules const EventEmitter = require('events'); +const Fs = require('fs'); +const Path = require('path'); +const Url = require('url'); const Util = require('util'); // 3rd party modules @@ -37,13 +40,34 @@ const NON_IMPORTABLE_STATES = [ const NEW_INSTANCE_ID = '__NEW__'; const UNKNOWN_INSTANCE_ID = '__UNKNOWN__'; +const DOCKER_HOST_URL = process.env.DOCKER_HOST ? Url.parse(process.env.DOCKER_HOST) : {}; + const internals = { defaults: { name: 'portal', db: { - test: false + host: 'rethinkdb' }, - dockerComposeHost: 'tcp://0.0.0.0:4242' + docker: { + protocol: 'https', + host: DOCKER_HOST_URL.hostname, + port: DOCKER_HOST_URL.port, + ca: process.env.DOCKER_CERT_PATH ? + Fs.readFileSync(Path.join(process.env.DOCKER_CERT_PATH, 'ca.pem')) : + undefined, + cert: process.env.DOCKER_CERT_PATH ? + Fs.readFileSync(Path.join(process.env.DOCKER_CERT_PATH, 'cert.pem')) : + undefined, + key: process.env.DOCKER_CERT_PATH ? + Fs.readFileSync(Path.join(process.env.DOCKER_CERT_PATH, 'key.pem')) : + undefined + }, + triton: { + url: process.env.SDC_URL, + account: process.env.SDC_ACCOUNT, + keyId: process.env.SDC_KEY_ID + }, + dockerComposeHost: 'tcp://compose-api:4242' }, tables: { 'portals': { id: { type: 'uuid' }, primary: 'id', secondary: false, purge: false }, @@ -82,18 +106,18 @@ class Data extends EventEmitter { constructor (options) { super(); - const settings = Hoek.applyToDefaults(internals.defaults, options || {}); + this._settings = Hoek.applyToDefaults(internals.defaults, options || {}); // Penseur will assert that the options are correct - this._db = new Penseur.Db(settings.name, settings.db); - this._dockerCompose = new DockerClient(settings.dockerComposeHost); - this._docker = new Dockerode(settings.docker); + this._db = new Penseur.Db(this._settings.name, this._settings.db); + this._dockerCompose = new DockerClient(this._settings.dockerComposeHost); + this._docker = new Dockerode(this._settings.docker); this._machines = null; this._triton = null; - this._server = settings.server; + this._server = this._settings.server; Triton.createClient({ - profile: settings.triton + profile: this._settings.triton }, (err, client) => { if (err) { this.emit('error', err); @@ -125,6 +149,19 @@ class Data extends EventEmitter { }); } + reconnectDb (db) { + this._settings.db = db; + + this._db.close(); + this._db = new Penseur.Db(this._settings.name, this._settings.db); + + this.connect((err) => { + if (err) { + this.emit('error', err); + } + }); + } + // portals createPortal (clientPortal, cb) { diff --git a/packages/portal-api/lib/index.js b/packages/portal-api/lib/index.js index 1866bce9..82c97623 100644 --- a/packages/portal-api/lib/index.js +++ b/packages/portal-api/lib/index.js @@ -2,6 +2,7 @@ const Schema = require('joyent-cp-gql-schema'); const Graphi = require('graphi'); +const Hoek = require('hoek'); const Piloted = require('piloted'); const Data = require('./data'); const Pack = require('../package.json'); @@ -9,34 +10,44 @@ const Resolvers = require('./resolvers'); const ContainerPilotWatcher = require('./watch/container-pilot'); const MachinesWatcher = require('./watch/machines'); -const { - NAMESPACE -} = process.env; - -const namespace = NAMESPACE ? - `/${NAMESPACE}` : - ''; - -const internals = {}; +const internals = { + namespace: process.env.NAMESPACE ? `/${process.env.NAMESPACE}` : '', + defaults: { + data: { + db: {} + }, + watch: { + url: process.env.SDC_URL, + account: process.env.SDC_ACCOUNT, + keyId: process.env.SDC_KEY_ID + } + } +}; module.exports = function (server, options, next) { + const settings = Hoek.applyToDefaults(internals.defaults, options || {}); + try { const docker = Piloted.service('docker-compose-api'); if (docker) { - options.data.dockerComposeHost = `tcp://${docker.address}:${docker.port}`; + settings.data.dockerComposeHost = `tcp://${docker.address}:${docker.port}`; + } + + const rethinkdb = Piloted.service('rethinkdb'); + if (rethinkdb) { + settings.data.db.host = rethinkdb.address; } } catch (ex) { - console.error(ex); + server.log(['error'], ex); } - options.watch.server = server; - options.data.server = server; - const data = new Data(options.data); - const cpWatcher = new ContainerPilotWatcher(Object.assign(options.watch, { data })); - const machinesWatcher = new MachinesWatcher(Object.assign(options.watch, { - data - })); + settings.watch.server = server; + settings.data.server = server; + + const data = new Data(settings.data); + const cpWatcher = new ContainerPilotWatcher(Object.assign(settings.watch, { data })); + const machinesWatcher = new MachinesWatcher(Object.assign(settings.watch, { data })); // watcher <-> watcher // portal depends on watcher and vice-versa @@ -68,8 +79,8 @@ module.exports = function (server, options, next) { { register: Graphi, options: { - graphqlPath: `${namespace}/graphql`, - graphiqlPath: `${namespace}/graphiql`, + graphqlPath: `${internals.namespace}/graphql`, + graphiqlPath: `${internals.namespace}/graphiql`, schema: Schema, resolvers: Resolvers(data) } @@ -91,10 +102,13 @@ module.exports.attributes = { internals.refresh = function (data) { return () => { const docker = Piloted.service('docker-compose-api'); - if (!docker) { - return; + if (docker) { + data.reconnectCompose(`tcp://${docker.address}:${docker.port}`); } - data.reconnectCompose(`tcp://${docker.address}:${docker.port}`); + const rethinkdb = Piloted.service('rethinkdb'); + if (rethinkdb) { + data.reconnectDb({ host: rethinkdb.address }); + } }; }; diff --git a/packages/portal-api/lib/watch/machines.js b/packages/portal-api/lib/watch/machines.js index 22f24d87..f38501ce 100644 --- a/packages/portal-api/lib/watch/machines.js +++ b/packages/portal-api/lib/watch/machines.js @@ -152,6 +152,10 @@ module.exports = class MachineWatcher { } getVersion (deploymentGroup, cb) { + if (typeof deploymentGroup.version !== 'function') { + return cb(new Error('version must be a function')); + } + deploymentGroup.version() .then((version) => { return cb(null, version); diff --git a/packages/portal-api/package.json b/packages/portal-api/package.json index 16f9ed04..68dc491f 100644 --- a/packages/portal-api/package.json +++ b/packages/portal-api/package.json @@ -10,6 +10,7 @@ "lint": "belly-button --fix", "lint-ci": "belly-button", "test": "lab -c", + "test-docker": "docker-compose -f test-compose.yml up --abort-on-container-exit --build --force-recreate api", "test-ci": "echo 0", "start": "node server.js", "dev": "CORS=1 NAMESPACE=api node server.js", @@ -21,17 +22,13 @@ "devDependencies": { "belly-button": "^3.1.0", "brule": "^2.0.0", - "code": "^4.1.0", "good": "^7.2.0", "good-console": "^6.4.0", "good-squeeze": "^5.0.2", "hapi": "^16.4.3", - "hapi-swagger": "^7.7.0", - "inert": "^4.2.0", - "lab": "^14.0.1", + "lab": "^14.1.1", "lodash.findindex": "^4.6.0", - "vision": "^4.1.1", - "wreck": "^12.2.2" + "wreck": "^12.2.3" }, "dependencies": { "boom": "^5.1.0", diff --git a/packages/portal-api/server.js b/packages/portal-api/server.js index c42f4c27..dad53f2c 100644 --- a/packages/portal-api/server.js +++ b/packages/portal-api/server.js @@ -3,15 +3,9 @@ const Brule = require('brule'); const Good = require('good'); const Hapi = require('hapi'); -const HapiSwagger = require('hapi-swagger'); -const Inert = require('inert'); const Toppsy = require('toppsy'); -const Vision = require('vision'); const Pack = require('./package'); const Portal = require('./lib'); -const Path = require('path'); -const Fs = require('fs'); -const Url = require('url'); const server = new Hapi.Server(); @@ -22,55 +16,6 @@ server.connection({ } }); -const swaggerOptions = { - info: { - 'title': 'Portal API Documentation', - 'version': Pack.version - } -}; - -const { - DOCKER_HOST, - DOCKER_CERT_PATH, - SDC_URL, - SDC_ACCOUNT, - SDC_KEY_ID -} = process.env; - -const DOCKER_HOST_URL = DOCKER_HOST ? Url.parse(DOCKER_HOST) : {}; - -const portalOptions = { - data: { - db: { - host: process.env.RETHINK_HOST || 'localhost' - }, - docker: { - protocol: 'https', - host: DOCKER_HOST_URL.hostname, - port: DOCKER_HOST_URL.port, - ca: DOCKER_CERT_PATH ? - Fs.readFileSync(Path.join(DOCKER_CERT_PATH, 'ca.pem')) : - undefined, - cert: DOCKER_CERT_PATH ? - Fs.readFileSync(Path.join(DOCKER_CERT_PATH, 'cert.pem')) : - undefined, - key: DOCKER_CERT_PATH ? - Fs.readFileSync(Path.join(DOCKER_CERT_PATH, 'key.pem')) : - undefined - }, - triton: { - url: SDC_URL, - account: SDC_ACCOUNT, - keyId: SDC_KEY_ID - } - }, - watch: { - url: SDC_URL, - account: SDC_ACCOUNT, - keyId: SDC_KEY_ID - } -}; - const goodOptions = { ops: { interval: 1000 @@ -88,20 +33,11 @@ const goodOptions = { server.register([ Brule, - Inert, - Vision, + Portal, { register: Good, options: goodOptions }, - { - register: Portal, - options: portalOptions - }, - { - register: HapiSwagger, - options: swaggerOptions - }, { register: Toppsy, options: { namespace: 'portal', subsystem: 'api' } diff --git a/packages/portal-api/test-Dockerfile b/packages/portal-api/test-Dockerfile new file mode 100644 index 00000000..e1c248d0 --- /dev/null +++ b/packages/portal-api/test-Dockerfile @@ -0,0 +1,30 @@ +FROM node:8-alpine + +# Install dependencies +RUN set -x \ + && apk update \ + && apk add --update curl bash build-base python zeromq-dev openssh \ + && apk upgrade \ + && rm -rf /var/cache/apk/* + +# Install ContainerPilot +ENV CP_SHA1 8d680939a8a5c8b27e764d55a78f5e3ae7b42ef4 +ENV CONTAINERPILOT_VERSION 3.3.3 +RUN curl -Lo /tmp/containerpilot.tar.gz "https://github.com/joyent/containerpilot/releases/download/${CONTAINERPILOT_VERSION}/containerpilot-${CONTAINERPILOT_VERSION}.tar.gz" \ + && echo "${CP_SHA1} /tmp/containerpilot.tar.gz" | sha1sum -c \ + && tar zxf /tmp/containerpilot.tar.gz -C /bin \ + && rm /tmp/containerpilot.tar.gz + +# Copy required files +RUN mkdir -p /opt/app/ +COPY ./ /opt/app/ +COPY test/containerpilot.json5 /etc/containerpilot.json5 +COPY test/prestart.sh /bin/prestart.sh +ENV CONTAINERPILOT /etc/containerpilot.json5 + + +# Install dependencies +WORKDIR /opt/app/ +RUN npm install + +CMD ["/bin/containerpilot"] diff --git a/packages/portal-api/test-compose.yml b/packages/portal-api/test-compose.yml new file mode 100644 index 00000000..ed3014a5 --- /dev/null +++ b/packages/portal-api/test-compose.yml @@ -0,0 +1,55 @@ +consul: + image: autopilotpattern/consul:0.7.2-r0.8 + command: > + /usr/local/bin/containerpilot + /bin/consul agent -server + -config-dir=/etc/consul + -log-level=err + -bootstrap-expect 1 + -ui-dir /ui + restart: always + mem_limit: 128m + ports: + - 8500:8500 + dns: + - 127.0.0.1 +compose-api: + image: joyent/copilot-compose + links: + - consul:consul + expose: + - 4242 + env_file: + - ../../_env + environment: + - CONSUL=consul + restart: always +rethinkdb: + image: autopilotpattern/rethinkdb:2.3.5r1 + restart: always + mem_limit: 1g + links: + - consul:consul + environment: + - CONSUL=consul + - CONSUL_AGENT=1 + ports: + - 8080:8080 + expose: + - 28015 + - 29015 + dns: + - 127.0.0.1 +api: + build: ./ + dockerfile: ./test-Dockerfile + links: + - consul:consul + - rethinkdb:rethinkdb + - compose-api:compose-api + env_file: + - ../../_env + environment: + - CONSUL=consul + dns: + - 127.0.0.1 diff --git a/packages/portal-api/test/containerpilot.json5 b/packages/portal-api/test/containerpilot.json5 new file mode 100644 index 00000000..f79c559c --- /dev/null +++ b/packages/portal-api/test/containerpilot.json5 @@ -0,0 +1,27 @@ +{ + consul: 'consul:8500', + jobs: [ + { + name: 'setup-config', + exec: '/bin/prestart.sh' + }, + { + name: 'tests', + exec: 'node node_modules/.bin/lab -c', + when: { + source: 'setup-config', + once: 'exitSuccess' + } + } + ], + watches: [ + { + name: 'rethinkdb', + interval: 5 + }, + { + name: 'docker-compose-api', + interval: 5 + } + ] +} diff --git a/packages/portal-api/test/data/index.js b/packages/portal-api/test/data/index.js index 0f843892..93e830aa 100644 --- a/packages/portal-api/test/data/index.js +++ b/packages/portal-api/test/data/index.js @@ -3,21 +3,17 @@ const Fs = require('fs'); const Path = require('path'); const Code = require('code'); -const Lab = require('lab'); +const { describe, it, afterEach, expect } = exports.lab = require('lab').script(); const PortalData = require('../../lib/data'); -const lab = exports.lab = Lab.script(); -const afterEach = lab.afterEach; -const it = lab.it; -const describe = lab.describe; -const expect = Code.expect; - - const internals = { options: { name: 'test', - db: { test: true } + db: { test: true }, + server: { + log: function () {} + } }, composeFile: Fs.readFileSync(Path.join(__dirname, 'docker-compose.yml')).toString() }; diff --git a/packages/portal-api/test/index.js b/packages/portal-api/test/index.js index 98de195d..113041f4 100644 --- a/packages/portal-api/test/index.js +++ b/packages/portal-api/test/index.js @@ -1,24 +1,13 @@ 'use strict'; -const Code = require('code'); const Hapi = require('hapi'); -const Lab = require('lab'); +const { describe, it, beforeEach, afterEach, expect } = exports.lab = require('lab').script(); const PortalData = require('../lib/data'); const PortalApi = require('../'); -// Test shortcuts - -const lab = exports.lab = Lab.script(); -const afterEach = lab.afterEach; -const beforeEach = lab.beforeEach; -const describe = lab.describe; -const it = lab.it; -const expect = Code.expect; - - const internals = { - options: { data: { test: true, name: 'test' } } + options: { data: { name: 'test', db: { test: true } } } }; internals.register = { register: PortalApi, options: internals.options }; diff --git a/packages/portal-api/test/prestart.sh b/packages/portal-api/test/prestart.sh new file mode 100755 index 00000000..04fcba52 --- /dev/null +++ b/packages/portal-api/test/prestart.sh @@ -0,0 +1,21 @@ +#!/bin/bash + +# Copy creds from env vars to files on disk +if [ -n ${!TRITON_CREDS_PATH} ] \ + && [ -n ${!TRITON_CA} ] \ + && [ -n ${!TRITON_CERT} ] \ + && [ -n ${!TRITON_KEY} ] +then + mkdir -p ${TRITON_CREDS_PATH} + echo -e "${TRITON_CA}" | tr '#' '\n' > ${TRITON_CREDS_PATH}/ca.pem + echo -e "${TRITON_CERT}" | tr '#' '\n' > ${TRITON_CREDS_PATH}/cert.pem + echo -e "${TRITON_KEY}" | tr '#' '\n' > ${TRITON_CREDS_PATH}/key.pem +fi + +eval `/usr/bin/ssh-agent -s` +mkdir -p ~/.ssh +echo -e "${SDC_KEY_PUB}" | tr '#' '\n' > ~/.ssh/id_rsa.pub +echo -e "${SDC_KEY}" | tr '#' '\n' > ~/.ssh/id_rsa +chmod 400 ~/.ssh/id_rsa.pub +chmod 400 ~/.ssh/id_rsa +ssh-add ~/.ssh/id_rsa diff --git a/packages/portal-api/test/watch/container-pilot.js b/packages/portal-api/test/watch/container-pilot.js index 93358a87..3c9c5522 100644 --- a/packages/portal-api/test/watch/container-pilot.js +++ b/packages/portal-api/test/watch/container-pilot.js @@ -1,6 +1,6 @@ 'use strict'; -const Lab = require('lab'); +const { it, expect } = exports.lab = require('lab').script(); const Uuid = require('uuid/v4'); const ContainerPilotWatch = require('../../lib/watch/container-pilot'); @@ -8,11 +8,6 @@ const DataMock = require('../_mocks/data'); const TritonMock = require('../_mocks/triton'); -const lab = exports.lab = Lab.script(); -const it = lab.it; -// const expect = Lab.expect; - - it('sets instance health statuses appropriately', (done) => { const networks = [{ id: Uuid(), diff --git a/packages/portal-api/test/watch/machines.js b/packages/portal-api/test/watch/machines.js index 175ee8b0..7e59e629 100644 --- a/packages/portal-api/test/watch/machines.js +++ b/packages/portal-api/test/watch/machines.js @@ -1,12 +1,7 @@ 'use strict'; -const Lab = require('lab'); -const PortalWatch = require('../../lib/watch'); - - -const lab = exports.lab = Lab.script(); -const it = lab.it; -const expect = Lab.expect; +const { describe, it, expect } = exports.lab = require('lab').script(); +const PortalWatch = require('../../lib/watch/machines'); it('updates instances with the current status', (done) => {