bug: fix bootstrap process

2017-09-14 19:08:35 -05:00 · 2017-09-14 19:08:35 -05:00 · 1b5a2bfa9a
parent beb8062572
commit 1b5a2bfa9a
11 changed files with 137 additions and 136 deletions
--- a/README.md
+++ b/README.md
@ -51,32 +51,35 @@ $ ./setup.sh ~/path/to/TRITON_PRIVATE_KEY keys-test.com/ca.crt keys-test.com/ser

 ## Usage

-You have 3 options for where to run CoPilot. You can either run it using the published docker images locally, or on Triton. The last option is to build the docker images and run docker containers from these locally built images.
+You have 3 options for where to run CoPilot. You can either run it using the published docker images on Triton or locally. The last option is to build the docker images and run docker containers from these locally built images.
+
+
+### Deploy and run CoPilot on Triton
+
+Optionally use [_triton-docker_](https://github.com/joyent/triton-docker-cli)
+```sh
+$ triton-compose up -d
+```
+
+Or use `docker-compose` with preconfigured Triton environment variables
+```sh
+$ docker-compose up -d
+```
+

 ### Start CoPilot using published docker images locally

 ```sh
-$ docker-compose up -d
+$ docker-compose -f local-compose.yml up -d
 ```

 Navigate to [https://localhost]() to load the dashboard.


-### Deploy and run CoPilot on Triton
+### Build and run CoPilot locally for development

 ```sh
-$ docker-compose -f triton-compose.yml up -d
-```
-
-Optionally use [_triton-docker_](https://github.com/joyent/triton-docker-cli)
-```sh
-$ triton-compose -f triton-compose.yml up -d
-```
-
-### Build and run CoPilot locally
-
-```sh
-$ docker-compose -f local-compose.yml up -d
+$ docker-compose -f dev-compose.yml up -d
 ```

 ## Contribute
--- a/triton-compose.yml
+++ b/triton-compose.yml
@ -16,8 +16,6 @@ consul:
      -ui-dir /ui
  restart: always
  mem_limit: 128m
-  labels:
-    - triton.cns.services=copilot-consul
  ports:
    - 8500:8500

@ -33,82 +31,80 @@ prometheus:
  mem_limit: 1g
  ports:
    - 9090:9090
-  labels:
-    # This label sets the CNS name, Triton's automatic DNS
-    # Learn more at https://docs.joyent.com/public-cloud/network/cns
-    - triton.cns.services=prometheus
-    # This label selects the proper Joyent resource package
-    # https://www.joyent.com/blog/optimizing-docker-on-triton#ram-cpu-and-disk-resources-for-your-containers
-    - com.joyent.package=g4-highcpu-1G
-  env_file:
-    - _env
+  links:
+    - consul:consul
  environment:
+    - CONSUL=consul
    - CONSUL_AGENT=1
-
-
-rethinkdb:
-  image: autopilotpattern/rethinkdb:2.3.5r1
-  restart: always
-  mem_limit: 1g
-  env_file:
-    - _env
-  environment:
-    - CONSUL_AGENT=1
-  expose:
-    - 28015
-    - 29015
+  dns:
+    - 127.0.0.1


 #############################################################################
 #  FRONTEND
 #############################################################################
 frontend:
-  image: joyent/copilot-frontend:1.3.4
+  build: docker/frontend
  mem_limit: 512m
-  labels:
-    - triton.cns.services=copilot
+  links:
+    - consul:consul
  env_file:
    - _env
  environment:
+    - CONSUL=consul
    - PORT=443
  ports:
    - "80:80"
    - "443:443"
-  restart: always
+  dns:
+    - 127.0.0.1


 #############################################################################
 #  BACKEND
 #############################################################################
 api:
-  image: joyent/copilot-api:1.8.8
-  mem_limit: 1g
-  expose:
-    - 3000
+  build: docker/api
+  mem_limit: 512m
+  links:
+    - consul:consul
  env_file:
    - _env
  environment:
+    - CONSUL=consul
    - PORT=3000
-  restart: always
+  expose:
+    - 3000

 # Docker-compose wrapper
 # Create _env file from running ./setup.sh
 compose-api:
-  image: joyent/copilot-compose:1.1.0
+  build: docker/compose-api
+  links:
+    - consul:consul
  expose:
    - 4242
  env_file:
    - _env
+  environment:
+    - CONSUL=consul
  restart: always

 rethinkdb:
-  image: autopilotpattern/rethinkdb:2.3.5r2
+  image: autopilotpattern/rethinkdb:2.3.5r1
  restart: always
  mem_limit: 1g
+  links:
+    - consul:consul
  env_file:
    - _env
  environment:
+    - CONSUL=consul
    - CONSUL_AGENT=1
+  ports:
+    - 8080:8080
  expose:
    - 28015
    - 29015
+  dns:
+    - 127.0.0.1
--- a/docker-compose.yml
+++ b/docker-compose.yml
@ -16,10 +16,10 @@ consul:
      -ui-dir /ui
  restart: always
  mem_limit: 128m
+  labels:
+    - triton.cns.services=copilot-consul
  ports:
    - 8500:8500
-  dns:
-    - 127.0.0.1

 #############################################################################
 #  PROMETHEUS
@ -28,87 +28,87 @@ consul:
 #  it is included here for demo purposes and is not required
 #############################################################################
 prometheus:
-  image: autopilotpattern/prometheus:1.7.1-r20
+  image: autopilotpattern/prometheus:1.7.1-r24
  restart: always
  mem_limit: 1g
  ports:
    - 9090:9090
-  links:
-    - consul:consul
+  labels:
+    # This label sets the CNS name, Triton's automatic DNS
+    # Learn more at https://docs.joyent.com/public-cloud/network/cns
+    - triton.cns.services=prometheus
+    # This label selects the proper Joyent resource package
+    # https://www.joyent.com/blog/optimizing-docker-on-triton#ram-cpu-and-disk-resources-for-your-containers
+    - com.joyent.package=g4-highcpu-1G
+  env_file:
+    - _env
  environment:
-    - CONSUL=consul
    - CONSUL_AGENT=1
-  dns:
-    - 127.0.0.1
+
+
+rethinkdb:
+  image: autopilotpattern/rethinkdb:2.3.5r1
+  restart: always
+  mem_limit: 1g
+  env_file:
+    - _env
+  environment:
+    - CONSUL_AGENT=1
+  expose:
+    - 28015
+    - 29015


 #############################################################################
 #  FRONTEND
 #############################################################################
 frontend:
-  image: joyent/copilot-frontend:1.0.0
+  image: joyent/copilot-frontend:1.3.4
  mem_limit: 512m
-  links:
-    - consul:consul
+  labels:
+    - triton.cns.services=copilot
  env_file:
    - _env
  environment:
-    - CONSUL=consul
    - PORT=443
  ports:
    - "80:80"
    - "443:443"
-  dns:
-    - 127.0.0.1
+  restart: always


 #############################################################################
 #  BACKEND
 #############################################################################
 api:
-  image: joyent/copilot-api:1.8.8
-  mem_limit: 512m
-  links:
-    - consul:consul
-    - rethinkdb:rethinkdb
+  image: joyent/copilot-api:1.8.9
+  mem_limit: 1g
+  expose:
+    - 3000
  env_file:
    - _env
  environment:
-    - CONSUL=consul
    - PORT=3000
-    - RETHINK_HOST=rethinkdb
-  expose:
-    - 3000
+  restart: always

 # Docker-compose wrapper
 # Create _env file from running ./setup.sh
 compose-api:
-  image: joyent/copilot-compose:1.0.0
-  links:
-    - consul:consul
+  image: joyent/copilot-compose:1.1.0
  expose:
    - 4242
  env_file:
    - _env
-  environment:
-    - CONSUL=consul
  restart: always

 rethinkdb:
-  image: autopilotpattern/rethinkdb:2.3.5r1
+  image: autopilotpattern/rethinkdb:2.3.5r2
  restart: always
  mem_limit: 1g
-  links:
-    - consul:consul
  env_file:
    - _env
  environment:
-    - CONSUL=consul
    - CONSUL_AGENT=1
-  ports:
-    - 8080:8080
  expose:
    - 28015
    - 29015
-  dns:
-    - 127.0.0.1
--- a/docker/api/bootstrap-data.js
+++ b/docker/api/bootstrap-data.js
@ -73,13 +73,9 @@ const bootstrap = function ({ docker, rethink }, cb) {
      return cb(err);
    }

-    data.getDatacenters((err, datacenters) => {
-      if (err) {
-        return cb(err);
-      }
-
+    data.getPortal({}, (err, portal) => {
      // Don't continue since data is already bootstrapped
-      if (datacenters && datacenters.length) {
+      if (portal) {
        return cb();
      }

@ -98,17 +94,13 @@ const bootstrap = function ({ docker, rethink }, cb) {
            return cb(err);
          }

-          cloudapi.getAccount((err, {
-            id,
-            firstName,
-            lastName,
-            email,
-            login
-          }) => {
+          cloudapi.getAccount({}, (err, body) => {
            if (err) {
              return cb(err);
            }

+            const { id, firstName, lastName, email, login } = body;
+
            data.createUser({
              tritonId: id,
              firstName,
--- a/docker/api/etc/containerpilot.json5
+++ b/docker/api/etc/containerpilot.json5
@ -6,13 +6,34 @@
      exec: '/bin/prestart.sh'
    },
    {
-      name: 'bootstrap',
-      exec: 'node bootstrap-data.js',
+      name: 'consul-agent',
+      exec: ['/usr/local/bin/consul', 'agent',
+                  '-data-dir=/data',
+                  '-config-dir=/config',
+                  '-log-level=err',
+                  '-rejoin',
+                  '-retry-join', '{{ .CONSUL | default "consul" }}',
+                  '-retry-max', '20',
+                  '-retry-interval', '5s'],
+      restarts: 'unlimited',
+      "health": {
+        "exec": "curl -so /dev/null http://localhost:8500",
+        "interval": 10,
+        "ttl": 25
+      },
      when: {
        source: 'setup-config',
        once: 'exitSuccess'
      }
    },
+    {
+      name: 'bootstrap',
+      exec: 'node bootstrap-data.js',
+      when: {
+        source: 'consul-agent',
+        once: 'healthy'
+      }
+    },
    {
      name: 'api',
      port: {{.PORT}},
@ -28,18 +49,6 @@
      },
      restarts: 'unlimited'
    },
-    {
-      name: 'consul-agent',
-      exec: ['/usr/local/bin/consul', 'agent',
-                  '-data-dir=/data',
-                  '-config-dir=/config',
-                  '-log-level=err',
-                  '-rejoin',
-                  '-retry-join', '{{ .CONSUL | default "consul" }}',
-                  '-retry-max', '20',
-                  '-retry-interval', '5s'],
-      restarts: 'unlimited'
-    },
    {
      name: 'sensor_memory_usage',
      exec: '/bin/sensors.sh memory',
--- a/docker/api/package.json
+++ b/docker/api/package.json
@ -21,7 +21,7 @@
    "joi": "^10.6.0",
    "joyent-cp-gql-schema": "^1.7.0",
    "piloted": "^3.1.1",
-    "portal-api": "^1.8.8",
+    "portal-api": "^1.8.9",
    "toppsy": "^1.1.0",
    "triton": "^5.2.0"
  }
--- a/local-compose.yml
+++ b/local-compose.yml
@ -18,6 +18,8 @@ consul:
  mem_limit: 128m
  ports:
    - 8500:8500
+  dns:
+    - 127.0.0.1

 #############################################################################
 #  PROMETHEUS
@ -26,7 +28,7 @@ consul:
 #  it is included here for demo purposes and is not required
 #############################################################################
 prometheus:
-  image: autopilotpattern/prometheus:1.7.1-r24
+  image: autopilotpattern/prometheus:1.7.1-r20
  restart: always
  mem_limit: 1g
  ports:
@ -36,13 +38,15 @@ prometheus:
  environment:
    - CONSUL=consul
    - CONSUL_AGENT=1
+  dns:
+    - 127.0.0.1


 #############################################################################
 #  FRONTEND
 #############################################################################
 frontend:
-  build: docker/frontend
+  image: joyent/copilot-frontend:1.3.4
  mem_limit: 512m
  links:
    - consul:consul
@ -54,28 +58,32 @@ frontend:
  ports:
    - "80:80"
    - "443:443"
+  dns:
+    - 127.0.0.1


 #############################################################################
 #  BACKEND
 #############################################################################
 api:
-  build: docker/api
+  image: joyent/copilot-api:1.8.9
  mem_limit: 512m
  links:
    - consul:consul
+    - rethinkdb:rethinkdb
  env_file:
    - _env
  environment:
    - CONSUL=consul
    - PORT=3000
+    - RETHINK_HOST=rethinkdb
  expose:
    - 3000

 # Docker-compose wrapper
 # Create _env file from running ./setup.sh
 compose-api:
-  build: docker/compose-api
+  image: joyent/copilot-compose:1.0.0
  links:
    - consul:consul
  expose:
--- a/packages/portal-api/lib/data/index.js
+++ b/packages/portal-api/lib/data/index.js
@ -263,7 +263,7 @@ class Data extends EventEmitter {
  // datacenters

  createDatacenter (datacenter, cb) {
-    this._db.datacenters.insert(datacenter, (err, key) => {
+    this._db.datacenters.insert(datacenter, { merge: true }, (err, key) => {
      if (err) {
        return cb(err);
      }
--- a/packages/portal-api/lib/index.js
+++ b/packages/portal-api/lib/index.js
@ -49,8 +49,10 @@ module.exports = function (server, options, next) {
  settings.data.server = server;

  const data = new Data(settings.data);
-  const cpWatcher = new ContainerPilotWatcher(Object.assign(settings.watch, { data }));
-  const machinesWatcher = new MachinesWatcher(Object.assign(settings.watch, { data }));
+  const watcherOptions = settings.watch;
+  watcherOptions.data = data;
+  const cpWatcher = new ContainerPilotWatcher(watcherOptions);
+  const machinesWatcher = new MachinesWatcher(watcherOptions);

  // watcher <-> watcher
  // portal depends on watcher and vice-versa
--- a/packages/portal-api/lib/watch/container-pilot.js
+++ b/packages/portal-api/lib/watch/container-pilot.js
@ -186,24 +186,15 @@ module.exports = class ContainerPilotWatcher extends Events {

    const fetchStatus = (ip, next) => {
      Wreck.get(`http://${ip}:9090/status`, {
-        timeout: 1000 // 1s
-      }, (err, res, payload) => {
+        timeout: 2000,    // 2 seconds
+        json: 'force'
+      }, (err, res, status) => {
        if (err) {
          this.emit('error', err);
          return next();
        }

-        if (Buffer.isBuffer(payload)) {
-          payload = payload.toString();
-        }
-
-        try {
-          const status = JSON.parse(payload);
-          next(null, status);
-        } catch (err) {
-          this.emit('error', err);
-          next();
-        }
+        next(null, status);
      });
    };

--- a/packages/portal-api/package.json
+++ b/packages/portal-api/package.json
@ -1,6 +1,6 @@
 {
  "name": "portal-api",
-  "version": "1.8.8",
+  "version": "1.8.9",
  "description": "",
  "main": "./lib/index.js",
  "scripts": {
@ -29,7 +29,7 @@
    "hapi": "^16.5.2",
    "lab": "^14.2.2",
    "lodash.findindex": "^4.6.0",
-    "wreck": "^12.4.0"
+    "wreck": "^12.5.0"
  },
  "dependencies": {
    "boom": "^5.2.0",