From bc679d6ac6daf144417548adc245d34fc0deaafb Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?S=C3=A9rgio=20Ramos?= <mail@sergioramos.me>
Date: Thu, 22 Jun 2017 18:09:13 +0100
Subject: [PATCH] feat(portal-api): integrate portal-watch (#510)

---
 local-compose.yml                             |   6 +
 packages/cp-frontend/package.json             |   1 +
 .../src/containers/services/list.js           |   1 +
 .../src/graphql/DeploymentGroupProvision.gql  |  18 +-
 packages/cp-frontend/src/state/selectors.js   |   3 +-
 packages/cp-gql-schema/schema.gql             |   5 +-
 packages/portal-api/lib/index.js              |  11 +-
 packages/portal-api/package.json              |   1 +
 packages/portal-api/server.js                 |  29 +
 packages/portal-data/lib/index.js             | 784 ++++++++++--------
 packages/portal-data/lib/transform.js         |  18 +-
 packages/portal-data/package.json             |   1 +
 packages/portal-data/test/index.js            |  16 +-
 packages/portal-watch/lib/index.js            | 103 ++-
 packages/portal-watch/test/index.js           |  85 +-
 yarn.lock                                     |  91 +-
 16 files changed, 705 insertions(+), 468 deletions(-)

diff --git a/local-compose.yml b/local-compose.yml
index a53b0b12..f5d1b793 100644
--- a/local-compose.yml
+++ b/local-compose.yml
@@ -101,6 +101,12 @@ api:
     - CONSUL=consul
     - PORT=3000
     - RETHINK_HOST=rethinkdb
+    - DOCKER_HOST=$DOCKER_HOST
+    - DOCKER_CERT_PATH=$DOCKER_CERT_PATH
+    - DOCKER_CLIENT_TIMEOUT=$DOCKER_CLIENT_TIMEOUT
+    - SDC_URL=$SDC_URL
+    - SDC_ACCOUNT=$SDC_ACCOUNT
+    - SDC_KEY_ID=$SDC_KEY_ID
   expose:
     - 3000
 
diff --git a/packages/cp-frontend/package.json b/packages/cp-frontend/package.json
index c5a20aea..ffad4fdf 100644
--- a/packages/cp-frontend/package.json
+++ b/packages/cp-frontend/package.json
@@ -22,6 +22,7 @@
     "apollo": "^0.2.2",
     "apr-intercept": "^1.0.4",
     "constant-case": "^2.0.0",
+    "force-array": "^3.1.0",
     "graphql-tag": "^2.4.0",
     "jest-cli": "^20.0.4",
     "joyent-manifest-editor": "^1.0.0",
diff --git a/packages/cp-frontend/src/containers/services/list.js b/packages/cp-frontend/src/containers/services/list.js
index 74f47e95..5c59d99a 100644
--- a/packages/cp-frontend/src/containers/services/list.js
+++ b/packages/cp-frontend/src/containers/services/list.js
@@ -138,6 +138,7 @@ const UiConnect = connect(mapStateToProps, mapDispatchToProps);
 const ServicesGql = graphql(ServicesQuery, {
   options(props) {
     return {
+      pollInterval: 1000,
       variables: {
         deploymentGroupSlug: props.match.params.deploymentGroup
       }
diff --git a/packages/cp-frontend/src/graphql/DeploymentGroupProvision.gql b/packages/cp-frontend/src/graphql/DeploymentGroupProvision.gql
index 40ee56a6..49bcc09d 100644
--- a/packages/cp-frontend/src/graphql/DeploymentGroupProvision.gql
+++ b/packages/cp-frontend/src/graphql/DeploymentGroupProvision.gql
@@ -1,9 +1,17 @@
 mutation provisionManifest($deploymentGroupId: ID!, $type: ManifestType!, $format: ManifestFormat!, $raw: String!) {
   provisionManifest(deploymentGroupId: $deploymentGroupId, type: $type, format: $format, raw: $raw) {
-    id
-    created
-    type
-    format
-    obj
+    manifestId
+    scale {
+      serviceName
+      replicas
+    }
+    plan {
+      running
+      actions {
+        type
+        service
+        machines
+      }
+    }
   }
 }
diff --git a/packages/cp-frontend/src/state/selectors.js b/packages/cp-frontend/src/state/selectors.js
index 31a3168c..e78d0df1 100644
--- a/packages/cp-frontend/src/state/selectors.js
+++ b/packages/cp-frontend/src/state/selectors.js
@@ -1,4 +1,5 @@
 import { createSelector } from 'reselect';
+import forceArray from 'force-array';
 
 const apollo = state => state.apollo;
 
@@ -76,7 +77,7 @@ const getService = (service, index, datacenter) => ({
 
 const processServices = (services, datacenter) => {
   console.log('services = ', services);
-  return services.reduce((ss, s, i) => {
+  return forceArray(services).reduce((ss, s, i) => {
     // Check whether it exits in thing, if so, add as child
     // if not, create and add as child
 
diff --git a/packages/cp-gql-schema/schema.gql b/packages/cp-gql-schema/schema.gql
index c747075d..1f5ef5be 100644
--- a/packages/cp-gql-schema/schema.gql
+++ b/packages/cp-gql-schema/schema.gql
@@ -53,7 +53,7 @@ type StateConvergencePlan {
 
 type Version {
   created: Date! # Either Int or define scalar
-  manifest: Manifest!
+  manifestId: ID!
   scale(serviceName: String): [ServiceScale]!
   plan(running: Boolean): StateConvergencePlan
 }
@@ -90,6 +90,7 @@ type Service {
   parent: ID # parent service id
   package: Package! # we don't have this in current mock data,
   environment: [Environment]
+  active: Boolean! # let's say that we update the manifest, and remove a service. we still want to track this service even though it might not exist because it might have machines running still
 }
 
 # for metrics max / min (I guess)
@@ -190,7 +191,7 @@ type Mutation {
   createDeploymentGroup(name: String!) : DeploymentGroup
   updateDeploymentGroup(id: ID!, name: String!) : DeploymentGroup
 
-  provisionManifest(deploymentGroupId: ID!, type: ManifestType!, format: ManifestFormat!, raw: String!) : Manifest
+  provisionManifest(deploymentGroupId: ID!, type: ManifestType!, format: ManifestFormat!, raw: String!) : Version
   scale(serviceId: ID!, replicas: Int!) : Version
 
   stopServices(ids: [ID]!) : [Service]
diff --git a/packages/portal-api/lib/index.js b/packages/portal-api/lib/index.js
index 4f049e05..07521aaf 100644
--- a/packages/portal-api/lib/index.js
+++ b/packages/portal-api/lib/index.js
@@ -4,6 +4,7 @@ const Schema = require('joyent-cp-gql-schema');
 const Graphi = require('graphi');
 const Piloted = require('piloted');
 const PortalData = require('portal-data');
+const PortalWatch = require('portal-watch');
 const Pack = require('../package.json');
 const Resolvers = require('./resolvers');
 
@@ -18,14 +19,22 @@ module.exports = function (server, options, next) {
   }
 
   const data = new PortalData(options.data);
+
+  const watch = new PortalWatch(Object.assign(options.watch, {
+    data
+  }));
+
   data.connect((err) => {
     if (err) {
       return next(err);
     }
 
-    server.bind(data);
+    server.bind(Object.assign(data, {
+      watch
+    }));
 
     Piloted.on('refresh', internals.refresh(data));
+    watch.poll();
 
     server.register([
       {
diff --git a/packages/portal-api/package.json b/packages/portal-api/package.json
index 981ab02e..09426686 100644
--- a/packages/portal-api/package.json
+++ b/packages/portal-api/package.json
@@ -37,6 +37,7 @@
     "joyent-cp-gql-schema": "^1.0.4",
     "piloted": "^3.1.1",
     "portal-data": "^1.1.0",
+    "portal-watch": "^1.0.0",
     "toppsy": "^1.1.0"
   }
 }
diff --git a/packages/portal-api/server.js b/packages/portal-api/server.js
index 0da8c73b..2bacd1de 100644
--- a/packages/portal-api/server.js
+++ b/packages/portal-api/server.js
@@ -9,6 +9,8 @@ const Toppsy = require('toppsy');
 const Vision = require('vision');
 const Pack = require('./package');
 const Portal = require('./lib');
+const Path = require('path');
+const Fs = require('fs');
 
 const server = new Hapi.Server();
 server.connection({ port: 3000 });
@@ -20,11 +22,38 @@ const swaggerOptions = {
   }
 };
 
+const {
+  DOCKER_HOST,
+  DOCKER_CERT_PATH,
+  DOCKER_CLIENT_TIMEOUT,
+  SDC_URL,
+  SDC_ACCOUNT,
+  SDC_KEY_ID
+} = process.env;
+
 const portalOptions = {
   data: {
     db: {
       host: process.env.RETHINK_HOST || 'localhost'
+    },
+    docker: {
+      host: DOCKER_HOST,
+      ca: DOCKER_CERT_PATH ?
+        Fs.readFileSync(Path.join(DOCKER_CERT_PATH, 'ca.pem')) :
+        undefined,
+      cert: DOCKER_CERT_PATH ?
+        Fs.readFileSync(Path.join(DOCKER_CERT_PATH, 'cert.pem')) :
+        undefined,
+      key: DOCKER_CERT_PATH ?
+        Fs.readFileSync(Path.join(DOCKER_CERT_PATH, 'key.pem')) :
+        undefined,
+      timeout: DOCKER_CLIENT_TIMEOUT
     }
+  },
+  watch: {
+    url: SDC_URL,
+    account: SDC_ACCOUNT,
+    keyId: SDC_KEY_ID
   }
 };
 
diff --git a/packages/portal-data/lib/index.js b/packages/portal-data/lib/index.js
index 07796859..59edb5e8 100644
--- a/packages/portal-data/lib/index.js
+++ b/packages/portal-data/lib/index.js
@@ -1,5 +1,6 @@
 'use strict';
 
+const ParamCase = require('param-case');
 const EventEmitter = require('events');
 const DockerClient = require('docker-compose-client');
 const Dockerode = require('dockerode');
@@ -30,9 +31,20 @@ const internals = {
   }
 };
 
+const resolveCb = (resolve, reject) => {
+  return (err, ...args) => {
+    if (err) {
+      return reject(err);
+    }
+
+    resolve(...args);
+  };
+};
+
 module.exports = class Data extends EventEmitter {
   constructor (options) {
     super();
+
     const settings = Hoek.applyToDefaults(internals.defaults, options || {});
 
     // Penseur will assert that the options are correct
@@ -74,41 +86,35 @@ module.exports = class Data extends EventEmitter {
         return cb();
       }
 
-      const portal = portals[0];
-      VAsync.parallel({
-        funcs: [
-          (next) => {
-            this.getDatacenter({ id: portal.datacenter_id }, next);
-          },
-          (next) => {
-            this.getUser({}, next);
-          }
-        ]
-      }, (err, results) => {
-        if (err) {
-          return cb(err);
-        }
+      const portal = portals.shift();
 
-        // Sub query/filter for deploymentGroups
-        const deploymentGroups = (args) => {
-          return new Promise((resolve, reject) => {
-            this.getDeploymentGroups(args, (err, groups) => {
-              if (err) {
-                return reject(err);
-              }
+      // Sub query/filter for deploymentGroups
+      const deploymentGroups = (args) => {
+        return new Promise((resolve, reject) => {
+          this.getDeploymentGroups(args, resolveCb(resolve, reject));
+        });
+      };
 
-              resolve(groups);
-            });
-          });
-        };
+      // Sub query/filter for user
+      const user = () => {
+        return new Promise((resolve, reject) => {
+          this.getUser({}, resolveCb(resolve, reject));
+        });
+      };
 
-        cb(null, Transform.fromPortal({
-          portal,
-          deploymentGroups,
-          datacenter: results.operations[0].result,
-          user: results.operations[1].result
-        }));
-      });
+      // Sub query/filter for datacenter
+      const datacenter = () => {
+        return new Promise((resolve, reject) => {
+          this.getDatacenter({ id: portal.datacenter_id }, resolveCb(resolve, reject));
+        });
+      };
+
+      cb(null, Transform.fromPortal({
+        portal,
+        deploymentGroups,
+        datacenter,
+        user
+      }));
     });
   }
 
@@ -179,26 +185,31 @@ module.exports = class Data extends EventEmitter {
   // deployment_groups
 
   createDeploymentGroup (clientDeploymentGroup, cb) {
-    // trigger deployment
-    // create deployment queue (we should think about what is a deployment queue)
-    // create the ConvergencePlans
-    // create a DeploymentPlan
-    // create a Version
-    // update the DeploymentGroup
-
     const deploymentGroup = Transform.toDeploymentGroup(clientDeploymentGroup);
-    this._db.deployment_groups.insert(deploymentGroup, (err, key) => {
+    this._db.deployment_groups.query({
+      slug: deploymentGroup.slug
+    }, (err, deploymentGroups) => {
       if (err) {
         return cb(err);
       }
 
-      deploymentGroup.id = key;
-      cb(null, Transform.fromDeploymentGroup(deploymentGroup));
+      if (deploymentGroups && deploymentGroups.length) {
+        return cb(new Error(`DeploymentGroup "${deploymentGroup.slug}" already exists (${deploymentGroups[0].id})`));
+      }
+
+      this._db.deployment_groups.insert(deploymentGroup, (err, key) => {
+        if (err) {
+          return cb(err);
+        }
+
+        deploymentGroup.id = key;
+        cb(null, Transform.fromDeploymentGroup(deploymentGroup));
+      });
     });
   }
 
   updateDeploymentGroup ({ id, name }, cb) {
-    this._db.deployment_groups.update(id, { name }, (err) => {
+    this._db.deployment_groups.update([{ id, name }], (err) => {
       if (err) {
         return cb(err);
       }
@@ -213,24 +224,22 @@ module.exports = class Data extends EventEmitter {
         return cb(err);
       }
 
-      const getServices = (service_ids) => {
+      const getServices = (deploymentGroupId) => {
         return (args) => {
           args = args || {};
-          args.ids = service_ids;
-          return new Promise((resolve, reject) => {
-            this.getServices(args, (err, services) => {
-              if (err) {
-                return reject(err);
-              }
+          args.deploymentGroupId = deploymentGroupId;
 
-              resolve(services);
-            });
+          return new Promise((resolve, reject) => {
+            this.getServices(args, resolveCb(resolve, reject));
           });
         };
       };
 
+      // todo getVersion
+      // todo getHistory
+
       const convertedGroups = deploymentGroups ? deploymentGroups.map((deploymentGroup) => {
-        return Transform.fromDeploymentGroup(deploymentGroup, getServices(deploymentGroup.service_ids));
+        return Transform.fromDeploymentGroup(deploymentGroup, getServices(deploymentGroup.id));
       }) : [];
 
       cb(null, convertedGroups);
@@ -252,6 +261,8 @@ module.exports = class Data extends EventEmitter {
   }
 
   getDeploymentGroup (query, cb) {
+    query = query || {};
+
     this._db.deployment_groups.query(query, (err, deploymentGroups) => {
       if (err) {
         return cb(err);
@@ -262,24 +273,21 @@ module.exports = class Data extends EventEmitter {
       }
 
       const deploymentGroup = deploymentGroups[0];
-      if (!deploymentGroup.service_ids || !deploymentGroup.service_ids.length) {
-        return cb(null, Transform.fromDeploymentGroup(deploymentGroup));
-      }
 
       const getServices = (args) => {
+        console.log(args);
         args = args || {};
-        args.ids = deploymentGroup.service_ids;
-        return new Promise((resolve, reject) => {
-          this.getServices(args, (err, services) => {
-            if (err) {
-              return reject(err);
-            }
+        args.deploymentGroupId = deploymentGroup.id;
+        console.log(args);
 
-            resolve(services || []);
-          });
+        return new Promise((resolve, reject) => {
+          this.getServices(args, resolveCb(resolve, reject));
         });
       };
 
+      // todo getVersion
+      // todo getHistory
+
       cb(err, Transform.fromDeploymentGroup(deploymentGroup, getServices));
     });
   }
@@ -292,12 +300,16 @@ module.exports = class Data extends EventEmitter {
     Hoek.assert(clientVersion.manifestId, 'manifestId is required');
     Hoek.assert(clientVersion.deploymentGroupId, 'deploymentGroupId is required');
 
+    console.log(`-> creating new Version for DeploymentGroup ${clientVersion.deploymentGroupId}`);
+
     const version = Transform.toVersion(clientVersion);
     this._db.versions.insert(version, (err, key) => {
       if (err) {
         return cb(err);
       }
 
+      console.log(`-> new Version for DeploymentGroup ${clientVersion.deploymentGroupId} created: ${key}`);
+
       const changes = {
         id: clientVersion.deploymentGroupId,
         version_id: key,
@@ -308,6 +320,8 @@ module.exports = class Data extends EventEmitter {
         changes['service_ids'] = clientVersion.serviceIds;
       }
 
+      console.log(`-> updating DeploymentGroup ${clientVersion.deploymentGroupId} to add Version ${key}`);
+
       this._db.deployment_groups.update([changes], (err) => {
         if (err) {
           return cb(err);
@@ -319,6 +333,16 @@ module.exports = class Data extends EventEmitter {
     });
   }
 
+  updateVersion (clientVersion, cb) {
+    this._db.versions.update([Transform.toVersion(clientVersion)], (err, versions) => {
+      if (err) {
+        return cb(err);
+      }
+
+      cb(null, versions && versions.length ? Transform.fromVersion(versions[0]) : {});
+    });
+  }
+
   getVersion ({ id, manifestId }, cb) {
     const query = id ? { id } : { manifest_id: manifestId };
     this._db.versions.single(query, (err, version) => {
@@ -360,11 +384,14 @@ module.exports = class Data extends EventEmitter {
     Hoek.assert(id, 'service id is required');
     Hoek.assert(typeof replicas === 'number' && replicas >= 0, 'replicas must be a number no less than 0');
 
-
     // get the service then get the deployment group
     // use the deployment group to find the current version and manifest
     // scale the service
-    // update the machine ids and instances
+    // maybe update the machine ids and instances
+
+    console.log('-> scale request received');
+
+    console.log(`-> fetching Service ${id}`);
 
     this._db.services.single({ id }, (err, service) => {
       if (err) {
@@ -375,6 +402,8 @@ module.exports = class Data extends EventEmitter {
         return cb(new Error(`service not found for id: ${id}`));
       }
 
+      console.log(`-> fetching DeploymentGroup ${service.deployment_group_id}`);
+
       this._db.deployment_groups.single({ id: service.deployment_group_id }, (err, deployment_group) => {
         if (err) {
           return cb(err);
@@ -384,6 +413,8 @@ module.exports = class Data extends EventEmitter {
           return cb(new Error(`deployment group not found for service with service id: ${id}`));
         }
 
+        console.log(`-> fetching Version ${deployment_group.version_id}`);
+
         this._db.versions.single({ id: deployment_group.version_id }, (err, version) => {
           if (err) {
             return cb(err);
@@ -393,6 +424,8 @@ module.exports = class Data extends EventEmitter {
             return cb(new Error(`version not found for service with service id: ${id}`));
           }
 
+          console.log(`-> fetching Manifest ${version.manifest_id}`);
+
           this._db.manifests.single({ id: version.manifest_id }, (err, manifest) => {
             if (err) {
               return cb(err);
@@ -411,92 +444,62 @@ module.exports = class Data extends EventEmitter {
 
   _scale ({ service, deployment_group, version, manifest, replicas }, cb) {
     let isFinished = false;
+
     const finish = () => {
       if (isFinished) {
         return;
       }
 
       isFinished = true;
-      const machineIds = [];
-      for (let i = 1; i <= replicas; ++i) {
-        machineIds.push(`${deployment_group.name}_${service.name}_${i}`);
+
+      console.log(`-> docker-compose scaled "${service.name}" from DeploymentGroup ${deployment_group.id} to ${replicas} replicas`);
+
+      if (!version.service_scales || !version.service_scales.length) {
+        console.log(`-> no scale data found for service "${service.name}" from DeploymentGroup ${deployment_group.id} in current Version (${version.id})`);
+
+        version.service_scales = [{
+          service_name: service.name
+        }];
       }
 
-      this._db.instances.remove(service.instance_ids, (err) => {
-        // emit error instead of returning early, this is a best effort to cleanup data
-        if (err) {
-          this.emit('error', err);
-        }
-
-        VAsync.forEachParallel({
-          func: (machineId, next) => {
-            const clientInstance = {
-              machineId,
-              status: 'CREATED',
-              name: service.name
-            };
-            this.createInstance(clientInstance, next);
-          },
-          inputs: machineIds
-        }, (err, results) => {
-          if (err) {
-            return cb(err);
+      const clientVersion = {
+        deploymentGroupId: deployment_group.id,
+        manifestId: manifest.id,
+        plan: version.plan,
+        scale: version.service_scales.map((scale) => {
+          if (scale.service_name !== service.name) {
+            return scale;
           }
 
-          const instanceIds = results.successes.map((instance) => {
-            return instance.id;
-          });
+          return {
+            serviceName: service.name,
+            replicas
+          };
+        })
+      };
 
-          this._db.services.update(service.id, { instance_ids: instanceIds }, (err) => {
-            if (err) {
-              return cb(err);
-            }
+      console.log(`-> creating new Version for DeploymentGroup ${deployment_group.id}`);
 
-            const clientVersion = {
-              deploymentGroupId: deployment_group.id,
-              manifestId: manifest.id,
-              plan: {
-                running: true,
-                actions: [{
-                  type: 'CREATE',
-                  service: service.name,
-                  machines: machineIds
-                }]
-              }
-            };
-
-            const scale = version.service_scales.find((scale) => {
-              return scale.service_name === service.name;
-            });
-
-            if (scale) {
-              scale.replicas = replicas;
-            } else {
-              version.service_scales.push({
-                service_name: service.name,
-                replicas
-              });
-            }
-
-            clientVersion.scales = version.service_scales.map(Transform.fromScale);
-
-            this.createVersion(clientVersion, cb);
-          });
-        });
+      // createVersion updates the deployment group
+      this.createVersion(clientVersion, (...args) => {
+        isFinished = true;
+        cb(...args);
       });
     };
 
-    const options = {
+    console.log(`-> requesting docker-compose to scale "${service.name}" from DeploymentGroup ${deployment_group.id} to ${replicas} replicas`);
+
+    this._dockerCompose.scale({
       projectName: deployment_group.name,
-      services: {},
+      services: {
+        [service.name]: replicas
+      },
       manifest: manifest.raw
-    };
-    options.services[service.name] = replicas;
-    this._dockerCompose.scale(options, (err, res) => {
+    }, (err, res) => {
       if (err) {
         return cb(err);
       }
-      console.log(JSON.stringify(res, null, '  '));
+
       finish();
     });
   }
@@ -505,12 +508,117 @@ module.exports = class Data extends EventEmitter {
   // manifests
 
   provisionManifest (clientManifest, cb) {
-    // get deployment group to verify it exists and get the name
-    // insert manifest
-    // callback with manifest
-    // provision containers and save service data
+    // 1. check that the deploymentgroup exists
+    // 2. create a new manifest
+    // 3. create a new version
+    // 4. return said version
+    // 5. request docker-compose-api to provision manifest
+    // 6. create/update/prune services by calling provisionServices with the respose from docker-compose-api
+    // 7. update version with the provision plan and new service ids
 
-    this.getDeploymentGroup({ id: clientManifest.deploymentGroupId }, (err, deploymentGroup) => {
+    // todo we are not doing anything with the action plans right now
+    // but if we were, we would do that in portal-watch. with that said, we might
+    // run into a race condition where the event happens before we update the
+    // new version with the plan
+
+    console.log('-> provision request received');
+
+    const provision = ({ deploymentGroup, manifestId, newVersion }) => {
+      let isHandled = false;
+
+      console.log(`-> requesting docker-compose provision for DeploymentGroup ${deploymentGroup.name}`);
+
+      this._dockerCompose.provision({
+        projectName: deploymentGroup.name,
+        manifest: clientManifest.raw
+      }, (err, provisionRes) => {
+        if (err) {
+          this.emit('error', err);
+          return;
+        }
+
+        // callback can execute multiple times, ensure responses are only handled once
+        if (isHandled) {
+          return;
+        }
+
+        isHandled = true;
+
+        console.log('-> update/create/remove services based on response from docker-compose');
+
+        // create/update services based on hashes
+        // return the new set of service ids
+        this.provisionServices({
+          deploymentGroup,
+          manifestId,
+          provisionRes
+        }, (err, newServiceIds) => {
+          if (err) {
+            this.emit('error', err);
+            return;
+          }
+
+          console.log(`-> update Version ${newVersion.id} based on docker-compose response and new service ids`);
+
+          const actions = Object.keys(provisionRes).map((serviceName) => {
+            return ({
+              type: provisionRes[serviceName].plan.action,
+              service: serviceName,
+              machines: provisionRes[serviceName].plan.containers.map(({ id }) => { return id; })
+            });
+          });
+
+          // create new version
+          this.updateVersion({
+            id: newVersion.id,
+            manifestId,
+            newServiceIds,
+            plan: {
+              running: true,
+              actions: actions
+            }
+          }, (err) => {
+            if (err) {
+              this.emit('error', err);
+              return;
+            }
+
+            console.log(`-> updated Version ${newVersion.id}`);
+            console.log('-> provisionManifest DONE');
+          });
+        });
+      });
+    };
+
+    const createVersion = ({ deploymentGroup, currentVersion, manifestId }) => {
+      // create new version
+      this.createVersion({
+        manifestId,
+        deploymentGroupId: deploymentGroup.id,
+        scale: currentVersion.scale,
+        plan: {
+          running: true,
+          actions: []
+        }
+      }, (err, newVersion) => {
+        if (err) {
+          return cb(err);
+        }
+
+        console.log(`-> new Version created with id ${newVersion.id}`);
+        console.log('newVersion', newVersion);
+
+        setImmediate(() => {
+          provision({ deploymentGroup, manifestId, newVersion });
+        });
+
+        cb(null, newVersion);
+      });
+    };
+
+    this.getDeploymentGroup({
+      id: clientManifest.deploymentGroupId
+    }, (err, deploymentGroup) => {
       if (err) {
         return cb(err);
       }
@@ -519,38 +627,41 @@ module.exports = class Data extends EventEmitter {
         return cb(new Error('Deployment group not found for manifest'));
       }
 
-      const manifest = Transform.toManifest(clientManifest);
-      this._db.manifests.insert(manifest, (err, key) => {
+      console.log(`-> new DeploymentGroup created with id ${deploymentGroup.id}`);
+
+      const newManifest = Transform.toManifest(clientManifest);
+      this._db.manifests.insert(newManifest, (err, manifestId) => {
         if (err) {
           return cb(err);
         }
 
-        setImmediate(() => {
-          let isHandled = false;
-          this._dockerCompose.provision({ projectName: deploymentGroup.name, manifest: clientManifest.raw }, (err, res) => {
-            if (err) {
-              this.emit('error', err);
-              return;
-            }
+        console.log(`-> new Manifest created with id ${manifestId}`);
 
-            // callback can execute multiple times, ensure responses are only handled once
-            if (isHandled) {
-              return;
-            }
+        if (!deploymentGroup.version) {
+          console.log(`-> detected first provision for DeploymentGroup ${deploymentGroup.id}`);
+          return createVersion({
+            deploymentGroup,
+            manifestId,
+            currentVersion: {}
+          });
+        }
 
-            isHandled = true;
-            const options = {
-              manifestServices: manifest.json.services || manifest.json,
-              deploymentGroup,
-              manifestId: key,
-              provisionRes: res
-            };
-            this.provisionServices(options);
+        // get current version for because we need current scale
+        this.getVersion({
+          id: deploymentGroup.version
+        }, (err, currentVersion) => {
+          if (err) {
+            return cb(err);
+          }
+
+          console.log(`-> creating new Version based on old version ${currentVersion.id}`);
+
+          return createVersion({
+            deploymentGroup,
+            manifestId,
+            currentVersion
           });
         });
-
-        manifest.id = key;
-        cb(null, Transform.fromManifest(manifest));
       });
     });
   }
@@ -580,108 +691,142 @@ module.exports = class Data extends EventEmitter {
 
   // services
 
-  provisionServices ({ manifestServices, deploymentGroup, manifestId, provisionRes }, cb) {
-    // insert instance information
-    // insert service information
-    // insert version information -- will update deploymentGroups
+  provisionServices ({ deploymentGroup, manifestId, provisionRes }, cb) {
+    // 1. get current set of services
+    // 2. compare names and hashes
+    // 3. if name doesn't exist anymore, disable service
+    // 4. if hash is new, update service
+    // 5. compare previous services with new ones
+    // 6. deactivate pruned ones
 
-    cb = cb || ((err) => {
-      if (err) {
-        this.emit('error', err);
-      }
-    });
+    console.log('-> provision services in our data layer');
 
-    VAsync.forEachPipeline({
-      func: (serviceName, next) => {
-        const manifestService = manifestServices[serviceName];
-        const container = provisionRes[serviceName].plan.containers[0];
+    const createService = ({ provision, serviceName }, cb) => {
+      console.log(`-> creating Service "${serviceName}" from DeploymentGroup ${deploymentGroup.id}`);
 
-        const clientInstance = {
-          name: serviceName,
-          machineId: container ? container.id : `${deploymentGroup.name}_${serviceName}_1`,
-          status: 'CREATED'
-        };
-        this.createInstance(clientInstance, (err, createdInstance) => {
-          if (err) {
-            return next(err);
-          }
-
-          const clientService = {
-            hash: manifestService.image,
-            name: serviceName,
-            slug: serviceName,
-            deploymentGroupId: deploymentGroup.id,
-            instances: [createdInstance],
-            info: provisionRes[serviceName]
-          };
-
-          this.createService(clientService, (err, createdService) => {
-            if (err) {
-              return next(err);
-            }
-
-            return next(null, {
-              action: {
-                type: 'CREATE',
-                service: serviceName,
-                machines: [createdInstance.machineId]
-              },
-              serviceId: createdService.id,
-              scale: {
-                serviceName,
-                replicas: 1
-              }
-            });
-          });
-        });
-      },
-      inputs: Object.keys(manifestServices)
-    }, (err, results) => {
-      if (err) {
-        return cb(err);
-      }
-      const successes = results.successes;
-      if (!successes || !successes.length) {
-        return cb();
-      }
-
-      const scales = successes.map((result) => {
-        return result.scale;
-      });
-
-      const actions = successes.map((result) => {
-        return result.action;
-      });
-
-      const serviceIds = successes.map((result) => {
-        return result.serviceId;
-      });
-
-      const plan = {
-        running: true,
-        actions
-      };
-
-      const clientVersion = {
+      this.createService({
+        hash: provision.hash,
         deploymentGroupId: deploymentGroup.id,
-        manifestId,
-        scales,
-        plan,
-        serviceIds
-      };
-
-      this.createVersion(clientVersion, (err, version) => {
+        name: serviceName,
+        slug: ParamCase(serviceName)
+      }, (err, service) => {
         if (err) {
           return cb(err);
         }
 
-        cb(null, version);
+        cb(null, service.id);
       });
-    });
+    };
+
+    const updateService = ({ provision, service }, cb) => {
+      console.log(`-> updating Service "${service.name}" from DeploymentGroup ${deploymentGroup.id}`);
+
+      this.updateService({
+        id: service.id,
+        hash: provision.hash
+      }, (err) => {
+        if (err) {
+          return cb(err);
+        }
+
+        cb(null, service.id);
+      });
+    };
+
+    const resolveService = (serviceName, next) => {
+      console.log(`-> fetching Service "${serviceName}" from DeploymentGroup ${deploymentGroup.id}`);
+
+      const provision = provisionRes[serviceName];
+
+      this.getServices({
+        name: serviceName,
+        deploymentGroupId: deploymentGroup.id
+      }, (err, services) => {
+        if (err) {
+          return cb(err);
+        }
+
+        // no services for given name
+        if (!services || !services.length) {
+          return createService({ provision, serviceName }, next);
+        }
+
+        const service = services.shift();
+
+        VAsync.forEachPipeline({
+          inputs: services,
+          // disable old services
+          func: ({ id }, next) => {
+            console.log(`-> deactivating Service ${id} from DeploymentGroup ${deploymentGroup.id}`);
+            this.updateService({ active: false, id }, next);
+          }
+        }, (err) => {
+          if (err) {
+            return cb(err);
+          }
+
+          // service changed
+          if (service.hash !== provision.hash) {
+            return updateService({ provision, service }, next);
+          }
+
+          console.log(`-> no changes for Service "${serviceName}" from DeploymentGroup ${deploymentGroup.id}`);
+          return next(null, service.id);
+        });
+      });
+    };
+
+    const pruneService = ({ id, instances }, next) => {
+      // if it has instances, just mark as inactive
+      console.log(`-> pruning Service ${id} from DeploymentGroup ${deploymentGroup.id}`);
+
+      const update = () => { return this.updateService({ active: false, id }, next); };
+      const remove = () => { return this.deleteServices({ ids: [id] }, next); };
+
+      return (instances && instances.length) ?
+        update() :
+        remove();
+    };
+
+    // deactivate pruned servcies
+    const pruneServices = (err, result) => {
+      if (err) {
+        return cb(err);
+      }
+
+      console.log(`-> pruning Services from DeploymentGroup ${deploymentGroup.id}`);
+
+      const new_service_ids = result.successes;
+
+      this.getServices({
+        deploymentGroupId: deploymentGroup.id
+      }, (err, oldServices) => {
+        if (err) {
+          return cb(err);
+        }
+
+        const servicesToPrune = oldServices
+          .filter(({ id }) => { return new_service_ids.indexOf(id) < 0; });
+
+        VAsync.forEachPipeline({
+          inputs: servicesToPrune,
+          func: pruneService
+        }, (err) => { return cb(err, new_service_ids); });
+      });
+    };
+
+    VAsync.forEachPipeline({
+      inputs: Object.keys(provisionRes),
+      func: resolveService
+    }, pruneServices);
   }
 
   createService (clientService, cb) {
-    this._db.services.insert(Transform.toService(clientService), (err, key) => {
+    const newService = Object.assign(Transform.toService(clientService), {
+      active: true
+    });
+
+    this._db.services.insert(newService, (err, key) => {
       if (err) {
         return cb(err);
       }
@@ -691,6 +836,16 @@ module.exports = class Data extends EventEmitter {
     });
   }
 
+  updateService (clientService, cb) {
+    this._db.services.update([Transform.toService(clientService)], (err, services) => {
+      if (err) {
+        return cb(err);
+      }
+
+      cb(null, services && services.length ? Transform.fromService(services[0]) : {});
+    });
+  }
+
   getService ({ id, hash }, cb) {
     const query = id ? { id } : { version_hash: hash };
     this._db.services.query(query, (err, service) => {
@@ -769,16 +924,12 @@ module.exports = class Data extends EventEmitter {
 
   _instancesFilter (instanceIds) {
     return (query) => {
+      query = query || {};
+
       return new Promise((resolve, reject) => {
         query.ids = instanceIds;
 
-        this.getInstances(query, (err, instances) => {
-          if (err) {
-            return reject(err);
-          }
-
-          resolve(instances);
-        });
+        this.getInstances(query, resolveCb(resolve, reject));
       });
     };
   }
@@ -793,10 +944,9 @@ module.exports = class Data extends EventEmitter {
         return cb();
       }
 
-      let instanceIds = [];
-      services.forEach((service) => {
-        instanceIds = instanceIds.concat(service.instance_ids);
-      });
+      const instanceIds = services.reduce((instanceIds, service) => {
+        return instanceIds.concat(service.instance_ids);
+      }, []);
 
       VAsync.forEachParallel({
         func: (instanceId, next) => {
@@ -806,14 +956,7 @@ module.exports = class Data extends EventEmitter {
             }
 
             const container = this._docker.getContainer(instance.machine_id);
-
-            container.stop((err) => {
-              if (err) {
-                return next(err);
-              }
-
-              this.updateInstance({ id: instance.id, status: 'STOPPED' }, next);
-            });
+            container.stop(next);
           });
         },
         inputs: instanceIds
@@ -837,10 +980,9 @@ module.exports = class Data extends EventEmitter {
         return cb();
       }
 
-      let instanceIds = [];
-      services.forEach((service) => {
-        instanceIds = instanceIds.concat(service.instance_ids);
-      });
+      const instanceIds = services.reduce((instanceIds, service) => {
+        return instanceIds.concat(service.instance_ids);
+      }, []);
 
       VAsync.forEachParallel({
         func: (instanceId, next) => {
@@ -850,14 +992,7 @@ module.exports = class Data extends EventEmitter {
             }
 
             const container = this._docker.getContainer(instance.machine_id);
-
-            container.start((err) => {
-              if (err) {
-                return next(err);
-              }
-
-              this.updateInstance({ id: instance.id, status: 'RUNNING' }, next);
-            });
+            container.start(next);
           });
         },
         inputs: instanceIds
@@ -881,10 +1016,9 @@ module.exports = class Data extends EventEmitter {
         return cb();
       }
 
-      let instanceIds = [];
-      services.forEach((service) => {
-        instanceIds = instanceIds.concat(service.instance_ids);
-      });
+      const instanceIds = services.reduce((instanceIds, service) => {
+        return instanceIds.concat(service.instance_ids);
+      }, []);
 
       VAsync.forEachParallel({
         func: (instanceId, next) => {
@@ -893,17 +1027,8 @@ module.exports = class Data extends EventEmitter {
               return next(err);
             }
 
-            this.updateInstance({ id: instance.id, status: 'RESTARTING' }, () => {
-              const container = this._docker.getContainer(instance.machine_id);
-
-              container.restart((err) => {
-                if (err) {
-                  return next(err);
-                }
-
-                this.updateInstance({ id: instance.id, status: 'RUNNING' }, next);
-              });
-            });
+            const container = this._docker.getContainer(instance.machine_id);
+            container.restart(next);
           });
         },
         inputs: instanceIds
@@ -918,6 +1043,8 @@ module.exports = class Data extends EventEmitter {
   }
 
   deleteServices ({ ids }, cb) {
+    // todo could this be done with scale = 0?
+
     this._db.services.get(ids, (err, services) => {
       if (err) {
         return cb(err);
@@ -927,10 +1054,9 @@ module.exports = class Data extends EventEmitter {
         return cb();
       }
 
-      let instanceIds = [];
-      services.forEach((service) => {
-        instanceIds = instanceIds.concat(service.instance_ids);
-      });
+      const instanceIds = services.reduce((instanceIds, service) => {
+        return instanceIds.concat(service.instance_ids);
+      }, []);
 
       VAsync.forEachParallel({
         func: (instanceId, next) => {
@@ -940,15 +1066,8 @@ module.exports = class Data extends EventEmitter {
             }
 
             const container = this._docker.getContainer(instance.machine_id);
-
             // Use force in case the container is running. TODO: should we keep force?
-            container.remove({ force: true }, (err) => {
-              if (err) {
-                return next(err);
-              }
-
-              this.updateInstance({ id: instance.id, status: 'DELETED' }, next);
-            });
+            container.remove({ force: true }, next);
           });
         },
         inputs: instanceIds
@@ -957,7 +1076,21 @@ module.exports = class Data extends EventEmitter {
           return cb(err);
         }
 
-        this.getServices({ ids }, cb);
+        VAsync.forEachParallel({
+          inputs: ids,
+          func: (serviceId, next) => {
+            this.updateService({
+              id: serviceId,
+              active: false
+            });
+          }
+        }, (err) => {
+          if (err) {
+            return cb(err);
+          }
+
+          this.getServices({ ids }, cb);
+        });
       });
     });
   }
@@ -1019,12 +1152,12 @@ module.exports = class Data extends EventEmitter {
   }
 
   updateInstance ({ id, status }, cb) {
-    this._db.instances.update(id, { status }, (err, instance) => {
+    this._db.instances.update([{ id, status }], (err, instances) => {
       if (err) {
         return cb(err);
       }
 
-      cb(null, instance ? Transform.fromInstance(instance) : {});
+      cb(null, instances && instances.length ? Transform.fromInstance(instances[0]) : {});
     });
   }
 
@@ -1041,14 +1174,7 @@ module.exports = class Data extends EventEmitter {
       VAsync.forEachParallel({
         func: (instance, next) => {
           const container = this._docker.getContainer(instance.machine_id);
-
-          container.stop((err) => {
-            if (err) {
-              return next(err);
-            }
-
-            this.updateInstance({ id: instance.id, status: 'STOPPED' }, next);
-          });
+          container.stop(next);
         },
         inputs: instances
       }, (err, results) => {
@@ -1074,14 +1200,7 @@ module.exports = class Data extends EventEmitter {
       VAsync.forEachParallel({
         func: (instance, next) => {
           const container = this._docker.getContainer(instance.machine_id);
-
-          container.start((err) => {
-            if (err) {
-              return next(err);
-            }
-
-            this.updateInstance({ id: instance.id, status: 'RUNNING' }, next);
-          });
+          container.start(next);
         },
         inputs: instances
       }, (err, results) => {
@@ -1108,14 +1227,7 @@ module.exports = class Data extends EventEmitter {
         func: (instance, next) => {
           this.updateInstance({ id: instance.id, status: 'RESTARTING' }, () => {
             const container = this._docker.getContainer(instance.machine_id);
-
-            container.restart((err) => {
-              if (err) {
-                return next(err);
-              }
-
-              this.updateInstance({ id: instance.id, status: 'RUNNING' }, next);
-            });
+            container.restart(next);
           });
         },
         inputs: instances
diff --git a/packages/portal-data/lib/transform.js b/packages/portal-data/lib/transform.js
index df25fddd..f757492a 100644
--- a/packages/portal-data/lib/transform.js
+++ b/packages/portal-data/lib/transform.js
@@ -38,7 +38,7 @@ exports.toDeploymentGroup = function ({ name }) {
   return {
     name,
     slug: name,
-    services: [],
+    service_ids: [],
     version_id: '',
     history_version_ids: []
   };
@@ -57,12 +57,15 @@ exports.fromService = function ({ service, instances, packages }) {
     currentMetrics: [],
     connections: service.service_dependency_ids,
     package: packages ? exports.fromPackage(packages) : {},
-    parent: service.parent_id || ''
+    parent: service.parent_id || '',
+    active: service.active
   };
 };
 
 exports.toService = function (clientService) {
-  return {
+  // wat??
+  return JSON.parse(JSON.stringify({
+    id: clientService.id,
     version_hash: clientService.hash || clientService.name,
     deployment_group_id: clientService.deploymentGroupId,
     name: clientService.name,
@@ -71,8 +74,9 @@ exports.toService = function (clientService) {
     instance_ids: clientService.instances ? clientService.instances.map((instance) => { return instance.id; }) : [],
     service_dependency_ids: clientService.connections || [],
     package_id: clientService.package ? clientService.package.id : '',
-    parent_id: clientService.parent || ''
-  };
+    parent_id: clientService.parent || '',
+    active: clientService.ative
+  }));
 };
 
 
@@ -81,7 +85,7 @@ exports.toVersion = function (clientVersion) {
     id: clientVersion.id,
     created: clientVersion.created || Date.now(),
     manifest_id: clientVersion.manifestId,
-    service_scales: clientVersion.scales ? clientVersion.scales.map(exports.toScale) : [],
+    service_scales: clientVersion.scale ? clientVersion.scale.map(exports.toScale) : [],
     plan: exports.toPlan(clientVersion.plan || {})
   };
 };
@@ -91,7 +95,7 @@ exports.fromVersion = function (version) {
     id: version.id,
     created: version.created,
     manifestId: version.manifest_id,
-    scales: version.service_scales ? version.service_scales.map(exports.fromScale) : [],
+    scale: version.service_scales ? version.service_scales.map(exports.fromScale) : [],
     plan: exports.fromPlan(version.plan || {})
   };
 };
diff --git a/packages/portal-data/package.json b/packages/portal-data/package.json
index ce762bdb..347e5473 100644
--- a/packages/portal-data/package.json
+++ b/packages/portal-data/package.json
@@ -19,6 +19,7 @@
     "docker-compose-client": "^1.0.8",
     "dockerode": "^2.5.0",
     "hoek": "^4.1.1",
+    "param-case": "^2.1.1",
     "penseur": "^7.12.3",
     "vasync": "^1.6.4",
     "yamljs": "^0.2.10"
diff --git a/packages/portal-data/test/index.js b/packages/portal-data/test/index.js
index f04cd017..e3e93f05 100644
--- a/packages/portal-data/test/index.js
+++ b/packages/portal-data/test/index.js
@@ -253,7 +253,7 @@ describe('versions', () => {
             const clientVersion = {
               deploymentGroupId: deploymentGroup.id,
               manifestId: manifest.id,
-              scales: [{
+              scale: [{
                 serviceName: 'consul',
                 replicas: 3
               }],
@@ -270,7 +270,7 @@ describe('versions', () => {
             data.createVersion(clientVersion, (err, result) => {
               expect(err).to.not.exist();
               expect(result.id).to.exist();
-              expect(result.scales).to.equal(clientVersion.scales);
+              expect(result.scale).to.equal(clientVersion.scale);
               done();
             });
           });
@@ -299,7 +299,7 @@ describe('versions', () => {
             const clientVersion = {
               manifestId: manifest.id,
               deploymentGroupId: deploymentGroup.id,
-              scales: [{
+              scale: [{
                 serviceName: 'consul',
                 replicas: 3
               }],
@@ -316,7 +316,7 @@ describe('versions', () => {
             data.createVersion(clientVersion, (err, result) => {
               expect(err).to.not.exist();
               expect(result.id).to.exist();
-              expect(result.scales).to.equal(clientVersion.scales);
+              expect(result.scale).to.equal(clientVersion.scale);
               data.getVersion({ id: result.id }, (err, retrievedVersion1) => {
                 expect(err).to.not.exist();
                 expect(retrievedVersion1.id).to.equal(result.id);
@@ -353,7 +353,7 @@ describe('versions', () => {
             const clientVersion = {
               manifestId: manifest.id,
               deploymentGroupId: deploymentGroup.id,
-              scales: [{
+              scale: [{
                 serviceName: 'consul',
                 replicas: 3
               }],
@@ -370,7 +370,7 @@ describe('versions', () => {
             data.createVersion(clientVersion, (err, result) => {
               expect(err).to.not.exist();
               expect(result.id).to.exist();
-              expect(result.scales).to.equal(clientVersion.scales);
+              expect(result.scale).to.equal(clientVersion.scale);
               data.getVersions({ manifestId: clientVersion.manifestId }, (err, versions) => {
                 expect(err).to.not.exist();
                 expect(versions.length).to.equal(1);
@@ -401,7 +401,7 @@ describe('versions', () => {
             const clientVersion = {
               manifestId: manifest.id,
               deploymentGroupId: deploymentGroup.id,
-              scales: [{
+              scale: [{
                 serviceName: 'consul',
                 replicas: 3
               }],
@@ -695,7 +695,7 @@ describe.skip('scale()', () => {
                 data.scale({ id: deploymentGroupServices[0].id, replicas: 3 }, (err, version) => {
                   expect(err).to.not.exist();
                   expect(version).to.exist();
-                  expect(version.scales[0].replicas).to.equal(3);
+                  expect(version.scale[0].replicas).to.equal(3);
                   done();
                 });
               });
diff --git a/packages/portal-watch/lib/index.js b/packages/portal-watch/lib/index.js
index 6585640d..40976420 100644
--- a/packages/portal-watch/lib/index.js
+++ b/packages/portal-watch/lib/index.js
@@ -2,6 +2,7 @@
 
 // const Assert = require('assert');
 const TritonWatch = require('triton-watch');
+const util = require('util');
 
 
 const DEPLOYMENT_GROUP = 'docker:label:com.docker.compose.project';
@@ -30,6 +31,10 @@ module.exports = class Watcher {
     this._tritonWatch.on('change', (container) => { return this.onChange(container); });
   }
 
+  poll () {
+    this._tritonWatch.poll();
+  }
+
   getDeploymentGroupId (name, cb) {
     this._data.getDeploymentGroup({ name }, (err, deploymentGroup) => {
       if (err) {
@@ -40,7 +45,7 @@ module.exports = class Watcher {
     });
   }
 
-  getServiceId ({ serviceName, deploymentGroupId }, cb) {
+  getService ({ serviceName, deploymentGroupId }, cb) {
     this._data.getServices({ name: serviceName, deploymentGroupId }, (err, services) => {
       if (err) {
         return cb(err);
@@ -50,62 +55,87 @@ module.exports = class Watcher {
         return cb();
       }
 
-      return cb(null, services.pop().id);
+      return cb(null, services.pop());
     });
   }
 
-  getInstance (machineId, cb) {
-    this._data.getInstances({ machineId }, (err, instances) => {
-      if (err) {
-        return cb(err);
-      }
-
-      if (!instances || !instances.length) {
-        return cb();
-      }
-
-      return cb(null, instances.pop());
-    });
+  getInstances (service, cb) {
+    service.instances()
+      .then((instances) => { return cb(null, instances); })
+      .catch((err) => { return cb(err); });
   }
 
-  resolveChanges ({ machine, deploymentGroupId, serviceId, instance }) {
-    const handleError = (err) => {
-      if (err) {
-        console.error(err);
-      }
+  resolveChanges ({ machine, service, instances }) {
+    // 1. if instance doesn't exist, create new
+    // 2. if instance exist, update status
+
+    const handleError = (cb) => {
+      return (err, data) => {
+        if (err) {
+          console.error(err);
+          return;
+        }
+
+        if (cb) {
+          cb(err, data);
+        }
+      };
     };
 
+    const isNew = instances
+      .every(({ machineId }) => { return machine.id !== machineId; });
+
+    const instance = instances
+      .filter(({ machineId }) => { return machine.id === machineId; })
+      .pop();
+
     const create = () => {
-      return this._data.updateInstance({
+      const instance = {
         name: machine.name,
-        status: machine.state.toUpperCase(),
+        status: (machine.state || '').toUpperCase(),
         machineId: machine.id
-      }, handleError);
+      };
+
+      console.log('-> creating instance', util.inspect(instance));
+      return this._data.createInstance(instance, handleError((_, instance) => {
+        const updatedService = {
+          id: service.id,
+          instances: instances.concat(instance)
+        };
+
+        console.log('-> updating service', util.inspect(updatedService));
+        return this._data.updateService(updatedService, handleError);
+      }));
     };
 
     const update = () => {
-      return this._data.updateInstance({
+      const updatedInstance = {
         id: instance.id,
-        status: machine.state.toUpperCase()
-      }, handleError);
+        status: (machine.state || '').toUpperCase()
+      };
+
+      console.log('-> updating instance', util.inspect(updatedInstance));
+      return this._data.updateInstance(updatedInstance, handleError);
     };
 
-    return (!instance || !instance.id) ?
+    return isNew ?
       create() :
       update();
   }
 
   onChange (machine) {
     if (!machine) {
-      console.error('`change` event received without machine data');
+      console.error('-> `change` event received without machine data');
       return;
     }
 
+    console.log('-> `change` event received', util.inspect(machine));
+
     const { id, tags = [] } = machine;
 
     // assert id existence
     if (!id) {
-      console.error('`change` event received for a machine without `id`');
+      console.error('-> `change` event received for a machine without `id`');
       return;
     }
 
@@ -115,7 +145,7 @@ module.exports = class Watcher {
     );
 
     if (!isCompose) {
-      console.error(`Changed machine ${id} was not provisioned by docker-compose`);
+      console.error(`-> Changed machine ${id} was not provisioned by docker-compose`);
       return;
     }
 
@@ -133,26 +163,25 @@ module.exports = class Watcher {
       };
     };
 
-    const getInstance = (deploymentGroupId, serviceId) => {
-      this.getInstance(id, handleError((instance) => {
+    const getInstances = (service) => {
+      this.getInstances(service, handleError((instances) => {
         return this.resolveChanges({
           machine,
-          deploymentGroupId,
-          serviceId,
-          instance
+          service,
+          instances
         });
       }));
     };
 
     // assert that service exists
     const assertService = (deploymentGroupId) => {
-      this.getServiceId({ serviceName, deploymentGroupId }, handleError((serviceId) => {
-        if (!serviceId) {
+      this.getService({ serviceName, deploymentGroupId }, handleError((service) => {
+        if (!service) {
           console.error(`Service "${serviceName}" form DeploymentGroup "${deploymentGroupName}" for machine ${id} not found`);
           return;
         }
 
-        getInstance(deploymentGroupId, serviceId);
+        getInstances(service);
       }));
     };
 
diff --git a/packages/portal-watch/test/index.js b/packages/portal-watch/test/index.js
index 2c2471e4..1c5f77e7 100644
--- a/packages/portal-watch/test/index.js
+++ b/packages/portal-watch/test/index.js
@@ -10,27 +10,6 @@ const expect = Lab.expect;
 
 
 it('updates instances with the current status', (done) => {
-  const data = {
-    getDeploymentGroup: (options, next) => {
-      expect(options.name).to.equal('test-project');
-      next(null, { id: 'deployment-group-id' });
-    },
-    getServices: (options, next) => {
-      expect(options.deploymentGroupId).to.equal('deployment-group-id');
-      expect(options.name).to.equal('test-service');
-      next(null, [{ id: 'service-id' }]);
-    },
-    getInstances: (options, next) => {
-      expect(options.machineId).to.equal('test-id');
-      next(null, [{ id: 'instance-id' }]);
-    },
-    updateInstance: (options, next) => {
-      expect(options.id).to.equal('instance-id');
-      expect(options.status).to.equal('DELETED');
-      done();
-    }
-  };
-
   const machine = {
     id: 'test-id',
     tags: {
@@ -41,6 +20,70 @@ it('updates instances with the current status', (done) => {
     state: 'deleted'
   };
 
+  const data = {
+    getDeploymentGroup: (options, next) => {
+      expect(options.name).to.equal('test-project');
+      next(null, { id: 'deployment-group-id' });
+    },
+    getServices: (options, next) => {
+      expect(options.deploymentGroupId).to.equal('deployment-group-id');
+      expect(options.name).to.equal('test-service');
+      next(null, [{
+        id: 'service-id',
+        instances: () => {
+          return Promise.resolve([{
+            machineId: machine.id,
+            id: 'instance-id'
+          }]);
+        }
+      }]);
+    },
+    updateInstance: (options, next) => {
+      expect(options.id).to.equal('instance-id');
+      expect(options.status).to.equal('DELETED');
+      done();
+    }
+  };
+
+  const portalOptions = { data, url: 'url', account: 'account', keyId: 'de:e7:73:9a:aa:91:bb:3e:72:8d:cc:62:ca:58:a2:ec' };
+  const portalWatch = new PortalWatch(portalOptions);
+  portalWatch._tritonWatch.removeAllListeners('change');
+
+  portalWatch.onChange(machine);
+});
+
+it('creates new instance', (done) => {
+  const machine = {
+    id: 'test-id',
+    tags: {
+      'docker:label:com.docker.compose.project': 'test-project',
+      'docker:label:com.docker.compose.service': 'test-service',
+      'docker:label:com.docker.compose.config-hash': 'test-hash'
+    },
+    state: 'created'
+  };
+
+  const data = {
+    getDeploymentGroup: (options, next) => {
+      expect(options.name).to.equal('test-project');
+      next(null, { id: 'deployment-group-id' });
+    },
+    getServices: (options, next) => {
+      expect(options.deploymentGroupId).to.equal('deployment-group-id');
+      expect(options.name).to.equal('test-service');
+      next(null, [{
+        id: 'service-id',
+        instances: () => { return Promise.resolve([]); }
+      }]);
+    },
+    createInstance: (options, next) => {
+      expect(options.id).to.equal(undefined);
+      expect(options.status).to.equal('CREATED');
+      expect(options.machineId).to.equal('test-id');
+      done();
+    }
+  };
+
   const portalOptions = { data, url: 'url', account: 'account', keyId: 'de:e7:73:9a:aa:91:bb:3e:72:8d:cc:62:ca:58:a2:ec' };
   const portalWatch = new PortalWatch(portalOptions);
   portalWatch._tritonWatch.removeAllListeners('change');
diff --git a/yarn.lock b/yarn.lock
index 49e31cac..efb6647b 100644
--- a/yarn.lock
+++ b/yarn.lock
@@ -502,10 +502,6 @@ ast-types@0.9.0:
   version "0.9.0"
   resolved "https://registry.yarnpkg.com/ast-types/-/ast-types-0.9.0.tgz#c8721c8747ae4d5b29b929e99c5317b4e8745623"
 
-ast-types@0.9.6:
-  version "0.9.6"
-  resolved "https://registry.yarnpkg.com/ast-types/-/ast-types-0.9.6.tgz#102c9e9e9005d3e7e3829bf0c4fa24ee862ee9b9"
-
 async-each@^1.0.0:
   version "1.0.1"
   resolved "https://registry.yarnpkg.com/async-each/-/async-each-1.0.1.tgz#19d386a1d9edc6e7c1c85d388aedbcc56d33602d"
@@ -1401,8 +1397,8 @@ balanced-match@^1.0.0:
   resolved "https://registry.yarnpkg.com/balanced-match/-/balanced-match-1.0.0.tgz#89b4d199ab2bee49de164ea02b89ce462d71b767"
 
 base64-js@^1.0.2:
-  version "1.2.0"
-  resolved "https://registry.yarnpkg.com/base64-js/-/base64-js-1.2.0.tgz#a39992d723584811982be5e290bb6a53d86700f1"
+  version "1.2.1"
+  resolved "https://registry.yarnpkg.com/base64-js/-/base64-js-1.2.1.tgz#a91947da1f4a516ea38e5b4ec0ec3773675e0886"
 
 bcrypt-pbkdf@^1.0.0:
   version "1.0.1"
@@ -1770,12 +1766,12 @@ camelcase@^4.0.0, camelcase@^4.1.0:
   resolved "https://registry.yarnpkg.com/camelcase/-/camelcase-4.1.0.tgz#d545635be1e33c542649c69173e5de6acfae34dd"
 
 caniuse-db@^1.0.30000187, caniuse-db@^1.0.30000634, caniuse-db@^1.0.30000639:
-  version "1.0.30000692"
-  resolved "https://registry.yarnpkg.com/caniuse-db/-/caniuse-db-1.0.30000692.tgz#3da9a99353adbcea1e142b99f60ecc6216df47a5"
+  version "1.0.30000693"
+  resolved "https://registry.yarnpkg.com/caniuse-db/-/caniuse-db-1.0.30000693.tgz#8510e7a9ab04adcca23a5dcefa34df9d28c1ce20"
 
 caniuse-lite@^1.0.30000684:
-  version "1.0.30000692"
-  resolved "https://registry.yarnpkg.com/caniuse-lite/-/caniuse-lite-1.0.30000692.tgz#34600fd7152352d85a47f4662a3b51b02d8b646f"
+  version "1.0.30000693"
+  resolved "https://registry.yarnpkg.com/caniuse-lite/-/caniuse-lite-1.0.30000693.tgz#c9c6298697c71fdf6cb13eefe8aa93926f2f8613"
 
 capture-stack-trace@^1.0.0:
   version "1.0.0"
@@ -2023,8 +2019,8 @@ code@4.1.x, code@^4.1.0:
     hoek "4.x.x"
 
 codemirror@^5.18.2:
-  version "5.26.0"
-  resolved "https://registry.yarnpkg.com/codemirror/-/codemirror-5.26.0.tgz#bcbee86816ed123870c260461c2b5c40b68746e5"
+  version "5.27.0"
+  resolved "https://registry.yarnpkg.com/codemirror/-/codemirror-5.27.0.tgz#0c72f70c321a7d494fd8db1976698c249c985eb3"
 
 coleman-liau@^1.0.0:
   version "1.0.0"
@@ -2861,12 +2857,6 @@ debug-log@^1.0.1:
   version "1.0.1"
   resolved "https://registry.yarnpkg.com/debug-log/-/debug-log-1.0.1.tgz#2307632d4c04382b8df8a32f70b895046d52745f"
 
-debug@2.2.0, debug@~2.2.0:
-  version "2.2.0"
-  resolved "https://registry.yarnpkg.com/debug/-/debug-2.2.0.tgz#f87057e995b1a1f6ae6a4960664137bc56f039da"
-  dependencies:
-    ms "0.7.1"
-
 debug@2.6.7:
   version "2.6.7"
   resolved "https://registry.yarnpkg.com/debug/-/debug-2.6.7.tgz#92bad1f6d05bbb6bba22cca88bcd0ec894c2861e"
@@ -2879,6 +2869,12 @@ debug@^2.1.1, debug@^2.2.0, debug@^2.6.0, debug@^2.6.3, debug@^2.6.8:
   dependencies:
     ms "2.0.0"
 
+debug@~2.2.0:
+  version "2.2.0"
+  resolved "https://registry.yarnpkg.com/debug/-/debug-2.2.0.tgz#f87057e995b1a1f6ae6a4960664137bc56f039da"
+  dependencies:
+    ms "0.7.1"
+
 debuglog@^1.0.1:
   version "1.0.1"
   resolved "https://registry.yarnpkg.com/debuglog/-/debuglog-1.0.1.tgz#aa24ffb9ac3df9a2351837cfb2d279360cd78492"
@@ -3334,10 +3330,10 @@ eslint-import-resolver-node@^0.2.0:
     resolve "^1.1.6"
 
 eslint-module-utils@^2.0.0:
-  version "2.0.0"
-  resolved "https://registry.yarnpkg.com/eslint-module-utils/-/eslint-module-utils-2.0.0.tgz#a6f8c21d901358759cdc35dbac1982ae1ee58bce"
+  version "2.1.1"
+  resolved "https://registry.yarnpkg.com/eslint-module-utils/-/eslint-module-utils-2.1.1.tgz#abaec824177613b8a95b299639e1b6facf473449"
   dependencies:
-    debug "2.2.0"
+    debug "^2.6.8"
     pkg-dir "^1.0.0"
 
 eslint-plugin-flowtype@^2.34.0:
@@ -3356,12 +3352,12 @@ eslint-plugin-hapi@4.x.x:
     no-arrowception "1.x.x"
 
 eslint-plugin-import@^2.3.0:
-  version "2.3.0"
-  resolved "https://registry.yarnpkg.com/eslint-plugin-import/-/eslint-plugin-import-2.3.0.tgz#37c801e0ada0e296cbdf20c3f393acb5b52af36b"
+  version "2.5.0"
+  resolved "https://registry.yarnpkg.com/eslint-plugin-import/-/eslint-plugin-import-2.5.0.tgz#293b5ea7910a901a05a47ccdd7546e611725406c"
   dependencies:
     builtin-modules "^1.1.1"
     contains-path "^0.1.0"
-    debug "^2.2.0"
+    debug "^2.6.8"
     doctrine "1.5.0"
     eslint-import-resolver-node "^0.2.0"
     eslint-module-utils "^2.0.0"
@@ -3502,7 +3498,7 @@ esprima@^2.6.0, esprima@~2.7.1:
   version "2.7.3"
   resolved "https://registry.yarnpkg.com/esprima/-/esprima-2.7.3.tgz#96e3b70d5779f6ad49cd032673d1c312767ba581"
 
-esprima@^3.1.1, esprima@~3.1.0:
+esprima@^3.1.1:
   version "3.1.3"
   resolved "https://registry.yarnpkg.com/esprima/-/esprima-3.1.3.tgz#fdca51cee6133895e3c88d535ce49dbff62a4633"
 
@@ -3712,11 +3708,11 @@ extsprintf@1.0.2:
   version "1.0.2"
   resolved "https://registry.yarnpkg.com/extsprintf/-/extsprintf-1.0.2.tgz#e1080e0658e300b06294990cc70e1502235fd550"
 
-extsprintf@1.2.0, extsprintf@^1.2.0:
+extsprintf@1.2.0:
   version "1.2.0"
   resolved "https://registry.yarnpkg.com/extsprintf/-/extsprintf-1.2.0.tgz#5ad946c22f5b32ba7f8cd7426711c6e8a3fc2529"
 
-extsprintf@1.3.0:
+extsprintf@1.3.0, extsprintf@^1.2.0:
   version "1.3.0"
   resolved "https://registry.yarnpkg.com/extsprintf/-/extsprintf-1.3.0.tgz#96918440e3041a7a414f8c52e3c574eb3c3e1e05"
 
@@ -4008,8 +4004,8 @@ get-caller-file@^1.0.1:
   resolved "https://registry.yarnpkg.com/get-caller-file/-/get-caller-file-1.0.2.tgz#f702e63127e7e231c160a80c1554acb70d5047e5"
 
 get-pkg-repo@^1.0.0:
-  version "1.3.0"
-  resolved "https://registry.yarnpkg.com/get-pkg-repo/-/get-pkg-repo-1.3.0.tgz#43c6b4c048b75dd604fc5388edecde557f6335df"
+  version "1.4.0"
+  resolved "https://registry.yarnpkg.com/get-pkg-repo/-/get-pkg-repo-1.4.0.tgz#c73b489c06d80cc5536c2c853f9e05232056972d"
   dependencies:
     hosted-git-info "^2.1.4"
     meow "^3.3.0"
@@ -4247,8 +4243,8 @@ graphql-anywhere@^3.0.0, graphql-anywhere@^3.0.1:
   resolved "https://registry.yarnpkg.com/graphql-anywhere/-/graphql-anywhere-3.1.0.tgz#3ea0d8e8646b5cee68035016a9a7557c15c21e96"
 
 graphql-tag@^2.0.0, graphql-tag@^2.4.0:
-  version "2.4.0"
-  resolved "https://registry.yarnpkg.com/graphql-tag/-/graphql-tag-2.4.0.tgz#0fe137348d4db2efaf29b52ba4c1cbf84ac138cb"
+  version "2.4.2"
+  resolved "https://registry.yarnpkg.com/graphql-tag/-/graphql-tag-2.4.2.tgz#6a63297d8522d03a2b72d26f1b239aab343840cd"
 
 graphql@^0.10.0, graphql@^0.10.1:
   version "0.10.3"
@@ -5200,8 +5196,8 @@ jest-snapshot@^20.0.3:
     pretty-format "^20.0.3"
 
 jest-styled-components@^3.0.2:
-  version "3.1.1"
-  resolved "https://registry.yarnpkg.com/jest-styled-components/-/jest-styled-components-3.1.1.tgz#d859a73c7844ecb47204b7240c00b499502a1fc8"
+  version "3.1.2"
+  resolved "https://registry.yarnpkg.com/jest-styled-components/-/jest-styled-components-3.1.2.tgz#4e1a1da604824c7f31507e80c43e88a2d8d8c4c9"
   dependencies:
     css "^2.2.1"
 
@@ -6141,7 +6137,7 @@ mooremachine@^2.0.1:
   optionalDependencies:
     dtrace-provider "~0.8"
 
-ms@0.7.1, ms@^0.7.1:
+ms@0.7.1:
   version "0.7.1"
   resolved "https://registry.yarnpkg.com/ms/-/ms-0.7.1.tgz#9cd13c03adbff25b65effde7ce864ee952017098"
 
@@ -6149,6 +6145,10 @@ ms@2.0.0:
   version "2.0.0"
   resolved "https://registry.yarnpkg.com/ms/-/ms-2.0.0.tgz#5608aeadfc00be6c2901df5f9861788de0d597c8"
 
+ms@^0.7.1:
+  version "0.7.3"
+  resolved "https://registry.yarnpkg.com/ms/-/ms-0.7.3.tgz#708155a5e44e33f5fd0fc53e81d0d40a91be1fff"
+
 msgpack@1.0.2:
   version "1.0.2"
   resolved "https://registry.yarnpkg.com/msgpack/-/msgpack-1.0.2.tgz#923e2c5cffa65c8418e9b228d1124793969c429c"
@@ -7454,8 +7454,8 @@ read@1.0.7:
     mute-stream "~0.0.4"
 
 "readable-stream@>= 1.0.2", readable-stream@^2, readable-stream@^2.0.0, readable-stream@^2.0.1, readable-stream@^2.0.2, readable-stream@^2.0.5, readable-stream@^2.0.6, readable-stream@^2.1.4, readable-stream@^2.1.5, readable-stream@^2.2.2, readable-stream@^2.2.6:
-  version "2.3.1"
-  resolved "https://registry.yarnpkg.com/readable-stream/-/readable-stream-2.3.1.tgz#84e26965bb9e785535ed256e8d38e92c69f09d10"
+  version "2.3.2"
+  resolved "https://registry.yarnpkg.com/readable-stream/-/readable-stream-2.3.2.tgz#5a04df05e4f57fe3f0dc68fdd11dc5c97c7e6f4d"
   dependencies:
     core-util-is "~1.0.0"
     inherits "~2.0.3"
@@ -7526,7 +7526,7 @@ readline2@^1.0.1:
     is-fullwidth-code-point "^1.0.0"
     mute-stream "0.0.5"
 
-recast@0.11.12:
+recast@0.11.12, recast@^0.11.5:
   version "0.11.12"
   resolved "https://registry.yarnpkg.com/recast/-/recast-0.11.12.tgz#a79e4d3f82d5d72a82ee177aeaa791e793bbe5d6"
   dependencies:
@@ -7535,15 +7535,6 @@ recast@0.11.12:
     private "~0.1.5"
     source-map "~0.5.0"
 
-recast@^0.11.5:
-  version "0.11.23"
-  resolved "https://registry.yarnpkg.com/recast/-/recast-0.11.23.tgz#451fd3004ab1e4df9b4e4b66376b2a21912462d3"
-  dependencies:
-    ast-types "0.9.6"
-    esprima "~3.1.0"
-    private "~0.1.5"
-    source-map "~0.5.0"
-
 rechoir@^0.6.2:
   version "0.6.2"
   resolved "https://registry.yarnpkg.com/rechoir/-/rechoir-0.6.2.tgz#85204b54dba82d5742e28c96756ef43af50e3384"
@@ -8081,8 +8072,8 @@ rx-lite@^3.1.2:
   resolved "https://registry.yarnpkg.com/rx-lite/-/rx-lite-3.1.2.tgz#19ce502ca572665f3b647b10939f97fd1615f102"
 
 safe-buffer@^5.0.1, safe-buffer@^5.1.0, safe-buffer@~5.1.0:
-  version "5.1.0"
-  resolved "https://registry.yarnpkg.com/safe-buffer/-/safe-buffer-5.1.0.tgz#fe4c8460397f9eaaaa58e73be46273408a45e223"
+  version "5.1.1"
+  resolved "https://registry.yarnpkg.com/safe-buffer/-/safe-buffer-5.1.1.tgz#893312af69b2123def71f57889001671eeb2c853"
 
 safe-buffer@~5.0.1:
   version "5.0.1"
@@ -9228,8 +9219,8 @@ typedarray@^0.0.6, typedarray@~0.0.5:
   resolved "https://registry.yarnpkg.com/typedarray/-/typedarray-0.0.6.tgz#867ac74e3864187b1d3d47d996a78ec5c8830777"
 
 ua-parser-js@^0.7.9:
-  version "0.7.12"
-  resolved "https://registry.yarnpkg.com/ua-parser-js/-/ua-parser-js-0.7.12.tgz#04c81a99bdd5dc52263ea29d24c6bf8d4818a4bb"
+  version "0.7.13"
+  resolved "https://registry.yarnpkg.com/ua-parser-js/-/ua-parser-js-0.7.13.tgz#cd9dd2f86493b3f44dbeeef3780fda74c5ee14be"
 
 uglify-js@^2.6, uglify-js@^2.8.27:
   version "2.8.29"