feat: track transitional states

packages/cp-frontend/src/components/services/instance-statuses.js

@@ -16,12 +16,8 @@ const StyledStatusContainer = styled.div`
 `;
 
 const InstanceStatuses = ({ instanceStatuses }) => {
-
   const statuses = instanceStatuses.map(instanceStatus => {
-    const {
-      status,
-      count
-    } = instanceStatus;
+    const { status, count } = instanceStatus;
 
     return (
       <StyledStatus>

packages/cp-frontend/src/containers/deployment-groups/manifest.js

@@ -35,11 +35,11 @@ const Manifest = ({
     deploymentGroup &&
     deploymentGroup.imported &&
     !manifest
-    ? null
-    : <span>
+    ? <span>
         Since this DeploymentGroup was imported, it doesn't have the initial
         manifest
-      </span>;
+      </span>
+    : null;
 
   return (
     <LayoutContainer>

packages/cp-frontend/src/containers/services/list.js

@@ -44,7 +44,10 @@ class ServiceList extends Component {
       startServices
     } = this.props;
 
-    if (loading) {
+    if (
+      loading ||
+      (deploymentGroup.status === 'PROVISIONING' && !services.length)
+    ) {
       return (
         <LayoutContainer>
           <Loader />

packages/cp-frontend/src/graphql/DeploymentGroupInfo.gql

@@ -3,4 +3,5 @@ fragment DeploymentGroupInfo on DeploymentGroup {
   name
   slug
   imported
+  status
 }

packages/cp-frontend/src/graphql/DeploymentGroupProvision.gql

@@ -5,12 +5,9 @@ mutation provisionManifest($deploymentGroupId: ID!, $type: ManifestType!, $forma
       replicas
     }
     plan {
-      running
-      actions {
-        type
-        service
-        machines
-      }
+      type
+      service
+      machines
     }
   }
 }

packages/cp-frontend/src/graphql/ServiceInfo.gql

@@ -2,4 +2,5 @@ fragment ServiceInfo on Service {
   id
   name
   slug
+  status
 }

packages/cp-gql-schema/schema.gql

@@ -16,6 +16,14 @@ type User {
   login: String!
 }
 
+enum DeploymentGroupStatus {
+  ACTIVE
+  PROVISIONING
+  DELETING
+  DELETED
+  UNKNOWN
+}
+
 type DeploymentGroup {
   id: ID!
   name: String!
@@ -24,6 +32,7 @@ type DeploymentGroup {
   version: Version
   history: [Version]
   imported: Boolean
+  status: DeploymentGroupStatus
 }
 
 type ServiceScale {
@@ -35,27 +44,25 @@ enum ConvergenceActionType {
   NOOP
   CREATE
   RECREATE
+  REMOVE
   START
+  EXISTING # special status to mark existing ids in previous version
 }
 
 type ConvergenceAction {
-  id: ID!
   type: ConvergenceActionType!
   service: String! # service name
-  machines: [String]! # instance machine ids
-}
-
-type StateConvergencePlan {
-  id: ID!
-  running: Boolean!
-  actions(type: ConvergenceActionType, service: String): [ConvergenceAction]!
+  toProcess: Int, # merely used for book keeping
+  processed: [String], # merely used for book keeping
+  machines: [String]! # current instance machine ids
 }
 
 type Version {
-  created: Date! # Either Int or define scalar
   manifest: Manifest!
   scale(serviceName: String): [ServiceScale]!
-  plan(running: Boolean): StateConvergencePlan
+  plan: [ConvergenceAction]
+  hasPlan: Boolean
+  error: String
 }
 
 enum ManifestType {
@@ -70,13 +77,24 @@ enum ManifestFormat {
 
 type Manifest {
   id: ID!
-  created: Float
   type: ManifestType!
   format: ManifestFormat!
   raw: String!
   obj: Object
 }
 
+enum ServiceStatus {
+  ACTIVE # this doesn't mean that the instances are all running
+  PROVISIONING
+  SCALING
+  STOPPING
+  STOPPED
+  DELETING
+  DELETED
+  RESTARTING
+  UNKNOWN
+}
+
 # immutable
 type Service {
   id: ID! # unique id for db row
@@ -84,14 +102,12 @@ type Service {
   name: String! # human readable name
   slug:  String!
   instances(name: String, machineId: ID, status: InstanceStatus): [Instance]!
-  # metrics: [MetricType]
-  currentMetrics: [CurrentMetric]
   connections: [String!] # list of serviceIds
   parent: ID # parent service id
   package: Package! # we don't have this in current mock data,
   environment: [Environment]
-  active: Boolean! # let's say that we update the manifest, and remove a service. we still want to track this service even though it might not exist because it might have machines running still
   image: String # used only for config
+  status: ServiceStatus
 }
 
 # for metrics max / min (I guess)
@@ -135,7 +151,6 @@ type Instance {
   machineId: ID!
   status: InstanceStatus!
   healthy: Boolean
-  # metrics: [InstanceMetric]!
 }
 
 type Datacenter {
@@ -145,28 +160,6 @@ type Datacenter {
   region: String!
 }
 
-type InstanceMetric {
-  type: MetricType!
-  data: [MetricData]!
-}
-
-type CurrentMetric {
-  name: String!
-  value: Float!
-  measurement: String!
-}
-
-type MetricType {
-  id: ID!
-  name: String!
-  id: ID!
-}
-
-type MetricData {
-  timestamp: Int!
-  value: Float!
-}
-
 # we probably wont use some of these queries or arguments
 # but this way we expose the entire db through gql
 type Query {
@@ -178,8 +171,6 @@ type Query {
   serviceScale(id: ID!): ServiceScale
   convergenceActions(type: ConvergenceActionType, service: String, versionId: ID): [ConvergenceAction]
   convergenceAction(id: ID!): ConvergenceAction
-  stateConvergencePlans(running: Boolean, versionId: ID): [StateConvergencePlan]
-  stateConvergencePlan(id: ID!): StateConvergencePlan
   versions(manifestId: ID, deploymentGroupId: ID): [Version]
   version(id: ID, manifestId: ID): Version
   manifests(type: String, deploymentGroupId: ID): [Manifest]

packages/eslint-config/index.js

@@ -2,7 +2,13 @@
 // TODO wait for eslint/eslint#3458
 
 module.exports = {
-  extends: ['eslint:recommended', 'xo-space/esnext', 'react-app', 'prettier', 'prettier/react'],
+  extends: [
+    'eslint:recommended',
+    'xo-space/esnext',
+    'react-app',
+    'prettier',
+    'prettier/react'
+  ],
   rules: {
     'capitalized-comments': 0
   }

packages/portal-api/lib/data/transform.js

@@ -1,7 +1,11 @@
 'use strict';
 
 const Yamljs = require('yamljs');
+const ParamCase = require('param-case');
 
+const clean = (v) => {
+  return JSON.parse(JSON.stringify(v));
+};
 
 exports.fromPortal = function ({ portal, datacenter, deploymentGroups, user }) {
   return {
@@ -30,18 +34,31 @@ exports.fromDeploymentGroup = function (deploymentGroup) {
     slug: deploymentGroup.slug,
     services: deploymentGroup.services,
     version: deploymentGroup.version,
-    history: deploymentGroup.history_version_ids || []
+    history: deploymentGroup.history,
+    imported: deploymentGroup.imported,
+    status: deploymentGroup.status
   };
 };
 
-exports.toDeploymentGroup = function ({ name }) {
-  return {
-    name,
-    slug: name,
-    service_ids: [],
-    version_id: '',
-    history_version_ids: []
-  };
+exports.toDeploymentGroup = function (clientDeploymentGroup) {
+  return clean({
+    id: clientDeploymentGroup.id,
+    name: clientDeploymentGroup.name,
+    slug: clientDeploymentGroup.slug ?
+      clientDeploymentGroup.slug :
+      clientDeploymentGroup.name ?
+        ParamCase(clientDeploymentGroup.name) :
+        undefined,
+    service_ids: Array.isArray(clientDeploymentGroup.services) ? clientDeploymentGroup.services.map(({ id }) => {
+      return id;
+    }).filter(Boolean) : undefined,
+    version_id: clientDeploymentGroup.version ? clientDeploymentGroup.version.id : undefined,
+    history_version_ids: Array.isArray(clientDeploymentGroup.history) ? clientDeploymentGroup.history.map(({ id }) => {
+      return id;
+    }).filter(Boolean) : undefined,
+    imported: clientDeploymentGroup.imported,
+    status: clientDeploymentGroup.status || 'ACTIVE'
+  });
 };
 
 
@@ -58,36 +75,43 @@ exports.fromService = function ({ service, instances, packages }) {
     connections: service.service_dependency_ids,
     package: packages ? exports.fromPackage(packages) : {},
     parent: service.parent_id || '',
-    active: service.active
+    status: service.status,
+    hasPlan: service.has_plan
   };
 };
 
 exports.toService = function (clientService) {
   // wat??
-  return JSON.parse(JSON.stringify({
+  return clean({
     id: clientService.id,
-    version_hash: clientService.hash || clientService.name,
+    version_hash: clientService.hash,
     deployment_group_id: clientService.deploymentGroupId,
     name: clientService.name,
     slug: clientService.slug,
-    environment: clientService.environment || {},
-    instance_ids: clientService.instances ? clientService.instances.map((instance) => { return instance.id; }) : undefined,
-    service_dependency_ids: clientService.connections || [],
-    package_id: clientService.package ? clientService.package.id : '',
-    parent_id: clientService.parent || '',
-    active: clientService.ative
-  }));
+    environment: clientService.environment,
+    instance_ids: clientService.instances ?
+      clientService.instances.map((instance) => {
+        return instance.id;
+      }) :
+      undefined,
+    service_dependency_ids: clientService.connections,
+    package_id: clientService.package ? clientService.package.id : undefined,
+    parent_id: clientService.parent ? clientService.parent : undefined,
+    status: clientService.status,
+    has_plan: clientService.hasPlan
+  });
 };
 
 
 exports.toVersion = function (clientVersion) {
-  return {
+  return clean({
     id: clientVersion.id,
     created: clientVersion.created || Date.now(),
     manifest_id: (clientVersion.manifest || {}).id,
-    service_scales: clientVersion.scale ? clientVersion.scale.map(exports.toScale) : [],
-    plan: exports.toPlan(clientVersion.plan || {})
-  };
+    service_scales: clientVersion.scale ? clientVersion.scale : undefined,
+    plan: clientVersion.plan ? clientVersion.plan : undefined,
+    error: clientVersion.version
+  });
 };
 
 exports.fromVersion = function (version) {
@@ -95,38 +119,9 @@ exports.fromVersion = function (version) {
     id: version.id,
     created: version.created,
     manifest: version.manifest,
-    scale: version.service_scales ? version.service_scales.map(exports.fromScale) : [],
-    plan: exports.fromPlan(version.plan || {})
-  };
-};
-
-
-exports.toScale = function (clientScale) {
-  return {
-    service_name: clientScale.serviceName,
-    replicas: clientScale.replicas
-  };
-};
-
-exports.fromScale = function (scale) {
-  return {
-    serviceName: scale.service_name,
-    replicas: scale.replicas
-  };
-};
-
-
-exports.toPlan = function (clientPlan) {
-  return {
-    running: clientPlan.running,
-    actions: clientPlan.actions
-  };
-};
-
-exports.fromPlan = function (plan) {
-  return {
-    running: plan.running,
-    actions: plan.actions
+    scale: version.service_scales,
+    plan: version.plan,
+    error: version.error
   };
 };
 
@@ -170,21 +165,21 @@ exports.fromInstance = function (instance) {
     id: instance.id,
     name: instance.name,
     machineId: instance.machine_id,
-    status: instance.status || '',
-    ips: instance.ips || [],
+    status: instance.status,
+    ips: instance.ips,
     healthy: instance.healthy
   };
 };
 
 exports.toInstance = function (clientInstance) {
-  return {
+  return clean({
     id: clientInstance.id,
     name: clientInstance.name,
     machine_id: clientInstance.machineId,
-    status: clientInstance.status || '',
-    ips: clientInstance.ips || [],
+    status: clientInstance.status,
+    ips: clientInstance.ips,
     healthy: clientInstance.healthy
-  };
+  });
 };
 
 

packages/portal-api/lib/watch/index.js

@@ -3,20 +3,62 @@
 // const Assert = require('assert');
 const Throat = require('throat');
 const TritonWatch = require('triton-watch');
-const util = require('util');
+const Get = require('lodash.get');
+const Find = require('lodash.find');
+const Util = require('util');
+const ForceArray = require('force-array');
+const VAsync = require('vasync');
 
 
 const DEPLOYMENT_GROUP = 'docker:label:com.docker.compose.project';
 const SERVICE = 'docker:label:com.docker.compose.service';
 const HASH = 'docker:label:com.docker.compose.config-hash';
 
+const ACTION_REMOVE_STATUSES = [
+  'STOPPING',
+  'STOPPED',
+  'OFFLINE',
+  'DELETED',
+  'DESTROYED',
+  'FAILED',
+  'INCOMPLETE',
+  'UNKNOWN'
+];
+
+const ACTION_CREATE_STATUSES = [
+  'READY',
+  'ACTIVE',
+  'RUNNING',
+  'STOPPED',
+  'OFFLINE',
+  'FAILED',
+  'INCOMPLETE',
+  'UNKNOWN'
+];
+
+const SERVICE_STOPPING_STATUSES = [
+  'STOPPED',
+  'OFFLINE',
+  'FAILED',
+  'INCOMPLETE',
+  'UNKNOWN'
+];
+
+const SERVICE_DELETING_STATUSES = [
+  'DELETED',
+  'DESTROYED',
+  'FAILED',
+  'INCOMPLETE',
+  'UNKNOWN'
+];
+
 module.exports = class Watcher {
   constructor (options) {
     options = options || {};
 
     // todo assert options
     this._data = options.data;
-    this._frequency = 500;
+    this._frequency = 200;
 
     this._tritonWatch = new TritonWatch({
       frequency: this._frequency,
@@ -30,50 +72,68 @@ module.exports = class Watcher {
     });
 
     this._queues = {};
+    this._waitingForPlan = [];
+    this._isTritonWatchPolling = false;
 
-    this._tritonWatch.on('change', (container) => {
-      return this.onChange(container);
+    this._tritonWatch.on('change', (machine) => {
+      return this.onChange(machine);
     });
 
-    this._tritonWatch.on('all', (containers) => {
-      containers.forEach((container) => {
-        this.onChange(container);
+    this._tritonWatch.on('all', (machines) => {
+      machines.forEach((machine) => {
+        this.onChange(machine);
       });
     });
   }
 
   poll () {
-    this._tritonWatch.poll();
+    if (!this._isTritonWatchPolling) {
+      this._tritonWatch.poll();
+      this._isTritonWatchPolling = true;
+    }
+
+    if (this._isWaitingPolling) {
+      return;
+    }
+
+    this._isWaitingPolling = true;
+
+    setTimeout(() => {
+      this._isWaitingPolling = false;
+      this._checkForWaiting();
+    }, this._frequency);
+  }
+
+  _checkForWaiting () {
+    this._waitingForPlan.forEach(this.onChange);
   }
 
   getContainers () {
     return this._tritonWatch.getContainers();
   }
 
-  pushToQueue ({ serviceName, deploymentGroupId }, cb) {
-    const name = `${deploymentGroupId}-${serviceName}`;
-
-    if (this._queues[name]) {
-      this._queues[name](cb);
+  pushToQueue (deploymentGroupId, cb) {
+    if (this._queues[deploymentGroupId]) {
+      this._queues[deploymentGroupId](cb);
       return;
     }
 
-    this._queues[name] = Throat(1);
-    this._queues[name](cb);
+    this._queues[deploymentGroupId] = Throat(1);
+    this._queues[deploymentGroupId](cb);
   }
 
-  getDeploymentGroupId (name, cb) {
+  getDeploymentGroup (name, cb) {
     this._data.getDeploymentGroup({ name }, (err, deploymentGroup) => {
       if (err) {
         return cb(err);
       }
 
-      return cb(null, deploymentGroup && deploymentGroup.id);
+      return cb(null, deploymentGroup);
     });
   }
 
-  getService ({ serviceName, serviceHash, deploymentGroupId }, cb) {
-    this._data.getServices({ name: serviceName, hash: serviceHash, deploymentGroupId }, (err, services) => {
+  getService ({ serviceName, deploymentGroupId }, cb) {
+    this._data.getServices({ name: serviceName, deploymentGroupId }, (err, services) => {
       if (err) {
         return cb(err);
       }
@@ -88,88 +148,395 @@ module.exports = class Watcher {
 
   getInstances (service, cb) {
     service.instances()
-      .then((instances) => { return cb(null, instances); })
-      .catch((err) => { return cb(err); });
+      .then((instances) => {
+        return cb(null, instances);
+      })
+      .catch((err) => {
+        return cb(err);
+      });
   }
 
-  resolveChanges ({ machine, service, instances }, cb) {
-    // 1. if instance doesn't exist, create new
-    // 2. if instance exist, update status
+  getVersion (deploymentGroup, cb) {
+    deploymentGroup.version()
+      .then((version) => {
+        return cb(null, version);
+      })
+      .catch((err) => {
+        return cb(err);
+      });
+  }
 
-    const handleError = (cb) => {
-      return (err, data) => {
-        if (err) {
-          console.error(err);
-          return;
-        }
+  createInstance ({ machine, instances, service }, cb) {
+    console.error(`-> detected that machine ${machine.name} was created`);
 
-        if (cb) {
-          cb(err, data);
-        }
+    const status = (machine.state || '').toUpperCase();
+
+    if (status === 'DELETED') {
+      return cb();
+    }
+
+    const instance = {
+      name: machine.name,
+      status,
+      ips: machine.ips,
+      machineId: machine.id
+    };
+
+    console.log('-> creating instance', Util.inspect(instance));
+    this._data.createInstance(instance, (err, instance) => {
+      if (err) {
+        return cb(err);
+      }
+
+      const payload = {
+        id: service.id,
+        instances: instances.concat(instance)
       };
+
+      console.log('-> updating service', Util.inspect(payload));
+      this._data.updateService(payload, cb);
+    });
+  }
+
+  updateInstance ({ machine, instance, instances, service }, cb) {
+    console.error(`-> detected that machine ${machine.name} was updated`);
+
+    const updatedInstance = {
+      id: instance.id,
+      ips: machine.ips,
+      status: (machine.state || '').toUpperCase()
     };
 
-    const isNew = instances
-      .every(({ machineId }) => { return machine.id !== machineId; });
+    console.log('-> updating instance', Util.inspect(updatedInstance));
+    this._data.updateInstance(updatedInstance, (err) => {
+      if (err) {
+        return cb(err);
+      }
 
-    const instance = instances
-      .filter(({ machineId }) => { return machine.id === machineId; })
-      .pop();
-
-    const updateService = (updatedService, cb) => {
-      console.log('-> updating service', util.inspect(updatedService));
-      return this._data.updateService(updatedService, handleError(cb));
-    };
-
-    const create = (cb) => {
-      const status = (machine.state || '').toUpperCase();
-
-      if (status === 'DELETED') {
+      if (['DELETED', 'DESTROYED'].indexOf(machine.state.toUpperCase()) < 0) {
         return cb();
       }
 
-      const instance = {
-        name: machine.name,
-        status,
-        machineId: machine.id,
-        ips: machine.ips
+
+      const payload = {
+        id: service.id,
+        instances: instances.filter(({ id }) => {
+          return id !== instance.id;
+        })
       };
 
-      console.log('-> creating instance', util.inspect(instance));
-      return this._data.createInstance(instance, handleError((_, instance) => {
-        return updateService({
-          id: service.id,
-          instances: instances.concat(instance)
-        }, cb);
-      }));
+      console.log('-> updating service', Util.inspect(payload));
+      this._data.updateService(payload, cb);
+    });
+  }
+
+  resolveChange ({ deploymentGroup, version, service, instances, machine }, cb) {
+    console.error(`-> resolving change for machine ${machine.name}`);
+
+    const SERVICE_STATUS = Get(service, 'status', 'UNKNOWN').toUpperCase();
+    const MACHINE_STATUS = Get(machine, 'state', 'UNKNOWN').toUpperCase();
+
+    const hasPlan = Boolean(Get(version, 'plan.hasPlan', true));
+    const serviceName = service.name;
+
+    console.error(`-> detected meta for machine ${machine.name} ${Util.inspect({
+      SERVICE_STATUS,
+      MACHINE_STATUS,
+      hasPlan,
+      serviceName
+    })}`);
+
+    const ActionResolvers = {
+      '_CREATE_OR_REMOVE': (action, cb) => {
+        console.error(`-> got _CREATE_OR_REMOVE action for "${machine.name}"`);
+
+        let processed = ForceArray(action.processed);
+        const completed = processed.length === action.toProcess;
+
+        if (completed) {
+          console.error('-> action was already completed');
+          return cb(null, {
+            action,
+            completed: true
+          });
+        }
+
+        if (processed.indexOf(machine.id) >= 0) {
+          console.error('-> machine was already processed');
+          return cb(null, {
+            action,
+            completed
+          });
+        }
+
+        processed = processed.concat([machine.id]);
+
+        cb(null, {
+          action: Object.assign({}, action, {
+            processed
+          }),
+          completed: processed.length === action.toProcess
+        });
+      },
+      'NOOP': (action, cb) => {
+        console.error(`-> got NOOP action for "${machine.name}"`);
+
+        cb(null, {
+          action,
+          completed: true
+        });
+      },
+      // scenarios: scale down or removed service
+      // so far, the logic is the same for CREATE and REMOVE
+      'REMOVE': (action, cb) => {
+        console.error(`-> got REMOVE action for "${machine.name}"`);
+
+        if (ACTION_REMOVE_STATUSES.indexOf(MACHINE_STATUS) < 0) {
+          console.error(`-> since "${machine.name}" is "${MACHINE_STATUS}", nothing to do here`);
+
+          return cb(null, {
+            action,
+            completed: false
+          });
+        }
+
+        if (action.machines.indexOf(machine.id) < 0) {
+          console.error(`-> since "${machine.name}" didn't exist, no need to process its removal`);
+          return cb(null, {
+            action,
+            completed: false
+          });
+        }
+
+        ActionResolvers._CREATE_OR_REMOVE(action, cb);
+      },
+      // scenarios: scale up, recreate, create
+      // so far, the logic is the same for CREATE and REMOVE
+      'CREATE': (action, cb) => {
+        console.error(`-> got CREATE action for "${machine.name}"`);
+
+        if (ACTION_CREATE_STATUSES.indexOf(MACHINE_STATUS) < 0) {
+          console.error(`-> since "${machine.name}" is "${MACHINE_STATUS}", nothing to do here`);
+
+          return cb(null, {
+            action,
+            completed: false
+          });
+        }
+
+        if (action.machines.indexOf(machine.id) >= 0) {
+          console.error(`-> since "${machine.name}" already existed, no need to process its creation`);
+          return cb(null, {
+            action,
+            completed: false
+          });
+        }
+
+        ActionResolvers._CREATE_OR_REMOVE(action, cb);
+      },
+      'START': (action, cb) => {
+        console.error(`-> got START action for "${machine.name}". redirecting`);
+        return ActionResolvers.NOOP(action, cb);
+      }
     };
 
-    const update = (cb) => {
-      const updatedInstance = {
-        id: instance.id,
-        status: (machine.state || '').toUpperCase()
-      };
+    const toBeActiveServiceResolver = (cb) => {
+      VAsync.forEachParallel({
+        inputs: version.plan,
+        func: (action, next) => {
+          if (action.service !== serviceName) {
+            return next(null, {
+              action
+            });
+          }
 
-      console.log('-> updating instance', util.inspect(updatedInstance));
-      return this._data.updateInstance(updatedInstance, handleError(() => {
-        if (['DELETED', 'DESTROYED'].indexOf(machine.state.toUpperCase()) < 0) {
+          const ACTION_TYPE = Get(action, 'type', 'NOOP').toUpperCase();
+          ActionResolvers[ACTION_TYPE](action, next);
+        }
+      }, (err, result) => {
+        if (err) {
+          return cb(err);
+        }
+
+        const newActions = ForceArray(result.successes);
+
+        console.error(`-> got new actions for "${service.name}" ${Util.inspect(newActions)}`);
+
+        const newServiceActions = newActions.filter(({ action }) => {
+          return action.service === serviceName;
+        });
+
+        const isCompleted = newServiceActions.every(({ completed }) => {
+          return completed;
+        });
+
+        console.error(`-> are all actions for "${service.name}" completed? ${isCompleted}`);
+
+        const newPlan = newActions.map(({ action }) => {
+          return action;
+        });
+
+        VAsync.parallel({
+          funcs: [
+            (cb) => {
+              console.error(`-> updating Version ${version.id} with new plan ${Util.inspect(newPlan)}`);
+
+              this._data.updateVersion({
+                id: version.id,
+                plan: newPlan
+              }, cb);
+            },
+            (cb) => {
+              if (!isCompleted) {
+                return cb();
+              }
+
+              console.error(`-> updating Service ${service.name} with new status: ACTIVE`);
+
+              return this._data.updateService({
+                id: service.id,
+                status: 'ACTIVE'
+              }, cb);
+            }
+          ]
+        }, cb);
+      });
+    };
+
+    const ServiceResolvers = {
+      'ACTIVE': (cb) => {
+        console.error(`-> got ACTIVE service "${service.name}". nothing to do`);
+
+        cb();
+      },
+      'PROVISIONING': (cb) => {
+        console.error(`-> got PROVISIONING service "${service.name}"`);
+
+        toBeActiveServiceResolver(cb);
+      },
+      'SCALING': (cb) => {
+        console.error(`-> got SCALING service "${service.name}"`);
+
+        toBeActiveServiceResolver(cb);
+      },
+      'STOPPING': (cb) => {
+        console.error(`-> got STOPPING service "${service.name}"`);
+
+        if (SERVICE_STOPPING_STATUSES.indexOf(MACHINE_STATUS) < 0) {
           return cb();
         }
 
-        return setTimeout(() => {
-          return updateService({
-            id: service.id,
-            instances: instances.filter(({ id }) => {
-              return id !== instance.id;
-            })
-          }, cb);
-        }, this._frequency * 3);
-      }));
+        const isComplete = instances
+          .filter(({ machineId }) => {
+            return machineId !== machine.id;
+          })
+          .every(({ status }) => {
+            return SERVICE_STOPPING_STATUSES.indexOf(status) >= 0;
+          });
+
+        if (!isComplete) {
+          return cb();
+        }
+
+        this._data.updateService({
+          id: service.id,
+          status: 'STOPPED'
+        }, cb);
+      },
+      'STOPPED': (cb) => {
+        return ServiceResolvers.ACTIVE(cb);
+      },
+      'DELETING': (cb) => {
+        console.error(`-> got DELETING service "${service.name}"`);
+
+        if (SERVICE_DELETING_STATUSES.indexOf(MACHINE_STATUS) < 0) {
+          return cb();
+        }
+
+        const isComplete = instances
+          .filter(({ machineId }) => {
+            return machineId !== machine.id;
+          })
+          .every(({ status }) => {
+            return SERVICE_DELETING_STATUSES.indexOf(status) >= 0;
+          });
+
+        if (!isComplete) {
+          return cb();
+        }
+
+        VAsync.parallel({
+          funcs: [
+            (cb) => {
+              console.error(`-> updating Service ${service.name} to set it DELETED`);
+
+              this._data.updateService({
+                id: service.id,
+                status: 'DELETED'
+              }, cb);
+            },
+            (cb) => {
+              console.error(`-> updating DeploymentGroup ${deploymentGroup.id} to remove Service ${service.name}`);
+
+              deploymentGroup.services({}, (err, services) => {
+                if (err) {
+                  return cb(err);
+                }
+
+                this._data.updateDeploymentGroup({
+                  id: deploymentGroup.id,
+                  services: services.filter(({ id }) => {
+                    return service.id !== id;
+                  })
+                }, cb);
+              });
+            }
+          ]
+        }, cb);
+      },
+      'DELETED': (cb) => {
+        return ServiceResolvers.ACTIVE(cb);
+      },
+      'RESTARTING': (cb) => {
+        return ServiceResolvers.ACTIVE(cb);
+      },
+      'UNKNOWN': (cb) => {
+        return ServiceResolvers.ACTIVE(cb);
+      }
     };
 
-    return isNew ?
-      create(cb) :
-      update(cb);
+    const instance = Find(instances, ['machineId', machine.id]);
+
+    const isNew = instances
+      .every(({ machineId }) => {
+        return machine.id !== machineId;
+      });
+
+    const handleCreateOrUpdatedInstance = (err) => {
+      if (err) {
+        return cb(err);
+      }
+
+      console.error(`-> created/updated machine ${machine.name}`);
+
+      if (!hasPlan) {
+        console.error(`-> plan for ${service.name} is still not available. queuing`);
+        this._waitingForPlan.push(machine);
+        return cb();
+      }
+
+      const serviceResolver = ServiceResolvers[SERVICE_STATUS] ?
+        ServiceResolvers[SERVICE_STATUS] :
+        ServiceResolvers.UNKNOWN;
+
+      serviceResolver(cb);
+    };
+
+    const createOrUpdateInstance = isNew ?
+      this.createInstance :
+      this.updateInstance;
+
+    createOrUpdateInstance.call(this, { machine, instances, instance, service }, handleCreateOrUpdatedInstance);
   }
 
   onChange (machine) {
@@ -178,7 +545,7 @@ module.exports = class Watcher {
       return;
     }
 
-    console.log('-> `change` event received', util.inspect(machine));
+    console.log('-> `change` event received', Util.inspect(machine));
 
     const { id, tags = {} } = machine;
 
@@ -190,7 +557,9 @@ module.exports = class Watcher {
 
     // assert that it's a docker-compose project
     const isCompose = [DEPLOYMENT_GROUP, SERVICE, HASH].every(
-      (name) => { return tags[name]; }
+      (name) => {
+        return tags[name];
+      }
     );
 
     if (!isCompose) {
@@ -201,56 +570,78 @@ module.exports = class Watcher {
     const deploymentGroupName = tags[DEPLOYMENT_GROUP];
     const serviceName = tags[SERVICE];
 
-    const handleError = (next) => {
-      return (err, item) => {
+    const getInstancesAndVersion = ({
+      service,
+      deploymentGroup
+    }, cb) => {
+      this.getInstances(service, (err, instances) => {
         if (err) {
-          console.error(err);
-          return;
+          return cb(err);
         }
 
-        next(item);
-      };
-    };
+        this.getVersion(deploymentGroup, (err, version) => {
+          if (err) {
+            return cb(err);
+          }
 
-    const getInstances = (service, cb) => {
-      this.getInstances(service, handleError((instances) => {
-        return this.resolveChanges({
-          machine,
-          service,
-          instances
-        }, cb);
-      }));
-    };
-
-    // assert that service exists
-    const assertService = (deploymentGroupId) => {
-      this.pushToQueue({ serviceName, deploymentGroupId }, () => {
-        return new Promise((resolve) => {
-          this.getService({ serviceName, deploymentGroupId }, handleError((service) => {
-            if (!service) {
-              console.error(`Service "${serviceName}" form DeploymentGroup "${deploymentGroupName}" for machine ${id} not found`);
-              return;
-            }
-
-            getInstances(service, resolve);
-          }));
+          this.resolveChange({
+            deploymentGroup,
+            version,
+            service,
+            instances,
+            machine
+          }, cb);
         });
       });
     };
 
-    // assert that project managed by this portal
-    const assertDeploymentGroup = () => {
-      this.getDeploymentGroupId(deploymentGroupName, handleError((deploymentGroupId) => {
-        if (!deploymentGroupId) {
-          console.error(`DeploymentGroup "${deploymentGroupName}" for machine ${id} not found`);
-          return;
+    // assert that service exists
+    const assertService = (deploymentGroup, cb) => {
+      this.getService({
+        serviceName,
+        deploymentGroupId: deploymentGroup.id
+      }, (err, service) => {
+        if (err) {
+          return cb(err);
         }
 
-        assertService(deploymentGroupId);
-      }));
+        if (!service) {
+          console.error(`Service "${serviceName}" form DeploymentGroup "${deploymentGroupName}" for machine ${id} not found`);
+          return cb();
+        }
+
+        getInstancesAndVersion({
+          service,
+          deploymentGroup
+        }, cb);
+      });
     };
 
-    assertDeploymentGroup();
+    // assert that project managed by this portal
+    // also, lock into `deploymentGroupId` queue
+    this.getDeploymentGroup(deploymentGroupName, (err, deploymentGroup) => {
+      if (err) {
+        console.error(err);
+        return;
+      }
+
+      if (!deploymentGroup) {
+        console.error(`DeploymentGroup "${deploymentGroupName}" for machine ${id} not found`);
+        return;
+      }
+
+      this.pushToQueue(deploymentGroup.id, () => {
+        return new Promise((resolve) => {
+          assertService(deploymentGroup, (err) => {
+            if (err) {
+              console.error(err);
+            }
+
+            resolve();
+          });
+        });
+      });
+    });
   }
 };
 

packages/portal-api/package.json

@@ -37,6 +37,9 @@
     "graphi": "^2.2.1",
     "hoek": "^4.1.1",
     "joyent-cp-gql-schema": "^1.0.4",
+    "lodash.find": "^4.6.0",
+    "lodash.flatten": "^4.4.0",
+    "lodash.get": "^4.4.2",
     "lodash.uniqby": "^4.7.0",
     "param-case": "^2.1.1",
     "penseur": "^7.12.3",

scripts/release

@@ -46,7 +46,8 @@ const tasks = [
     task: [
       {
         title: 'Branch',
-        description: 'Checks if the current branch is `master`. To ignore use the `--any-branch` flag',
+        description:
+          'Checks if the current branch is `master`. To ignore use the `--any-branch` flag',
         filter: () => !argv['any-branch'],
         task: async () => {
           const branch = await execa.stdout('git', [
@@ -64,7 +65,8 @@ const tasks = [
       },
       {
         title: 'Working tree',
-        description: 'Checks if working tree is clean. To ignore use the `--force` flag',
+        description:
+          'Checks if working tree is clean. To ignore use the `--force` flag',
         filter: () => !argv.force,
         task: async () => {
           const status = await execa.stdout('git', ['status', '--porcelain']);
@@ -105,7 +107,8 @@ const tasks = [
       },
       {
         title: 'Publish',
-        description: 'Publish updated packages, based on the changes from last tag',
+        description:
+          'Publish updated packages, based on the changes from last tag',
         task: async ({ prefix }) => {
           const { publish } = await inquirer.prompt([
             {
@@ -176,7 +179,11 @@ const tasks = [
               name: 'release',
               type: 'confirm',
               default: false,
-              message: `${prefix}No lerna publish detected. Are you sure you want to release? \n ${prefix}${chalk.dim(`(${chalk.yellow(figures.warning)} this can have negative effects on future lerna publishes since it detects changes based on tags)`)}`
+              message: `${prefix}No lerna publish detected. Are you sure you want to release? \n ${prefix}${chalk.dim(
+                `(${chalk.yellow(
+                  figures.warning
+                )} this can have negative effects on future lerna publishes since it detects changes based on tags)`
+              )}`
             }
           ])
       },
@@ -348,16 +355,20 @@ const tasks = [
           const tagBody = `${EOL}${lastCommits}`;
 
           console.log(
-            `${prefix}${chalk.yellow('Tag Name: ')}\n${prefix}${prefix}${chalk.dim(tagName)}`
+            `${prefix}${chalk.yellow(
+              'Tag Name: '
+            )}\n${prefix}${prefix}${chalk.dim(tagName)}`
           );
 
           console.log(`${prefix}${chalk.yellow('Tag Description: ')}`);
 
           console.log(
-            `${chalk.dim(lastCommits
+            `${chalk.dim(
+              lastCommits
                 .split(/\n/)
                 .map(line => `${prefix}${prefix}${line}`)
-                .join('\n'))}`
+                .join('\n')
+            )}`
           );
 
           const { createTag } = await inquirer.prompt([
@@ -393,7 +404,9 @@ const tasks = [
             {
               name: 'pushTag',
               type: 'confirm',
-              message: `${prefix}Should ${chalk.yellow(tagName)} be pushed to origin?`
+              message: `${prefix}Should ${chalk.yellow(
+                tagName
+              )} be pushed to origin?`
             }
           ]);
 

yarn.lock

@@ -2167,6 +2167,13 @@ consulite@1.6.x:
     or-promise "1.x.x"
     wreck "10.x.x"
 
+consulite@^2.0.0:
+  version "2.0.0"
+  resolved "https://registry.yarnpkg.com/consulite/-/consulite-2.0.0.tgz#4d35228d24f70a8fb01fdb8ed921d4e54d8bbb16"
+  dependencies:
+    or-promise "1.x.x"
+    wreck "12.x.x"
+
 contains-path@^0.1.0:
   version "0.1.0"
   resolved "https://registry.yarnpkg.com/contains-path/-/contains-path-0.1.0.tgz#fe8cf184ff6670b6baef01a9d4861a5cbec4120a"