joyent-portal/packages/portal-api/lib/watch/machines.js

655 lines
17 KiB
JavaScript

'use strict';
// const Assert = require('assert');
const TritonWatch = require('triton-watch');
const Get = require('lodash.get');
const Find = require('lodash.find');
const Boom = require('boom');
const Util = require('util');
const ForceArray = require('force-array');
const VAsync = require('vasync');
const Queue = require('./queue');
const DEPLOYMENT_GROUP = 'docker:label:com.docker.compose.project';
const SERVICE = 'docker:label:com.docker.compose.service';
const HASH = 'docker:label:com.docker.compose.config-hash';
const ACTION_REMOVE_STATUSES = [
'STOPPING',
'STOPPED',
'OFFLINE',
'DELETED',
'DESTROYED',
'FAILED',
'INCOMPLETE',
'UNKNOWN'
];
const ACTION_CREATE_STATUSES = [
'READY',
'ACTIVE',
'RUNNING',
'STOPPED',
'OFFLINE',
'FAILED',
'INCOMPLETE',
'UNKNOWN'
];
const SERVICE_STOPPING_STATUSES = [
'STOPPED',
'OFFLINE',
'FAILED',
'INCOMPLETE',
'UNKNOWN'
];
const SERVICE_DELETING_STATUSES = [
'DELETED',
'DESTROYED',
'FAILED',
'INCOMPLETE',
'UNKNOWN'
];
const isNotFound = (err) => {
return err && (err['typeof'] === Boom.notFound);
};
module.exports = class MachineWatcher {
constructor (options) {
options = options || {};
// todo assert options
this._data = options.data;
this._server = options.server;
this._frequency = 200;
this._tritonWatch = new TritonWatch({
frequency: this._frequency,
triton: {
profile: {
url: options.url || process.env.SDC_URL,
account: options.account || process.env.SDC_ACCOUNT,
keyId: options.keyId || process.env.SDC_KEY_ID
}
}
});
this._waitingForPlan = [];
this._isTritonWatchPolling = false;
this._tritonWatch.on('change', (machine) => {
return this.onChange(machine);
});
this._tritonWatch.on('all', (machines) => {
machines.forEach((machine) => {
this.onChange(machine);
});
});
}
poll () {
if (!this._isTritonWatchPolling) {
this._tritonWatch.poll();
this._isTritonWatchPolling = true;
}
if (this._isWaitingPolling) {
return;
}
this._isWaitingPolling = true;
setTimeout(() => {
this._isWaitingPolling = false;
this._checkForWaiting();
}, this._frequency);
}
_checkForWaiting () {
this._waitingForPlan.forEach(this.onChange);
}
getContainers () {
return this._tritonWatch.getContainers();
}
getDeploymentGroup (name, cb) {
this._data.getDeploymentGroup({ name }, (err, deploymentGroup) => {
if (err) {
return cb(err);
}
return cb(null, deploymentGroup);
});
}
getService ({ serviceName, deploymentGroupId }, cb) {
this._data.getServices({ name: serviceName, deploymentGroupId }, (err, services) => {
if (err) {
return cb(err);
}
if (!services || !services.length) {
return cb();
}
return cb(null, services.pop());
});
}
getInstances (service, cb) {
service.instances()
.then((instances) => {
return cb(null, instances);
})
.catch((err) => {
return cb(err);
});
}
getVersion (deploymentGroup, cb) {
if (typeof deploymentGroup.version !== 'function') {
return cb(new Error('version must be a function'));
}
deploymentGroup.version()
.then((version) => {
return cb(null, version);
})
.catch((err) => {
return cb(err);
});
}
createInstance ({ deploymentGroup, machine, instances, service }, cb) {
this._server.log(['debug', 'error'], `-> detected that machine ${machine.name} was created`);
const status = (machine.state || '').toUpperCase();
if (status === 'DELETED') {
return cb();
}
const instance = {
name: machine.name,
status,
deploymentGroupId: deploymentGroup.id,
machineId: machine.id
};
this._server.log(['debug'], '-> creating instance', Util.inspect(instance));
this._data.createInstance(instance, (err, instance) => {
if (err) {
return cb(err);
}
const payload = {
id: service.id,
instances: instances.concat(instance)
};
this._server.log(['debug'], '-> updating service', Util.inspect(payload));
this._data.updateService(payload, cb);
});
}
updateInstance ({ machine, instance, instances, service }, cb) {
console.error(`-> detected that machine ${machine.name} was updated`);
const updatedInstance = {
id: instance.id,
status: (machine.state || '').toUpperCase()
};
this._server.log(['debug'], '-> updating instance', Util.inspect(updatedInstance));
this._data.updateInstance(updatedInstance, (err) => {
if (err) {
return cb(err);
}
if (['DELETED', 'DESTROYED'].indexOf(machine.state.toUpperCase()) < 0) {
return cb();
}
const payload = {
id: service.id,
instances: instances.filter(({ id }) => {
return id !== instance.id;
})
};
this._server.log(['debug'], '-> updating service', Util.inspect(payload));
this._data.updateService(payload, cb);
});
}
resolveChange ({ deploymentGroup, version, service, instances, machine }, cb) {
console.error(`-> resolving change for machine ${machine.name}`);
const SERVICE_STATUS = Get(service, 'status', 'UNKNOWN').toUpperCase();
const MACHINE_STATUS = Get(machine, 'state', 'UNKNOWN').toUpperCase();
const hasPlan = Boolean(Get(version, 'plan.hasPlan', true));
const serviceName = service.name;
console.error(`-> detected meta for machine ${machine.name} ${Util.inspect({
SERVICE_STATUS,
MACHINE_STATUS,
hasPlan,
serviceName
})}`);
const ActionResolvers = {
'_CREATE_OR_REMOVE': (action, cb) => {
console.error(`-> got _CREATE_OR_REMOVE action for "${machine.name}"`);
let processed = ForceArray(action.processed);
const completed = processed.length === action.toProcess;
if (completed) {
console.error('-> action was already completed');
return cb(null, {
action,
completed: true
});
}
if (processed.indexOf(machine.id) >= 0) {
console.error('-> machine was already processed');
return cb(null, {
action,
completed
});
}
processed = processed.concat([machine.id]);
cb(null, {
action: Object.assign({}, action, {
processed
}),
completed: processed.length === action.toProcess
});
},
'NOOP': (action, cb) => {
console.error(`-> got NOOP action for "${machine.name}"`);
cb(null, {
action,
completed: true
});
},
// scenarios: scale down or removed service
// so far, the logic is the same for CREATE and REMOVE
'REMOVE': (action, cb) => {
console.error(`-> got REMOVE action for "${machine.name}"`);
if (ACTION_REMOVE_STATUSES.indexOf(MACHINE_STATUS) < 0) {
console.error(`-> since "${machine.name}" is "${MACHINE_STATUS}", nothing to do here`);
return cb(null, {
action,
completed: false
});
}
if (action.machines.indexOf(machine.id) < 0) {
console.error(`-> since "${machine.name}" didn't exist, no need to process its removal`);
return cb(null, {
action,
completed: false
});
}
ActionResolvers._CREATE_OR_REMOVE(action, cb);
},
// scenarios: scale up, recreate, create
// so far, the logic is the same for CREATE and REMOVE
'CREATE': (action, cb) => {
console.error(`-> got CREATE action for "${machine.name}"`);
if (ACTION_CREATE_STATUSES.indexOf(MACHINE_STATUS) < 0) {
console.error(`-> since "${machine.name}" is "${MACHINE_STATUS}", nothing to do here`);
return cb(null, {
action,
completed: false
});
}
if (action.machines.indexOf(machine.id) >= 0) {
console.error(`-> since "${machine.name}" already existed, no need to process its creation`);
return cb(null, {
action,
completed: false
});
}
ActionResolvers._CREATE_OR_REMOVE(action, cb);
},
'START': (action, cb) => {
console.error(`-> got START action for "${machine.name}". redirecting`);
return ActionResolvers.NOOP(action, cb);
}
};
const toBeActiveServiceResolver = (cb) => {
VAsync.forEachParallel({
inputs: version.plan,
func: (action, next) => {
if (action.service !== serviceName) {
return next(null, {
action
});
}
const ACTION_TYPE = Get(action, 'type', 'NOOP').toUpperCase();
ActionResolvers[ACTION_TYPE](action, next);
}
}, (err, result) => {
if (err) {
return cb(err);
}
const newActions = ForceArray(result.successes);
console.error(`-> got new actions for "${service.name}" ${Util.inspect(newActions)}`);
const newServiceActions = newActions.filter(({ action }) => {
return action.service === serviceName;
});
const isCompleted = newServiceActions.every(({ completed }) => {
return completed;
});
console.error(`-> are all actions for "${service.name}" completed? ${isCompleted}`);
const newPlan = newActions.map(({ action }) => {
return action;
});
VAsync.parallel({
funcs: [
(cb) => {
console.error(`-> updating Version ${version.id} with new plan ${Util.inspect(newPlan)}`);
this._data.updateVersion({
id: version.id,
plan: newPlan
}, cb);
},
(cb) => {
if (!isCompleted) {
return cb();
}
console.error(`-> updating Service ${service.name} with new status: ACTIVE`);
return this._data.updateService({
id: service.id,
status: 'ACTIVE'
}, cb);
}
]
}, cb);
});
};
const ServiceResolvers = {
'ACTIVE': (cb) => {
console.error(`-> got ACTIVE service "${service.name}". nothing to do`);
cb();
},
'PROVISIONING': (cb) => {
console.error(`-> got PROVISIONING service "${service.name}"`);
toBeActiveServiceResolver(cb);
},
'SCALING': (cb) => {
console.error(`-> got SCALING service "${service.name}"`);
toBeActiveServiceResolver(cb);
},
'STOPPING': (cb) => {
console.error(`-> got STOPPING service "${service.name}"`);
if (SERVICE_STOPPING_STATUSES.indexOf(MACHINE_STATUS) < 0) {
return cb();
}
const isComplete = instances
.filter(({ machineId }) => {
return machineId !== machine.id;
})
.every(({ status }) => {
return SERVICE_STOPPING_STATUSES.indexOf(status) >= 0;
});
if (!isComplete) {
return cb();
}
this._data.updateService({
id: service.id,
status: 'STOPPED'
}, cb);
},
'STOPPED': (cb) => {
return ServiceResolvers.ACTIVE(cb);
},
'DELETING': (cb) => {
console.error(`-> got DELETING service "${service.name}"`);
if (SERVICE_DELETING_STATUSES.indexOf(MACHINE_STATUS) < 0) {
return cb();
}
const isComplete = instances
.filter(({ machineId }) => {
return machineId !== machine.id;
})
.every(({ status }) => {
return SERVICE_DELETING_STATUSES.indexOf(status) >= 0;
});
if (!isComplete) {
return cb();
}
VAsync.parallel({
funcs: [
(cb) => {
console.error(`-> updating Service ${service.name} to set it DELETED`);
this._data.updateService({
id: service.id,
status: 'DELETED'
}, cb);
},
(cb) => {
console.error(`-> updating DeploymentGroup ${deploymentGroup.id} to remove Service ${service.name}`);
deploymentGroup.services({}, (err, services) => {
if (err) {
return cb(err);
}
this._data.updateDeploymentGroup({
id: deploymentGroup.id,
services: services.filter(({ id }) => {
return service.id !== id;
})
}, cb);
});
}
]
}, cb);
},
'DELETED': (cb) => {
return ServiceResolvers.ACTIVE(cb);
},
'RESTARTING': (cb) => {
return ServiceResolvers.ACTIVE(cb);
},
'UNKNOWN': (cb) => {
return ServiceResolvers.ACTIVE(cb);
}
};
const instance = Find(instances, ['machineId', machine.id]);
const isNew = instances
.every(({ machineId }) => {
return machine.id !== machineId;
});
const handleCreateOrUpdatedInstance = (err) => {
if (err) {
return cb(err);
}
console.error(`-> created/updated machine ${machine.name}`);
if (!hasPlan) {
console.error(`-> plan for ${service.name} is still not available. queuing`);
this._waitingForPlan.push(machine);
return cb();
}
const serviceResolver = ServiceResolvers[SERVICE_STATUS] ?
ServiceResolvers[SERVICE_STATUS] :
ServiceResolvers.UNKNOWN;
serviceResolver(cb);
};
const createOrUpdateInstance = isNew ?
this.createInstance :
this.updateInstance;
createOrUpdateInstance.call(this, {
deploymentGroup,
machine,
instances,
instance,
service
}, handleCreateOrUpdatedInstance);
}
onChange (machine) {
if (!machine) {
console.error('-> `change` event received without machine data');
return;
}
// console.log('-> `change` event received', Util.inspect(machine));
const { id, tags = {} } = machine;
// assert id existence
if (!id) {
console.error('-> `change` event received for a machine without `id`');
return;
}
// assert that it's a docker-compose project
const isCompose = [DEPLOYMENT_GROUP, SERVICE, HASH].every(
(name) => {
return tags[name];
}
);
if (!isCompose) {
console.error(`-> Changed machine ${id} was not provisioned by docker-compose`);
return;
}
const deploymentGroupName = tags[DEPLOYMENT_GROUP];
const serviceName = tags[SERVICE];
const getInstancesAndVersion = ({
service,
deploymentGroup
}, cb) => {
this.getInstances(service, (err, instances = []) => {
if (err && !isNotFound(err)) {
return cb(err);
}
this.getVersion(deploymentGroup, (err, version) => {
if (err) {
return cb(err);
}
this.resolveChange({
deploymentGroup,
version,
service,
instances,
machine
}, cb);
});
});
};
// assert that service exists
const assertService = (deploymentGroup, cb) => {
this.getService({
serviceName,
deploymentGroupId: deploymentGroup.id
}, (err, service) => {
if (isNotFound(err) || !service) {
console.error(`Service "${serviceName}" form DeploymentGroup "${deploymentGroupName}" for machine ${id} not found`);
return cb();
}
if (err) {
return cb(err);
}
getInstancesAndVersion({
service,
deploymentGroup
}, cb);
});
};
// assert that project managed by this portal
// also, lock into `deploymentGroupId` queue
this.getDeploymentGroup(deploymentGroupName, (err, deploymentGroup) => {
if (isNotFound(err) || !deploymentGroup) {
console.error(`DeploymentGroup "${deploymentGroupName}" for machine ${id} not found`);
return;
}
if (err) {
console.error(err);
return;
}
Queue(deploymentGroup.id, () => {
return new Promise((resolve) => {
assertService(deploymentGroup, (err) => {
if (err) {
console.error(err);
}
resolve();
});
});
});
});
}
};
module.exports.DEPLOYMENT_GROUP = DEPLOYMENT_GROUP;
module.exports.SERVICE = SERVICE;
module.exports.HASH = HASH;