1
0
mirror of https://github.com/yldio/copilot.git synced 2024-11-14 23:30:05 +02:00

feat: support scale and cb in docker

This commit is contained in:
geek 2017-06-02 17:16:49 -05:00 committed by Sérgio Ramos
parent 372665f978
commit 37bd108f40
13 changed files with 544 additions and 2892 deletions

View File

@ -206,7 +206,7 @@
createDeploymentGroup(name: String!) : DeploymentGroup
updateDeploymentGroup(id: ID!, name: String!) : DeploymentGroup
provisionManifest(deploymentGroupId: ID!, type: ManifestType!, format: ManifestFormat!, raw: String!) : Version
provisionManifest(deploymentGroupId: ID!, type: ManifestType!, format: ManifestFormat!, raw: String!) : Manifest
scale(service: ID!, replicas: Int!) : Version
stopServices(ids: [ID]!) : [Service]

View File

@ -1,7 +0,0 @@
{
"env": {
"test": {
"plugins": ["istanbul"]
}
}
}

View File

@ -55,7 +55,7 @@ docker-compose-api
```js
const client = new DockerComposeClient();
const res = await client.provision({
client.provision({
projectName: 'docker-compose-client',
manifest: `
hello:
@ -65,6 +65,8 @@ const res = await client.provision({
node:
image: node:latest
`
}, (err, res, more) => {
// can be called multiple times, check 'more' if that is the case
});
```

View File

@ -0,0 +1,42 @@
const { Client } = require('zerorpc');
const EventEmitter = require('events');
module.exports = class DockerComposeClient extends EventEmitter {
constructor(endpoint = 'tcp://0.0.0.0:4242', timeout = 60 * 30) {
super();
this.client = new Client({
heartbeatInterval: 60 * 4 * 1000, // 4m
timeout // 30m
});
this.client.on('error', err => this.emit('error', err));
this.client.connect(endpoint);
}
_invoke(method, options, manifest, cb) {
return this.client.invoke(method, options, manifest, cb);
}
close() {
return this.client.close();
}
provision({ projectName, manifest }, cb) {
// eslint-disable-next-line camelcase
return this._invoke('up', { project_name: projectName }, manifest, cb);
}
scale({ projectName, services, manifest }, cb) {
const options = {
// eslint-disable-next-line camelcase
project_name: projectName,
services: Object.keys(services).map(name => ({
name,
num: services[name]
}))
};
return this._invoke('scale', options, manifest, cb);
}
};

View File

@ -3,48 +3,21 @@
"version": "1.0.4",
"license": "MPL-2.0",
"repository": "github:yldio/joyent-portal",
"main": "src/index.js",
"main": "lib",
"scripts": {
"lint": "eslint . --fix",
"lint-ci": "eslint . --format junit --output-file $CIRCLE_TEST_REPORTS/lint/docker-compose-client.xml",
"test": "cross-env NODE_ENV=test nyc --reporter=lcov --reporter=text ava",
"test-ci": "cross-env NODE_ENV=test nyc --report-dir=$CIRCLE_ARTIFACTS/docker-compose-client --reporter=lcov --reporter=text ava --tap | tap-xunit > $CIRCLE_TEST_REPORTS/test/docker-compose-client.xml"
"test": "lab -t 100",
"test-ci": "lab -t 100 -r console -o stdout -r tap -o $CIRCLE_TEST_REPORTS/test/docker-compose-client.xml"
},
"dependencies": {
"apr-awaitify": "^1.0.4",
"zerorpc": "^0.9.7"
},
"devDependencies": {
"apr-intercept": "^1.0.4",
"ava": "0.19.1",
"babel-plugin-istanbul": "^4.1.3",
"babel-register": "^6.24.1",
"cross-env": "^5.0.0",
"code": "^4.0.0",
"eslint": "^3.19.0",
"eslint-config-joyent-portal": "1.0.0",
"js-yaml": "^3.8.4",
"nyc": "^10.3.2",
"tap-xunit": "^1.7.0"
},
"nyc": {
"sourceMap": false,
"instrument": false
},
"babel": {
"sourceMaps": "inline",
"env": {
"test": {
"plugins": [
"istanbul"
]
}
}
},
"ava": {
"tap": true,
"require": [
"babel-register"
],
"babel": "inherit"
"lab": "^13.1.0"
}
}

View File

@ -1,50 +0,0 @@
const { Client } = require('zerorpc');
const { EventEmitter } = require('events');
const awaitify = require('apr-awaitify');
class DockerComposeClient extends EventEmitter {
constructor(endpoint = 'tcp://0.0.0.0:4242') {
super();
this.client = new Client({
heartbeatInterval: 60 * 4 * 1000, // 4m
timeout: 60 * 30 // 30m
});
this.client.connect(endpoint);
this.client.on('error', err => this.emit('error', err));
this._invoke = awaitify(this._invoke.bind(this));
}
// Why isn't client.connect async with error??
_invoke(name, ...args) {
return this.client.invoke(name, ...args);
}
close() {
return this.client.close();
}
provision({ projectName, manifest }) {
// eslint-disable-next-line camelcase
return this._invoke('up', { project_name: projectName }, manifest);
}
scale({ projectName, services, manifest }) {
return this._invoke(
'scale',
{
// eslint-disable-next-line camelcase
project_name: projectName,
services: Object.keys(services).map(name => ({
name,
num: services[name]
}))
},
manifest
);
}
}
module.exports = DockerComposeClient;

View File

@ -1,11 +1,23 @@
const { name } = require('../package.json');
'use strict';
const { expect } = require('code');
const Lab = require('lab');
const Package = require('../package.json');
const { safeLoad } = require('js-yaml');
const { Server } = require('zerorpc');
const intercept = require('apr-intercept');
const test = require('ava');
// Test shortcuts
const lab = exports.lab = Lab.script();
const after = lab.after;
const it = lab.it;
const projectName = Package.name;
const endpoint = 'tcp://0.0.0.0:4040';
const DockerComposeClient = require('../');
const client = new DockerComposeClient();
const client = new DockerComposeClient(endpoint);
const server = new Server({
// eslint-disable-next-line object-shorthand
@ -63,58 +75,65 @@ const server = new Server({
}
});
server.bind('tcp://0.0.0.0:4242');
server.bind(endpoint);
test('provision', async t => {
const [err, res] = await intercept(
client.provision({
projectName: name,
manifest: `
hello:
image: hello-world:latest
world:
image: consul:latest
node:
image: node:latest
`
})
);
it('provision()', (done) => {
const manifest = `
hello:
image: hello-world:latest
world:
image: consul:latest
node:
image: node:latest
`;
t.ifError(err);
client.provision({ projectName, manifest }, (err, res) => {
expect(err).to.not.exist();
t.deepEqual(res, {
projectName: name
expect(res.projectName).to.equal(projectName);
done();
});
});
test('scale', async t => {
const [err, res] = await intercept(
client.scale({
projectName: name,
services: {
hello: 2,
world: 3
},
manifest: `
hello:
image: hello-world:latest
world:
image: consul:latest
node:
image: node:latest
`
})
);
it('scale()', (done) => {
const manifest = `
hello:
image: hello-world:latest
world:
image: consul:latest
node:
image: node:latest
`;
t.ifError(err);
client.scale({
projectName,
services: {
hello: 2,
world: 3
},
manifest
}, (err, res) => {
expect(err).to.not.exist();
t.deepEqual(res, {
projectName: name,
services: [{ name: 'hello', num: 2 }, { name: 'world', num: 3 }]
expect(res).to.equal({
projectName,
services: [{ name: 'hello', num: 2 }, { name: 'world', num: 3 }]
});
done();
});
});
test.after(() => {
it('handles errors', (done) => {
client.once('error', (err) => {
expect(err).to.exist();
done();
});
client.client.emit('error', new Error('test'));
});
after((done) => {
client.close();
server.close();
done();
});

File diff suppressed because it is too large Load Diff

View File

@ -1,6 +1,7 @@
'use strict';
const EventEmitter = require('events');
const DockerClient = require('docker-compose-client');
const Hoek = require('hoek');
const Penseur = require('penseur');
const VAsync = require('vasync');
@ -34,6 +35,11 @@ module.exports = class Data extends EventEmitter {
// Penseur will assert that the options are correct
this._db = new Penseur.Db(settings.name, settings.db);
this._docker = new DockerClient(settings.dockerHost);
this._docker.on('error', (err) => {
this.emit('error', err);
});
}
connect (cb) {
@ -163,12 +169,24 @@ module.exports = class Data extends EventEmitter {
}
getDeploymentGroup (query, cb) {
this._db.deployment_groups.single(query, (err, deploymentGroup) => {
if (err) {
return cb(err);
}
this._db.deployment_groups.sync(() => {
this._db.deployment_groups.single(query, (err, deploymentGroup) => {
if (err) {
return cb(err);
}
cb(null, Transform.fromDeploymentGroup(deploymentGroup || {}));
if (!deploymentGroup) {
return cb(null, {});
}
if (!deploymentGroup.service_ids || !deploymentGroup.service_ids.length) {
return cb(null, Transform.fromDeploymentGroup(deploymentGroup));
}
this._db.services.get(deploymentGroup.service_ids, (err, services) => {
cb(err, Transform.fromDeploymentGroup(deploymentGroup, services));
});
});
});
}
@ -244,36 +262,205 @@ module.exports = class Data extends EventEmitter {
});
}
scale ({ id, replicas }, cb) {
Hoek.assert(id, 'service id is required');
Hoek.assert(typeof replicas === 'number' && replicas >= 0, 'replicas must be a number no less than 0');
// manifests
provisionManifest (clientManifest, cb) {
// insert manifest
// callback with manifest
// provision services
// get the service then get the deployment group
// use the deployment group to find the current version and manifest
// scale the service
// update the machine ids and instances
const manifest = Transform.toManifest(clientManifest);
this._db.manifests.insert(manifest, (err, key) => {
this._db.services.single({ id }, (err, service) => {
if (err) {
return cb(err);
}
setImmediate(() => {
const options = {
manifestServices: manifest.json.services || manifest.json,
deploymentGroupId: clientManifest.deploymentGroupId,
manifestId: key
};
this.provisionServices(options);
});
if (!service) {
return cb(new Error(`service not found for id: ${id}`));
}
manifest.id = key;
cb(null, Transform.fromManifest(manifest));
this._db.deployment_groups.single({ id: service.deployment_group_id }, (err, deployment_group) => {
if (err) {
return cb(err);
}
if (!deployment_group) {
return cb(new Error(`deployment group not found for service with service id: ${id}`));
}
this._db.versions.single({ id: deployment_group.version_id }, (err, version) => {
if (err) {
return cb(err);
}
if (!version) {
return cb(new Error(`version not found for service with service id: ${id}`));
}
this._db.manifests.single({ id: version.manifest_id }, (err, manifest) => {
if (err) {
return cb(err);
}
if (!manifest) {
return cb(new Error(`manifest not found for service with service id: ${id}`));
}
this._scale({ service, deployment_group, version, manifest, replicas }, cb);
});
});
});
});
}
_scale ({ service, deployment_group, version, manifest, replicas }, cb) {
let isFinished = false;
const finish = () => {
if (isFinished) {
return;
}
isFinished = true;
const machineIds = [];
for (let i = 1; i <= replicas; ++i) {
machineIds.push(`${deployment_group.name}_${service.name}_${i}`);
}
this._db.instances.remove(service.instance_ids, (err) => {
// emit error instead of returning early, this is a best effort to cleanup data
if (err) {
this.emit('error', err);
}
VAsync.forEachParallel({
func: (machineId, next) => {
const clientInstance = {
machineId,
status: 'CREATED',
name: service.name
};
this.createInstance(clientInstance, next);
},
inputs: machineIds
}, (err, results) => {
if (err) {
return cb(err);
}
const instanceIds = results.successes.map((instance) => {
return instance.id;
});
this._db.services.update(service.id, { instance_ids: instanceIds }, (err) => {
if (err) {
return cb(err);
}
const clientVersion = {
deploymentGroupId: deployment_group.id,
manifestId: manifest.id,
plan: {
running: true,
actions: [{
type: 'CREATE',
service: service.name,
machines: machineIds
}]
}
};
const scale = version.service_scales.find((scale) => {
return scale.service_name === service.name;
});
if (scale) {
scale.replicas = replicas;
} else {
version.service_scales.push({
service_name: service.name,
replicas
});
}
clientVersion.scales = version.service_scales.map(Transform.fromScale);
this.createVersion(clientVersion, cb);
});
});
});
};
const options = {
provisionName: deployment_group.name,
services: {},
manifest: manifest.raw
};
options.services[service.name] = replicas;
this._docker.scale(options, (err, res) => {
if (err) {
return cb(err);
}
finish();
});
}
// manifests
provisionManifest (clientManifest, cb) {
// get deployment group to verify it exists and get the name
// insert manifest
// callback with manifest
// provision containers and save service data
this.getDeploymentGroup({ id: clientManifest.deploymentGroupId }, (err, deploymentGroup) => {
if (err) {
return cb(err);
}
if (!deploymentGroup) {
return cb(new Error('Deployment group not found for manifest'));
}
const manifest = Transform.toManifest(clientManifest);
this._db.manifests.insert(manifest, (err, key) => {
if (err) {
return cb(err);
}
setImmediate(() => {
let isHandled = false;
this._docker.provision({ projectName: deploymentGroup.name, manifest: clientManifest.raw }, (err, res) => {
if (err) {
this.emit('error', err);
return;
}
// callback can execute multiple times, ensure responses are only handled once
if (isHandled) {
return;
}
isHandled = true;
const options = {
manifestServices: manifest.json.services || manifest.json,
deploymentGroup,
manifestId: key
};
this.provisionServices(options);
});
});
manifest.id = key;
cb(null, Transform.fromManifest(manifest));
});
});
}
getManifest ({ id }, cb) {
console.log(id);
this._db.manifests.single({ id }, (err, manifest) => {
if (err) {
return cb(err);
@ -298,8 +485,7 @@ module.exports = class Data extends EventEmitter {
// services
provisionServices ({ manifestServices, deploymentGroupId, manifestId }, cb) {
// call to docker and create containers
provisionServices ({ manifestServices, deploymentGroup, manifestId }, cb) {
// insert instance information
// insert service information
// insert version information -- will update deploymentGroups
@ -310,14 +496,12 @@ module.exports = class Data extends EventEmitter {
}
});
// TODO: call out to docker for each service and provision a new instance
VAsync.forEachPipeline({
func: (serviceName, next) => {
const manifestService = manifestServices[serviceName];
const clientInstance = {
name: serviceName,
machineId: 'unknown',
machineId: `${deploymentGroup.name}_${serviceName}_1`,
status: 'CREATED'
};
this.createInstance(clientInstance, (err, createdInstance) => {
@ -329,7 +513,7 @@ module.exports = class Data extends EventEmitter {
hash: manifestService.image,
name: serviceName,
slug: serviceName,
deploymentGroupId,
deploymentGroupId: deploymentGroup.id,
instances: [createdInstance]
};
@ -381,19 +565,19 @@ module.exports = class Data extends EventEmitter {
};
const clientVersion = {
deploymentGroupId,
deploymentGroupId: deploymentGroup.id,
manifestId,
scales,
plan,
serviceIds
};
this.createVersion(clientVersion, (err) => {
this.createVersion(clientVersion, (err, version) => {
if (err) {
return cb(err);
}
cb();
cb(null, version);
});
});
}

View File

@ -9,13 +9,14 @@
"bootstrap": "node ./bootstrap-data",
"lint": "belly-button --fix",
"lint-ci": "belly-button",
"test": "lab -t 40",
"test-ci": "echo 0"
"test": "lab -c",
"test-ci": "lab -c -r console -o stdout -r tap -o $CIRCLE_TEST_REPORTS/test/portal-data.xml"
},
"keywords": [],
"author": "wyatt",
"license": "MPL-2.0",
"dependencies": {
"docker-compose-client": "^1.0.7",
"hoek": "^4.1.1",
"penseur": "^7.8.1",
"vasync": "^1.6.4",

View File

@ -1,24 +1,2 @@
version: '2.1'
services:
# Service definition for Consul cluster with a minimum of 3 nodes.
# Nodes will use Triton CNS for the service (passed in via the CONSUL
# env var) to find each other and bootstrap the cluster.
consul:
image: autopilotpattern/consul:${TAG:-latest}
labels:
- triton.cns.services=consul
restart: always
mem_limit: 128m
ports:
- 8500
env_file:
- _env
network_mode: bridge
command: >
/usr/local/bin/containerpilot
/bin/consul agent -server
-bootstrap-expect 3
-config-dir=/etc/consul
-ui-dir /ui
hello:
image: hello-world:latest

View File

@ -378,7 +378,7 @@ describe('versions', () => {
expect(result.scales).to.equal(clientVersion.scales);
data.getVersions({ manifestId: clientVersion.manifestId }, (err, versions) => {
expect(err).to.not.exist();
expect(versions.length).to.equal(2);
expect(versions.length).to.equal(1);
done();
});
});
@ -502,8 +502,7 @@ describe('manifests', () => {
deploymentGroupId: deploymentGroup.id,
type: 'compose',
format: 'yml',
raw: 'docker compose raw contents',
json: { services: [] }
raw: internals.composeFile
};
data.provisionManifest(clientManifest, (err, result) => {
@ -677,3 +676,35 @@ describe('packages', () => {
});
});
});
// skipping by default since it takes so long
describe.skip('scale()', () => {
it('creates new instances of a service', { timeout: 180000 }, (done) => {
const data = new PortalData(internals.options);
data.connect((err) => {
expect(err).to.not.exist();
data.createDeploymentGroup({ name: 'something' }, (err, deploymentGroup) => {
expect(err).to.not.exist();
const clientManifest = {
deploymentGroupId: deploymentGroup.id,
type: 'compose',
format: 'yml',
raw: internals.composeFile
};
data.provisionManifest(clientManifest, (err, manifest) => {
expect(err).to.not.exist();
setTimeout(() => {
data.getDeploymentGroup({ id: deploymentGroup.id }, (err, deploymentGroup) => {
expect(err).to.not.exist();
data.scale({ id: deploymentGroup.services[0].id, replicas: 2 }, (err, version) => {
expect(err).to.not.exist();
expect(version).to.exist();
done();
});
});
}, 80000);
});
});
});
});
});

View File

@ -51,6 +51,10 @@ ansi-styles@^2.2.1:
version "2.2.1"
resolved "https://registry.yarnpkg.com/ansi-styles/-/ansi-styles-2.2.1.tgz#b432dd3358b634cf75e1e4664368240533c1ddbe"
apr-awaitify@^1.0.4:
version "1.0.4"
resolved "https://registry.yarnpkg.com/apr-awaitify/-/apr-awaitify-1.0.4.tgz#a72074a0d333e090bb120be9f710fd106b48a90a"
argparse@^1.0.7:
version "1.0.9"
resolved "https://registry.yarnpkg.com/argparse/-/argparse-1.0.9.tgz#73d83bc263f86e97f8cc4f6bae1b0e90a7d22c86"
@ -97,6 +101,10 @@ belly-button@^3.1.0:
glob "7.x.x"
insync "2.x.x"
bindings@~1.2.1:
version "1.2.1"
resolved "https://registry.yarnpkg.com/bindings/-/bindings-1.2.1.tgz#14ad6113812d2d37d72e67b4cacb4bb726505f11"
"bluebird@>= 2.3.2 < 3":
version "2.11.0"
resolved "https://registry.yarnpkg.com/bluebird/-/bluebird-2.11.0.tgz#534b9033c022c9579c56ba3b3e5a5caafbb650e1"
@ -244,6 +252,13 @@ diff@3.x.x:
version "3.2.0"
resolved "https://registry.yarnpkg.com/diff/-/diff-3.2.0.tgz#c9ce393a4b7cbd0b058a725c93df299027868ff9"
docker-compose-client@^1.0.7:
version "1.0.7"
resolved "https://registry.yarnpkg.com/docker-compose-client/-/docker-compose-client-1.0.7.tgz#a2f351aff998fd5323b9b6bb27d4400fff95e43c"
dependencies:
apr-awaitify "^1.0.4"
zerorpc "^0.9.7"
doctrine@^2.0.0:
version "2.0.0"
resolved "https://registry.yarnpkg.com/doctrine/-/doctrine-2.0.0.tgz#c73d8d2909d22291e1a007a395804da8b665fe63"
@ -737,10 +752,24 @@ ms@2.0.0:
version "2.0.0"
resolved "https://registry.yarnpkg.com/ms/-/ms-2.0.0.tgz#5608aeadfc00be6c2901df5f9861788de0d597c8"
msgpack@1.0.2:
version "1.0.2"
resolved "https://registry.yarnpkg.com/msgpack/-/msgpack-1.0.2.tgz#923e2c5cffa65c8418e9b228d1124793969c429c"
dependencies:
nan "^2.0.9"
mute-stream@0.0.5:
version "0.0.5"
resolved "https://registry.yarnpkg.com/mute-stream/-/mute-stream-0.0.5.tgz#8fbfabb0a98a253d3184331f9e8deb7372fac6c0"
nan@^2.0.9:
version "2.6.2"
resolved "https://registry.yarnpkg.com/nan/-/nan-2.6.2.tgz#e4ff34e6c95fdfb5aecc08de6596f43605a7db45"
nan@~2.3.0:
version "2.3.5"
resolved "https://registry.yarnpkg.com/nan/-/nan-2.3.5.tgz#822a0dc266290ce4cd3a12282ca3e7e364668a08"
natural-compare@^1.4.0:
version "1.4.0"
resolved "https://registry.yarnpkg.com/natural-compare/-/natural-compare-1.4.0.tgz#4abebfeed7541f2c27acfb29bdbbd15c8d5ba4f7"
@ -1059,6 +1088,10 @@ uglify-to-browserify@~1.0.0:
version "1.0.2"
resolved "https://registry.yarnpkg.com/uglify-to-browserify/-/uglify-to-browserify-1.0.2.tgz#6e0924d6bda6b5afe349e39a6d632850a0f882b7"
underscore@1.3.3:
version "1.3.3"
resolved "https://registry.yarnpkg.com/underscore/-/underscore-1.3.3.tgz#47ac53683daf832bfa952e1774417da47817ae42"
user-home@^2.0.0:
version "2.0.0"
resolved "https://registry.yarnpkg.com/user-home/-/user-home-2.0.0.tgz#9c70bfd8169bc1dcbf48604e0f04b8b49cde9e9f"
@ -1069,6 +1102,10 @@ util-deprecate@~1.0.1:
version "1.0.2"
resolved "https://registry.yarnpkg.com/util-deprecate/-/util-deprecate-1.0.2.tgz#450d4dc9fa70de732762fbd2d4a28981419a0ccf"
uuid@^3.0.0:
version "3.0.1"
resolved "https://registry.yarnpkg.com/uuid/-/uuid-3.0.1.tgz#6544bba2dfda8c1cf17e629a3a305e2bb1fee6c1"
vasync@^1.6.4:
version "1.6.4"
resolved "https://registry.yarnpkg.com/vasync/-/vasync-1.6.4.tgz#dfe93616ad0e7ae801b332a9d88bfc5cdc8e1d1f"
@ -1126,3 +1163,19 @@ yargs@~3.10.0:
cliui "^2.1.0"
decamelize "^1.0.0"
window-size "0.1.0"
zerorpc@^0.9.7:
version "0.9.7"
resolved "https://registry.yarnpkg.com/zerorpc/-/zerorpc-0.9.7.tgz#64ddb32ce8c934bea5434ec81ca22e971045a860"
dependencies:
msgpack "1.0.2"
underscore "1.3.3"
uuid "^3.0.0"
zmq "2.x"
zmq@2.x:
version "2.15.3"
resolved "https://registry.yarnpkg.com/zmq/-/zmq-2.15.3.tgz#66c6de82cc36b09734b820703776490a6fbbe624"
dependencies:
bindings "~1.2.1"
nan "~2.3.0"