Spearhead/joyent-portal
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

feat: support scale and cb in docker

Browse Source
This commit is contained in:
geek 2017-06-02 17:16:49 -05:00 committed by Sérgio Ramos
parent 372665f978
commit 37bd108f40
13 changed files with 544 additions and 2892 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
packages/cp-gql-schema/schema.gql
View File

@ -206,7 +206,7 @@
createDeploymentGroup(name: String!) : DeploymentGroup
updateDeploymentGroup(id: ID!, name: String!) : DeploymentGroup
provisionManifest(deploymentGroupId: ID!, type: ManifestType!, format: ManifestFormat!, raw: String!) : Version
provisionManifest(deploymentGroupId: ID!, type: ManifestType!, format: ManifestFormat!, raw: String!) : Manifest
scale(service: ID!, replicas: Int!) : Version
stopServices(ids: [ID]!) : [Service]

7
packages/docker-compose-client/.baberc
View File

@ -1,7 +0,0 @@
{
"env": {
"test": {
"plugins": ["istanbul"]
}
}
}

4
packages/docker-compose-client/README.md
View File

@ -55,7 +55,7 @@ docker-compose-api
```js
const client = new DockerComposeClient();
const res = await client.provision({
client.provision({
projectName: 'docker-compose-client',
manifest: `
hello:
@ -65,6 +65,8 @@ const res = await client.provision({
node:
image: node:latest
`
}, (err, res, more) => {
// can be called multiple times, check 'more' if that is the case
});
```

42
packages/docker-compose-client/lib/index.js Normal file
View File

@ -0,0 +1,42 @@
const { Client } = require('zerorpc');
const EventEmitter = require('events');
module.exports = class DockerComposeClient extends EventEmitter {
constructor(endpoint = 'tcp://0.0.0.0:4242', timeout = 60 * 30) {
super();
this.client = new Client({
heartbeatInterval: 60 * 4 * 1000, // 4m
timeout // 30m
});
this.client.on('error', err => this.emit('error', err));
this.client.connect(endpoint);
}
_invoke(method, options, manifest, cb) {
return this.client.invoke(method, options, manifest, cb);
}
close() {
return this.client.close();
}
provision({ projectName, manifest }, cb) {
// eslint-disable-next-line camelcase
return this._invoke('up', { project_name: projectName }, manifest, cb);
}
scale({ projectName, services, manifest }, cb) {
const options = {
// eslint-disable-next-line camelcase
project_name: projectName,
services: Object.keys(services).map(name => ({
name,
num: services[name]
}))
};
return this._invoke('scale', options, manifest, cb);
}
};

37
packages/docker-compose-client/package.json
View File

@ -3,48 +3,21 @@
"version": "1.0.4",
"license": "MPL-2.0",
"repository": "github:yldio/joyent-portal",
"main": "src/index.js",
"main": "lib",
"scripts": {
"lint": "eslint . --fix",
"lint-ci": "eslint . --format junit --output-file $CIRCLE_TEST_REPORTS/lint/docker-compose-client.xml",
"test": "cross-env NODE_ENV=test nyc --reporter=lcov --reporter=text ava",
"test-ci": "cross-env NODE_ENV=test nyc --report-dir=$CIRCLE_ARTIFACTS/docker-compose-client --reporter=lcov --reporter=text ava --tap | tap-xunit > $CIRCLE_TEST_REPORTS/test/docker-compose-client.xml"
"test": "lab -t 100",
"test-ci": "lab -t 100 -r console -o stdout -r tap -o $CIRCLE_TEST_REPORTS/test/docker-compose-client.xml"
},
"dependencies": {
"apr-awaitify": "^1.0.4",
"zerorpc": "^0.9.7"
},
"devDependencies": {
"apr-intercept": "^1.0.4",
"ava": "0.19.1",
"babel-plugin-istanbul": "^4.1.3",
"babel-register": "^6.24.1",
"cross-env": "^5.0.0",
"code": "^4.0.0",
"eslint": "^3.19.0",
"eslint-config-joyent-portal": "1.0.0",
"js-yaml": "^3.8.4",
"nyc": "^10.3.2",
"tap-xunit": "^1.7.0"
},
"nyc": {
"sourceMap": false,
"instrument": false
},
"babel": {
"sourceMaps": "inline",
"env": {
"test": {
"plugins": [
"istanbul"
]
}
}
},
"ava": {
"tap": true,
"require": [
"babel-register"
],
"babel": "inherit"
"lab": "^13.1.0"
}
}

50
packages/docker-compose-client/src/index.js
View File

@ -1,50 +0,0 @@
const { Client } = require('zerorpc');
const { EventEmitter } = require('events');
const awaitify = require('apr-awaitify');
class DockerComposeClient extends EventEmitter {
constructor(endpoint = 'tcp://0.0.0.0:4242') {
super();
this.client = new Client({
heartbeatInterval: 60 * 4 * 1000, // 4m
timeout: 60 * 30 // 30m
});
this.client.connect(endpoint);
this.client.on('error', err => this.emit('error', err));
this._invoke = awaitify(this._invoke.bind(this));
}
// Why isn't client.connect async with error??
_invoke(name, ...args) {
return this.client.invoke(name, ...args);
}
close() {
return this.client.close();
}
provision({ projectName, manifest }) {
// eslint-disable-next-line camelcase
return this._invoke('up', { project_name: projectName }, manifest);
}
scale({ projectName, services, manifest }) {
return this._invoke(
'scale',
{
// eslint-disable-next-line camelcase
project_name: projectName,
services: Object.keys(services).map(name => ({
name,
num: services[name]
}))
},
manifest
);
}
}
module.exports = DockerComposeClient;

109
packages/docker-compose-client/test/index.js
View File

@ -1,11 +1,23 @@
const { name } = require('../package.json');
'use strict';
const { expect } = require('code');
const Lab = require('lab');
const Package = require('../package.json');
const { safeLoad } = require('js-yaml');
const { Server } = require('zerorpc');
const intercept = require('apr-intercept');
const test = require('ava');
// Test shortcuts
const lab = exports.lab = Lab.script();
const after = lab.after;
const it = lab.it;
const projectName = Package.name;
const endpoint = 'tcp://0.0.0.0:4040';
const DockerComposeClient = require('../');
const client = new DockerComposeClient();
const client = new DockerComposeClient(endpoint);
const server = new Server({
// eslint-disable-next-line object-shorthand
@ -63,58 +75,65 @@ const server = new Server({
}
});
server.bind('tcp://0.0.0.0:4242');
server.bind(endpoint);
test('provision', async t => {
const [err, res] = await intercept(
client.provision({
projectName: name,
manifest: `
hello:
image: hello-world:latest
world:
image: consul:latest
node:
image: node:latest
`
})
);
it('provision()', (done) => {
const manifest = `
hello:
image: hello-world:latest
world:
image: consul:latest
node:
image: node:latest
`;
t.ifError(err);
client.provision({ projectName, manifest }, (err, res) => {
expect(err).to.not.exist();
t.deepEqual(res, {
projectName: name
expect(res.projectName).to.equal(projectName);
done();
});
});
test('scale', async t => {
const [err, res] = await intercept(
client.scale({
projectName: name,
services: {
hello: 2,
world: 3
},
manifest: `
hello:
image: hello-world:latest
world:
image: consul:latest
node:
image: node:latest
`
})
);
it('scale()', (done) => {
const manifest = `
hello:
image: hello-world:latest
world:
image: consul:latest
node:
image: node:latest
`;
t.ifError(err);
client.scale({
projectName,
services: {
hello: 2,
world: 3
},
manifest
}, (err, res) => {
expect(err).to.not.exist();
t.deepEqual(res, {
projectName: name,
services: [{ name: 'hello', num: 2 }, { name: 'world', num: 3 }]
expect(res).to.equal({
projectName,
services: [{ name: 'hello', num: 2 }, { name: 'world', num: 3 }]
});
done();
});
});
test.after(() => {
it('handles errors', (done) => {
client.once('error', (err) => {
expect(err).to.exist();
done();
});
client.client.emit('error', new Error('test'));
});
after((done) => {
client.close();
server.close();
done();
});

2816
packages/docker-compose-client/yarn.lock
View File

File diff suppressed because it is too large Load Diff

248
packages/portal-data/lib/index.js
View File

@ -1,6 +1,7 @@
'use strict';
const EventEmitter = require('events');
const DockerClient = require('docker-compose-client');
const Hoek = require('hoek');
const Penseur = require('penseur');
const VAsync = require('vasync');
@ -34,6 +35,11 @@ module.exports = class Data extends EventEmitter {
// Penseur will assert that the options are correct
this._db = new Penseur.Db(settings.name, settings.db);
this._docker = new DockerClient(settings.dockerHost);
this._docker.on('error', (err) => {
this.emit('error', err);
});
}
connect (cb) {
@ -163,12 +169,24 @@ module.exports = class Data extends EventEmitter {
}
getDeploymentGroup (query, cb) {
this._db.deployment_groups.single(query, (err, deploymentGroup) => {
if (err) {
return cb(err);
}
this._db.deployment_groups.sync(() => {
this._db.deployment_groups.single(query, (err, deploymentGroup) => {
if (err) {
return cb(err);
}
cb(null, Transform.fromDeploymentGroup(deploymentGroup || {}));
if (!deploymentGroup) {
return cb(null, {});
}
if (!deploymentGroup.service_ids || !deploymentGroup.service_ids.length) {
return cb(null, Transform.fromDeploymentGroup(deploymentGroup));
}
this._db.services.get(deploymentGroup.service_ids, (err, services) => {
cb(err, Transform.fromDeploymentGroup(deploymentGroup, services));
});
});
});
}
@ -244,36 +262,205 @@ module.exports = class Data extends EventEmitter {
});
}
scale ({ id, replicas }, cb) {
Hoek.assert(id, 'service id is required');
Hoek.assert(typeof replicas === 'number' && replicas >= 0, 'replicas must be a number no less than 0');
// manifests
provisionManifest (clientManifest, cb) {
// insert manifest
// callback with manifest
// provision services
// get the service then get the deployment group
// use the deployment group to find the current version and manifest
// scale the service
// update the machine ids and instances
const manifest = Transform.toManifest(clientManifest);
this._db.manifests.insert(manifest, (err, key) => {
this._db.services.single({ id }, (err, service) => {
if (err) {
return cb(err);
}
setImmediate(() => {
const options = {
manifestServices: manifest.json.services || manifest.json,
deploymentGroupId: clientManifest.deploymentGroupId,
manifestId: key
};
this.provisionServices(options);
});
if (!service) {
return cb(new Error(`service not found for id: ${id}`));
}
manifest.id = key;
cb(null, Transform.fromManifest(manifest));
this._db.deployment_groups.single({ id: service.deployment_group_id }, (err, deployment_group) => {
if (err) {
return cb(err);
}
if (!deployment_group) {
return cb(new Error(`deployment group not found for service with service id: ${id}`));
}
this._db.versions.single({ id: deployment_group.version_id }, (err, version) => {
if (err) {
return cb(err);
}
if (!version) {
return cb(new Error(`version not found for service with service id: ${id}`));
}
this._db.manifests.single({ id: version.manifest_id }, (err, manifest) => {
if (err) {
return cb(err);
}
if (!manifest) {
return cb(new Error(`manifest not found for service with service id: ${id}`));
}
this._scale({ service, deployment_group, version, manifest, replicas }, cb);
});
});
});
});
}
_scale ({ service, deployment_group, version, manifest, replicas }, cb) {
let isFinished = false;
const finish = () => {
if (isFinished) {
return;
}
isFinished = true;
const machineIds = [];
for (let i = 1; i <= replicas; ++i) {
machineIds.push(`${deployment_group.name}_${service.name}_${i}`);
}
this._db.instances.remove(service.instance_ids, (err) => {
// emit error instead of returning early, this is a best effort to cleanup data
if (err) {
this.emit('error', err);
}
VAsync.forEachParallel({
func: (machineId, next) => {
const clientInstance = {
machineId,
status: 'CREATED',
name: service.name
};
this.createInstance(clientInstance, next);
},
inputs: machineIds
}, (err, results) => {
if (err) {
return cb(err);
}
const instanceIds = results.successes.map((instance) => {
return instance.id;
});
this._db.services.update(service.id, { instance_ids: instanceIds }, (err) => {
if (err) {
return cb(err);
}
const clientVersion = {
deploymentGroupId: deployment_group.id,
manifestId: manifest.id,
plan: {
running: true,
actions: [{
type: 'CREATE',
service: service.name,
machines: machineIds
}]
}
};
const scale = version.service_scales.find((scale) => {
return scale.service_name === service.name;
});
if (scale) {
scale.replicas = replicas;
} else {
version.service_scales.push({
service_name: service.name,
replicas
});
}
clientVersion.scales = version.service_scales.map(Transform.fromScale);
this.createVersion(clientVersion, cb);
});
});
});
};
const options = {
provisionName: deployment_group.name,
services: {},
manifest: manifest.raw
};
options.services[service.name] = replicas;
this._docker.scale(options, (err, res) => {
if (err) {
return cb(err);
}
finish();
});
}
// manifests
provisionManifest (clientManifest, cb) {
// get deployment group to verify it exists and get the name
// insert manifest
// callback with manifest
// provision containers and save service data
this.getDeploymentGroup({ id: clientManifest.deploymentGroupId }, (err, deploymentGroup) => {
if (err) {
return cb(err);
}
if (!deploymentGroup) {
return cb(new Error('Deployment group not found for manifest'));
}
const manifest = Transform.toManifest(clientManifest);
this._db.manifests.insert(manifest, (err, key) => {
if (err) {
return cb(err);
}
setImmediate(() => {
let isHandled = false;
this._docker.provision({ projectName: deploymentGroup.name, manifest: clientManifest.raw }, (err, res) => {
if (err) {
this.emit('error', err);
return;
}
// callback can execute multiple times, ensure responses are only handled once
if (isHandled) {
return;
}
isHandled = true;
const options = {
manifestServices: manifest.json.services || manifest.json,
deploymentGroup,
manifestId: key
};
this.provisionServices(options);
});
});
manifest.id = key;
cb(null, Transform.fromManifest(manifest));
});
});
}
getManifest ({ id }, cb) {
console.log(id);
this._db.manifests.single({ id }, (err, manifest) => {
if (err) {
return cb(err);
@ -298,8 +485,7 @@ module.exports = class Data extends EventEmitter {
// services
provisionServices ({ manifestServices, deploymentGroupId, manifestId }, cb) {
// call to docker and create containers
provisionServices ({ manifestServices, deploymentGroup, manifestId }, cb) {
// insert instance information
// insert service information
// insert version information -- will update deploymentGroups
@ -310,14 +496,12 @@ module.exports = class Data extends EventEmitter {
}
});
// TODO: call out to docker for each service and provision a new instance
VAsync.forEachPipeline({
func: (serviceName, next) => {
const manifestService = manifestServices[serviceName];
const clientInstance = {
name: serviceName,
machineId: 'unknown',
machineId: `${deploymentGroup.name}_${serviceName}_1`,
status: 'CREATED'
};
this.createInstance(clientInstance, (err, createdInstance) => {
@ -329,7 +513,7 @@ module.exports = class Data extends EventEmitter {
hash: manifestService.image,
name: serviceName,
slug: serviceName,
deploymentGroupId,
deploymentGroupId: deploymentGroup.id,
instances: [createdInstance]
};
@ -381,19 +565,19 @@ module.exports = class Data extends EventEmitter {
};
const clientVersion = {
deploymentGroupId,
deploymentGroupId: deploymentGroup.id,
manifestId,
scales,
plan,
serviceIds
};
this.createVersion(clientVersion, (err) => {
this.createVersion(clientVersion, (err, version) => {
if (err) {
return cb(err);
}
cb();
cb(null, version);
});
});
}

5
packages/portal-data/package.json
View File

@ -9,13 +9,14 @@
"bootstrap": "node ./bootstrap-data",
"lint": "belly-button --fix",
"lint-ci": "belly-button",
"test": "lab -t 40",
"test-ci": "echo 0"
"test": "lab -c",
"test-ci": "lab -c -r console -o stdout -r tap -o $CIRCLE_TEST_REPORTS/test/portal-data.xml"
},
"keywords": [],
"author": "wyatt",
"license": "MPL-2.0",
"dependencies": {
"docker-compose-client": "^1.0.7",
"hoek": "^4.1.1",
"penseur": "^7.8.1",
"vasync": "^1.6.4",

26
packages/portal-data/test/docker-compose.yml
View File

@ -1,24 +1,2 @@
version: '2.1'
services:
# Service definition for Consul cluster with a minimum of 3 nodes.
# Nodes will use Triton CNS for the service (passed in via the CONSUL
# env var) to find each other and bootstrap the cluster.
consul:
image: autopilotpattern/consul:${TAG:-latest}
labels:
- triton.cns.services=consul
restart: always
mem_limit: 128m
ports:
- 8500
env_file:
- _env
network_mode: bridge
command: >
/usr/local/bin/containerpilot
/bin/consul agent -server
-bootstrap-expect 3
-config-dir=/etc/consul
-ui-dir /ui
hello:
image: hello-world:latest

37
packages/portal-data/test/index.js
View File

@ -378,7 +378,7 @@ describe('versions', () => {
expect(result.scales).to.equal(clientVersion.scales);
data.getVersions({ manifestId: clientVersion.manifestId }, (err, versions) => {
expect(err).to.not.exist();
expect(versions.length).to.equal(2);
expect(versions.length).to.equal(1);
done();
});
});
@ -502,8 +502,7 @@ describe('manifests', () => {
deploymentGroupId: deploymentGroup.id,
type: 'compose',
format: 'yml',
raw: 'docker compose raw contents',
json: { services: [] }
raw: internals.composeFile
};
data.provisionManifest(clientManifest, (err, result) => {
@ -677,3 +676,35 @@ describe('packages', () => {
});
});
});
// skipping by default since it takes so long
describe.skip('scale()', () => {
it('creates new instances of a service', { timeout: 180000 }, (done) => {
const data = new PortalData(internals.options);
data.connect((err) => {
expect(err).to.not.exist();
data.createDeploymentGroup({ name: 'something' }, (err, deploymentGroup) => {
expect(err).to.not.exist();
const clientManifest = {
deploymentGroupId: deploymentGroup.id,
type: 'compose',
format: 'yml',
raw: internals.composeFile
};
data.provisionManifest(clientManifest, (err, manifest) => {
expect(err).to.not.exist();
setTimeout(() => {
data.getDeploymentGroup({ id: deploymentGroup.id }, (err, deploymentGroup) => {
expect(err).to.not.exist();
data.scale({ id: deploymentGroup.services[0].id, replicas: 2 }, (err, version) => {
expect(err).to.not.exist();
expect(version).to.exist();
done();
});
});
}, 80000);
});
});
});
});
});

53
packages/portal-data/yarn.lock
View File

@ -51,6 +51,10 @@ ansi-styles@^2.2.1:
version "2.2.1"
resolved "https://registry.yarnpkg.com/ansi-styles/-/ansi-styles-2.2.1.tgz#b432dd3358b634cf75e1e4664368240533c1ddbe"
apr-awaitify@^1.0.4:
version "1.0.4"
resolved "https://registry.yarnpkg.com/apr-awaitify/-/apr-awaitify-1.0.4.tgz#a72074a0d333e090bb120be9f710fd106b48a90a"
argparse@^1.0.7:
version "1.0.9"
resolved "https://registry.yarnpkg.com/argparse/-/argparse-1.0.9.tgz#73d83bc263f86e97f8cc4f6bae1b0e90a7d22c86"
@ -97,6 +101,10 @@ belly-button@^3.1.0:
glob "7.x.x"
insync "2.x.x"
bindings@~1.2.1:
version "1.2.1"
resolved "https://registry.yarnpkg.com/bindings/-/bindings-1.2.1.tgz#14ad6113812d2d37d72e67b4cacb4bb726505f11"
"bluebird@>= 2.3.2 < 3":
version "2.11.0"
resolved "https://registry.yarnpkg.com/bluebird/-/bluebird-2.11.0.tgz#534b9033c022c9579c56ba3b3e5a5caafbb650e1"
@ -244,6 +252,13 @@ diff@3.x.x:
version "3.2.0"
resolved "https://registry.yarnpkg.com/diff/-/diff-3.2.0.tgz#c9ce393a4b7cbd0b058a725c93df299027868ff9"
docker-compose-client@^1.0.7:
version "1.0.7"
resolved "https://registry.yarnpkg.com/docker-compose-client/-/docker-compose-client-1.0.7.tgz#a2f351aff998fd5323b9b6bb27d4400fff95e43c"
dependencies:
apr-awaitify "^1.0.4"
zerorpc "^0.9.7"
doctrine@^2.0.0:
version "2.0.0"
resolved "https://registry.yarnpkg.com/doctrine/-/doctrine-2.0.0.tgz#c73d8d2909d22291e1a007a395804da8b665fe63"
@ -737,10 +752,24 @@ ms@2.0.0:
version "2.0.0"
resolved "https://registry.yarnpkg.com/ms/-/ms-2.0.0.tgz#5608aeadfc00be6c2901df5f9861788de0d597c8"
msgpack@1.0.2:
version "1.0.2"
resolved "https://registry.yarnpkg.com/msgpack/-/msgpack-1.0.2.tgz#923e2c5cffa65c8418e9b228d1124793969c429c"
dependencies:
nan "^2.0.9"
mute-stream@0.0.5:
version "0.0.5"
resolved "https://registry.yarnpkg.com/mute-stream/-/mute-stream-0.0.5.tgz#8fbfabb0a98a253d3184331f9e8deb7372fac6c0"
nan@^2.0.9:
version "2.6.2"
resolved "https://registry.yarnpkg.com/nan/-/nan-2.6.2.tgz#e4ff34e6c95fdfb5aecc08de6596f43605a7db45"
nan@~2.3.0:
version "2.3.5"
resolved "https://registry.yarnpkg.com/nan/-/nan-2.3.5.tgz#822a0dc266290ce4cd3a12282ca3e7e364668a08"
natural-compare@^1.4.0:
version "1.4.0"
resolved "https://registry.yarnpkg.com/natural-compare/-/natural-compare-1.4.0.tgz#4abebfeed7541f2c27acfb29bdbbd15c8d5ba4f7"
@ -1059,6 +1088,10 @@ uglify-to-browserify@~1.0.0:
version "1.0.2"
resolved "https://registry.yarnpkg.com/uglify-to-browserify/-/uglify-to-browserify-1.0.2.tgz#6e0924d6bda6b5afe349e39a6d632850a0f882b7"
underscore@1.3.3:
version "1.3.3"
resolved "https://registry.yarnpkg.com/underscore/-/underscore-1.3.3.tgz#47ac53683daf832bfa952e1774417da47817ae42"
user-home@^2.0.0:
version "2.0.0"
resolved "https://registry.yarnpkg.com/user-home/-/user-home-2.0.0.tgz#9c70bfd8169bc1dcbf48604e0f04b8b49cde9e9f"
@ -1069,6 +1102,10 @@ util-deprecate@~1.0.1:
version "1.0.2"
resolved "https://registry.yarnpkg.com/util-deprecate/-/util-deprecate-1.0.2.tgz#450d4dc9fa70de732762fbd2d4a28981419a0ccf"
uuid@^3.0.0:
version "3.0.1"
resolved "https://registry.yarnpkg.com/uuid/-/uuid-3.0.1.tgz#6544bba2dfda8c1cf17e629a3a305e2bb1fee6c1"
vasync@^1.6.4:
version "1.6.4"
resolved "https://registry.yarnpkg.com/vasync/-/vasync-1.6.4.tgz#dfe93616ad0e7ae801b332a9d88bfc5cdc8e1d1f"
@ -1126,3 +1163,19 @@ yargs@~3.10.0:
cliui "^2.1.0"
decamelize "^1.0.0"
window-size "0.1.0"
zerorpc@^0.9.7:
version "0.9.7"
resolved "https://registry.yarnpkg.com/zerorpc/-/zerorpc-0.9.7.tgz#64ddb32ce8c934bea5434ec81ca22e971045a860"
dependencies:
msgpack "1.0.2"
underscore "1.3.3"
uuid "^3.0.0"
zmq "2.x"
zmq@2.x:
version "2.15.3"
resolved "https://registry.yarnpkg.com/zmq/-/zmq-2.15.3.tgz#66c6de82cc36b09734b820703776490a6fbbe624"
dependencies:
bindings "~1.2.1"
nan "~2.3.0"