From 8cb5138f9ea846b236e3ac468d9d42f3d12e065b Mon Sep 17 00:00:00 2001 From: Trent Mick Date: Wed, 19 Feb 2014 19:52:58 -0800 Subject: [PATCH] switch over to cloudapi2 (drop node-smartdc piggybacking); prefer 'user' to 'account' in APIs (a la node-manta) --- README.md | 22 ++- TODO.md | 19 ++- examples/example-get-account.js | 40 +++++ lib/cli.js | 6 +- lib/cloudapi2.js | 281 ++++++++++++++++++++++++++++++++ lib/sdc.js | 34 ++-- package.json | 3 +- 7 files changed, 387 insertions(+), 18 deletions(-) create mode 100755 examples/example-get-account.js create mode 100644 lib/cloudapi2.js diff --git a/README.md b/README.md index dc1091c..5dd7ce2 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,5 @@ `sdc` is a CLI for Joyent SmartDataCenter and the -Joyent Public Cloud (, +Joyent Public Cloud (, ). # Installation @@ -22,3 +22,23 @@ TODO # Getting Started TODO + + + +# cloudapi2.js differences with node-smartdc/lib/cloudapi.js + +The old node-smartdc module included an lib for talking directly to the SDC +Cloud API (node-smartdc/lib/cloudapi.js). Part of this module (node-sdc) is a +re-write of the Cloud API lib with some backward incompatibilities. The +differences and backward incompatibilities are discussed here. + +- Currently no caching options in cloudapi2.js (this should be re-added in + some form). The `noCache` option to many of the cloudapi.js methods will not + be re-added, it was a wart. +- The leading `account` option to each cloudapi.js method has been dropped. It + was redundant for the constructor `account` option. +- "account" is now "user" in the CloudAPI constructor. +- All (all? at least at the time of this writing) methods in cloudapi2.js have + a signature of `function (options, callback)` instead of the sometimes + haphazard extra arguments. + diff --git a/TODO.md b/TODO.md index aaaa5ff..dcd27c3 100644 --- a/TODO.md +++ b/TODO.md @@ -1,6 +1,5 @@ # first -- lib/cloudapi2.js and drop using node-smartdc - machines: - short default output - long '-l' output, -H, -o, -s @@ -10,13 +9,29 @@ - uuid caching - UUID prefix support - profile command (adding profile, edit, etc.) -- multi-dc support... profile.dcs # later (in no particular order) +- signing: should sigstr include more than just the date? How about the request + path??? Not according to the cloudapi docs. - restify-client and bunyan-light without dtrace-provider +- Get node-smartdc-auth to take a log option. Perhaps borrow from imgapi.js' + cliSigner et al. +- node-smartdc-auth: Support a path to a priv key for "keyId" arg. Or a separate + alternative arg. Copy this from imgapi.cliSigner. + sign: cloudapi.cliSigner({ + keyId: , + user: , + log: , + }), +- the error reporting for a signing error sucks: + getAccount: err { message: 'error signing request', + code: 'Signing', + exitStatus: 1 } + e.g. when the KEY_ID is nonsense. Does imgapi's auth have better error + reporting? - how to add/exclude DCs? - cmdln.js support for bash tab completion - node-smartdc installs joyentcloud and warns about deprecation on stderr. diff --git a/examples/example-get-account.js b/examples/example-get-account.js new file mode 100755 index 0000000..aa78c9b --- /dev/null +++ b/examples/example-get-account.js @@ -0,0 +1,40 @@ +#!/usr/bin/env node +/** + * Example using cloudapi2.js to call cloudapi's GetAccount endpoint. + * + * Usage: + * ./example-get-account.js | bunyan + */ + +var p = console.log; +var auth = require('smartdc-auth'); +var bunyan = require('bunyan'); +var cloudapi = require('../lib/cloudapi2'); + +var log = bunyan.createLogger({ + name: 'test-get-account', + level: 'trace' +}) + +var USER = process.env.SDC_ACCOUNT || process.env.SDC_USER || 'bob'; +var KEY_ID = process.env.SDC_KEY_ID || 'b4:f0:b4:6c:18:3b:44:63:b4:4e:58:22:74:43:d4:bc'; + +var sign = auth.cliSigner({ + keyId: KEY_ID, + user: USER, + log: log +}); +var client = cloudapi.createClient({ + url: 'https://us-sw-1.api.joyentcloud.com', + user: USER, + version: '*', + sign: sign, + agent: false, // don't want KeepAlive + log: log +}); + +log.info('start') +client.getAccount(function (err, account) { + p('getAccount: err', err) + p('getAccount: account', account) +}); diff --git a/lib/cli.js b/lib/cli.js index 0595dc0..a218f86 100644 --- a/lib/cli.js +++ b/lib/cli.js @@ -106,9 +106,9 @@ CLI.prototype.do_profile = function (subcmd, opts, args, callback) { p(JSON.stringify(profs, null, 4)); } else { common.tabulate(profs, { - columns: 'curr,name,dcs,account,keyId', - sort: 'name,account', - validFields: 'curr,name,dcs,account,keyId' + columns: 'curr,name,dcs,user,keyId', + sort: 'name,user', + validFields: 'curr,name,dcs,user,keyId' }); } callback(); diff --git a/lib/cloudapi2.js b/lib/cloudapi2.js new file mode 100644 index 0000000..edc0aee --- /dev/null +++ b/lib/cloudapi2.js @@ -0,0 +1,281 @@ +/* + * Copyright (c) 2014, Joyent, Inc. All rights reserved. + * + * Client library for the SmartDataCenter Cloud API (cloudapi). + * http://apidocs.joyent.com/cloudapi/ + * + * Usage example:: + * + * var auth = require('smartdc-auth'); + * var cloudapi = require('./lib/cloudapi2'); + * var client = cloudapi.createClient({ + * url: , // 'https://us-sw-1.api.joyentcloud.com', + * user: , // 'bob' + * log: , + * sign: auth.cliSigner({ + * keyId: , // ssh fingerprint + * user: , // 'bob' + * log: , + * }), + * ... + * }); + * client.listImages(function (err, images) { ... }); + * ... + * + */ + +var p = console.log; + +var assert = require('assert-plus'); +var async = require('async'); +var auth = require('smartdc-auth'); +var os = require('os'); +var querystring = require('querystring'); +var restify = require('restify'); +var sprintf = require('util').format; + +var errors = require('./errors'); + + + +// ---- globals + +var SDC_VERSION = require('../package.json').version; +var OS_ARCH = os.arch(); +var OS_PLATFORM = os.platform(); + + + +// ---- internal support stuff + +// A no-op bunyan logger shim. +function BunyanNoopLogger() {} +BunyanNoopLogger.prototype.trace = function () {}; +BunyanNoopLogger.prototype.debug = function () {}; +BunyanNoopLogger.prototype.info = function () {}; +BunyanNoopLogger.prototype.warn = function () {}; +BunyanNoopLogger.prototype.error = function () {}; +BunyanNoopLogger.prototype.fatal = function () {}; +BunyanNoopLogger.prototype.child = function () { return this; }; +BunyanNoopLogger.prototype.end = function () {}; + + + +// ---- client API + +/** + * Create a cloudapi client. + * + * @param options {Object} + * - {String} url (required) Cloud API base url + * - {String} user (required) The user login name. + * For backward compat, 'options.account' is accepted as a synonym. + * - {Function} sign (required) An http-signature auth signing function + * - {String} version (optional) Used for the accept-version header. This + * defaults to '*', meaning that over time you could experience breaking + * changes. Specifying a value is strongly recommended. E.g. '~7.1'. + * - {Bunyan Logger} log (optional) + * - ... and any other standard restify client options, e.g.: + * {String} userAgent + * {Boolean} rejectUnauthorized + * {Boolean} agent Set to `false` to not get KeepAlive. You want + * this for CLIs. + * TODO doc the backoff/retry available options + * @throws {TypeError} on bad input. + * @constructor + * + * TODO: caching options (copy node-manta/node-moray/node-smartdc?) + * - {Boolean} noCache (optional) disable client caching (default false). + * - {Boolean} cacheSize (optional) number of cache entries (default 1k). + * - {Boolean} cacheExpiry (optional) entry age in seconds (default 60). + */ +function CloudAPI(options) { + assert.object(options, 'options'); + assert.string(options.url, 'options.url'); + assert.string(options.user || options.account, 'options.user'); + assert.func(options.sign, 'options.sign'); + assert.optionalString(options.version, 'options.version'); + assert.optionalObject(options.log, 'options.log'); + + this.url = options.url; + this.user = options.user || options.account; + this.sign = options.sign; + this.log = options.log || new BunyanNoopLogger(); + if (!options.version) { + options.version = '*'; + } + if (!options.userAgent) { + options.userAgent = sprintf('sdc/%s (%s-%s; node/%s)', + SDC_VERSION, OS_ARCH, OS_PLATFORM, process.versions.node); + } + + // XXX relevant? + //options.retryCallback = function checkFor500(code) { + // return (code === 500); + //}; + + // XXX relevant? + //this.token = options.token; + + this.client = restify.createJsonClient(options); +} + + +CloudAPI.prototype._getAuthHeaders = function _getAuthHeaders(callback) { + assert.func(callback, 'callback'); + var self = this; + + var headers = {}; + headers.date = new Date().toUTCString(); + var sigstr = 'date: ' + headers.date; + + //XXX + //if (this.token !== undefined) { + // obj.headers['X-Auth-Token'] = this.token; + //} + + self.sign(sigstr, function (err, sig) { + if (err || !sig) { + callback(new errors.SigningError(err)); + return; + } + + headers.authorization = sprintf( + 'Signature keyId="/%s/keys/%s",algorithm="%s",signature="%s"', + self.user, sig.keyId, sig.algorithm, sig.signature); + callback(null, headers); + }); +}; + + +// ---- accounts + +/** + * Get the user's account data. + * + * + * @param {Object} options (optional) + * @param {Function} callback of the form `function (err, user)` + */ +CloudAPI.prototype.getAccount = function (options, callback) { + var self = this; + if (callback === undefined) { + callback = options; + options = {}; + } + assert.object(options, 'options'); + assert.func(callback, 'callback'); + + var path = '/' + self.user; + self._getAuthHeaders(function (hErr, headers) { + if (hErr) { + callback(hErr); + return; + } + var opts = { + path: path, + headers: headers + }; + self.client.get(opts, function (err, req, res, body) { + if (err) { + callback(err, null, res); + } else { + callback(null, body, res); + } + }); + }); +}; + + +// ---- machines + +/** + * List the user's machines. + * + * + * If no `offset` is given, then this will return all machines, calling + * multiple times if necessary. If `offset` is specified given, then just + * a single response will be made. + * + * @param {Object} options (optional) + * - {Number} offset (optional) An offset number of machine at which to + * return results. + * - {Number} limit (optional) Max number of machines to return. + * @param {Function} callback of the form `function (err, machines, responses)` + * where `responses` is an array of response objects in retrieving all + * the machines. ListMachines has a max number of machines, so can require + * multiple requests to list all of them. + */ +CloudAPI.prototype.listMachines = function (options, callback) { + var self = this; + if (callback === undefined) { + callback = options; + options = {}; + } + assert.object(options, 'options'); + assert.func(callback, 'callback'); + + var query = { + limit: options.limit + }; + + var paging = options.offset === undefined; + var offset = options.offset || 0; + var lastHeaders; + var responses = []; + var bodies = []; + async.doWhilst( + function getPage(next) { + self._getAuthHeaders(function (hErr, headers) { + if (hErr) { + next(hErr); + return; + } + query.offset = offset; + var path = sprintf('/%s/machines?%s', self.user, + querystring.stringify(query)); + var opts = { + path: path, + headers: headers + }; + self.client.get(opts, function (err, req, res, body) { + lastHeaders = res.headers; + responses.push(res); + bodies.push(body); + next(err); + }); + }); + }, + function testContinue() { + if (!paging) { + return false; + } + xQueryLimit = Number(lastHeaders['x-query-limit']); + xResourceCount = Number(lastHeaders['x-resource-count']); + assert.number(xQueryLimit, 'x-query-limit header'); + assert.number(xResourceCount, 'x-resource-count header'); + offset += Number(lastHeaders['x-resource-count']); + return xResourceCount >= xQueryLimit; + }, + function doneMachines(err) { + if (err) { + callback(err, null, responses); + } else if (bodies.length === 1) { + callback(null, bodies[0], responses); + } else { + var machines = Array.prototype.concat.apply([], bodies); + callback(null, machines, responses); + } + } + ) +}; + + + +// --- Exports + +module.exports = { + createClient: function (options) { + return new CloudAPI(options); + } +}; diff --git a/lib/sdc.js b/lib/sdc.js index fc84528..2f0f525 100644 --- a/lib/sdc.js +++ b/lib/sdc.js @@ -7,12 +7,14 @@ var p = console.log; var assert = require('assert-plus'); var async = require('async'); +var auth = require('smartdc-auth'); var EventEmitter = require('events').EventEmitter; var format = require('util').format; var fs = require('fs'); var path = require('path'); -var smartdc = require('smartdc'); +var restify = require('restify'); +var cloudapi = require('./cloudapi2'); var common = require('./common'); var loadConfigSync = require('./config').loadConfigSync; @@ -33,7 +35,18 @@ function SDC(options) { assert.object(options.log, 'options.log'); assert.optionalString(options.profile, 'options.profile'); - this.log = options.log; + // Make sure a given bunyan logger has reasonable client_re[qs] serializers. + // Note: This was fixed in restify, then broken again in + // https://github.com/mcavage/node-restify/pull/501 + if (options.log.serializers && + (!options.log.serializers.client_req || + !options.log.serializers.client_req)) { + this.log = options.log.child({ + serializers: restify.bunyan.serializers + }); + } else { + this.log = options.log; + } this.config = loadConfigSync(); this.profiles = this.config.profiles; this.profile = this.getProfile( @@ -125,24 +138,23 @@ SDC.prototype._clientFromDc = function _clientFromDc(dc) { var prof = this.profile; var sign; if (prof.privKey) { - sign = smartdc.privateKeySigner({ - user: prof.account, + sign = auth.privateKeySigner({ + user: prof.user, keyId: prof.keyId, key: prof.privKey }); } else { - sign = smartdc.cliSigner({keyId: prof.keyId, user: prof.account}); + sign = auth.cliSigner({ + keyId: prof.keyId, + user: prof.user + }); } - var client = smartdc.createClient({ + var client = cloudapi.createClient({ url: this.config.dcs[dc], - account: prof.account, + user: prof.user, version: '*', - noCache: true, //XXX rejectUnauthorized: Boolean(prof.rejectUnauthorized), sign: sign, - // XXX cloudapi.js stupidly uses its own logger, but takes logLevel - logLevel: this.log && this.log.level(), - // Pass our logger to underlying restify client. log: this.log }); this._clientFromDcCache[dc] = client; diff --git a/package.json b/package.json index 24a3809..2d6c65b 100644 --- a/package.json +++ b/package.json @@ -15,7 +15,8 @@ "mkdirp": "0.3.5", "node-uuid": "1.4.1", "once": "1.3.0", - "smartdc": "git+ssh://git@github.com:joyent/node-smartdc.git#master", + "restify": "git+ssh://git@github.com:mcavage/node-restify.git#9bab8b7f", + "smartdc-auth": "git+ssh://git@github.com:joyent/node-smartdc-auth.git#9f21966", "verror": "1.3.7" }, "engines": {