switch over to cloudapi2 (drop node-smartdc piggybacking); prefer 'user' to 'account' in APIs (a la node-manta)

README.md

@@ -1,5 +1,5 @@
 `sdc` is a CLI for Joyent SmartDataCenter and the
-Joyent Public Cloud (<https://my.joyentcloud.com>, 
+Joyent Public Cloud (<https://my.joyentcloud.com>,
 <http://www.joyent.com/products/compute-service>).
 
 # Installation
@@ -22,3 +22,23 @@ TODO
 # Getting Started
 
 TODO
+
+
+
+# cloudapi2.js differences with node-smartdc/lib/cloudapi.js
+
+The old node-smartdc module included an lib for talking directly to the SDC
+Cloud API (node-smartdc/lib/cloudapi.js). Part of this module (node-sdc) is a
+re-write of the Cloud API lib with some backward incompatibilities. The
+differences and backward incompatibilities are discussed here.
+
+- Currently no caching options in cloudapi2.js (this should be re-added in
+  some form). The `noCache` option to many of the cloudapi.js methods will not
+  be re-added, it was a wart.
+- The leading `account` option to each cloudapi.js method has been dropped. It
+  was redundant for the constructor `account` option.
+- "account" is now "user" in the CloudAPI constructor.
+- All (all? at least at the time of this writing) methods in cloudapi2.js have
+  a signature of `function (options, callback)` instead of the sometimes
+  haphazard extra arguments.
+

TODO.md

@@ -1,6 +1,5 @@
 # first
 
-- lib/cloudapi2.js and drop using node-smartdc
 - machines:
     - short default output
     - long '-l' output, -H, -o, -s
@@ -10,13 +9,29 @@
 - uuid caching
 - UUID prefix support
 - profile command (adding profile, edit, etc.)
-- multi-dc support... profile.dcs
 
 
 
 # later (in no particular order)
 
+- signing: should sigstr include more than just the date? How about the request
+  path??? Not according to the cloudapi docs.
 - restify-client and bunyan-light without dtrace-provider
+- Get node-smartdc-auth to take a log option. Perhaps borrow from imgapi.js'
+  cliSigner et al.
+- node-smartdc-auth: Support a path to a priv key for "keyId" arg. Or a separate
+  alternative arg. Copy this from imgapi.cliSigner.
+        sign: cloudapi.cliSigner({
+            keyId: <KEY-ID>,
+            user: <USER>,
+            log: <BUNYAN-LOGGER>,
+        }),
+- the error reporting for a signing error sucks:
+    getAccount: err { message: 'error signing request',
+        code: 'Signing',
+        exitStatus: 1 }
+    e.g. when the KEY_ID is nonsense. Does imgapi's auth have better error
+    reporting?
 - how to add/exclude DCs?
 - cmdln.js support for bash tab completion
 - node-smartdc installs joyentcloud and warns about deprecation on stderr.

examples/example-get-account.js

@@ -0,0 +1,40 @@
+#!/usr/bin/env node
+/**
+ * Example using cloudapi2.js to call cloudapi's GetAccount endpoint.
+ *
+ * Usage:
+ *      ./example-get-account.js | bunyan
+ */
+
+var p = console.log;
+var auth = require('smartdc-auth');
+var bunyan = require('bunyan');
+var cloudapi = require('../lib/cloudapi2');
+
+var log = bunyan.createLogger({
+    name: 'test-get-account',
+    level: 'trace'
+})
+
+var USER = process.env.SDC_ACCOUNT || process.env.SDC_USER || 'bob';
+var KEY_ID = process.env.SDC_KEY_ID || 'b4:f0:b4:6c:18:3b:44:63:b4:4e:58:22:74:43:d4:bc';
+
+var sign = auth.cliSigner({
+    keyId: KEY_ID,
+    user: USER,
+    log: log
+});
+var client = cloudapi.createClient({
+    url: 'https://us-sw-1.api.joyentcloud.com',
+    user: USER,
+    version: '*',
+    sign: sign,
+    agent: false, // don't want KeepAlive
+    log: log
+});
+
+log.info('start')
+client.getAccount(function (err, account) {
+    p('getAccount: err', err)
+    p('getAccount: account', account)
+});

lib/cli.js

@@ -106,9 +106,9 @@ CLI.prototype.do_profile = function (subcmd, opts, args, callback) {
         p(JSON.stringify(profs, null, 4));
     } else {
         common.tabulate(profs, {
-            columns: 'curr,name,dcs,account,keyId',
-            sort: 'name,account',
-            validFields: 'curr,name,dcs,account,keyId'
+            columns: 'curr,name,dcs,user,keyId',
+            sort: 'name,user',
+            validFields: 'curr,name,dcs,user,keyId'
         });
     }
     callback();

lib/cloudapi2.js

@@ -0,0 +1,281 @@
+/*
+ * Copyright (c) 2014, Joyent, Inc. All rights reserved.
+ *
+ * Client library for the SmartDataCenter Cloud API (cloudapi).
+ * http://apidocs.joyent.com/cloudapi/
+ *
+ * Usage example::
+ *
+ *      var auth = require('smartdc-auth');
+ *      var cloudapi = require('./lib/cloudapi2');
+ *      var client = cloudapi.createClient({
+ *              url: <URL>,                 // 'https://us-sw-1.api.joyentcloud.com',
+ *              user: <USER>,               // 'bob'
+ *              log: <BUNYAN-LOGGER>,
+ *              sign: auth.cliSigner({
+ *                  keyId: <KEY-ID>,        // ssh fingerprint
+ *                  user: <USER>,           // 'bob'
+ *                  log: <BUNYAN-LOGGER>,
+ *              }),
+ *              ...
+ *      });
+ *      client.listImages(function (err, images) { ... });
+ *      ...
+ *
+ */
+
+var p = console.log;
+
+var assert = require('assert-plus');
+var async = require('async');
+var auth = require('smartdc-auth');
+var os = require('os');
+var querystring = require('querystring');
+var restify = require('restify');
+var sprintf = require('util').format;
+
+var errors = require('./errors');
+
+
+
+// ---- globals
+
+var SDC_VERSION = require('../package.json').version;
+var OS_ARCH = os.arch();
+var OS_PLATFORM = os.platform();
+
+
+
+// ---- internal support stuff
+
+// A no-op bunyan logger shim.
+function BunyanNoopLogger() {}
+BunyanNoopLogger.prototype.trace = function () {};
+BunyanNoopLogger.prototype.debug = function () {};
+BunyanNoopLogger.prototype.info = function () {};
+BunyanNoopLogger.prototype.warn = function () {};
+BunyanNoopLogger.prototype.error = function () {};
+BunyanNoopLogger.prototype.fatal = function () {};
+BunyanNoopLogger.prototype.child = function () { return this; };
+BunyanNoopLogger.prototype.end = function () {};
+
+
+
+// ---- client API
+
+/**
+ * Create a cloudapi client.
+ *
+ * @param options {Object}
+ *      - {String} url (required) Cloud API base url
+ *      - {String} user (required) The user login name.
+ *        For backward compat, 'options.account' is accepted as a synonym.
+ *      - {Function} sign (required) An http-signature auth signing function
+ *      - {String} version (optional) Used for the accept-version header. This
+ *        defaults to '*', meaning that over time you could experience breaking
+ *        changes. Specifying a value is strongly recommended. E.g. '~7.1'.
+ *      - {Bunyan Logger} log (optional)
+ *      - ... and any other standard restify client options, e.g.:
+ *          {String} userAgent
+ *          {Boolean} rejectUnauthorized
+ *          {Boolean} agent  Set to `false` to not get KeepAlive. You want
+ *              this for CLIs.
+ *          TODO doc the backoff/retry available options
+ * @throws {TypeError} on bad input.
+ * @constructor
+ *
+ * TODO: caching options (copy node-manta/node-moray/node-smartdc?)
+ *        - {Boolean} noCache (optional) disable client caching (default false).
+ *        - {Boolean} cacheSize (optional) number of cache entries (default 1k).
+ *        - {Boolean} cacheExpiry (optional) entry age in seconds (default 60).
+ */
+function CloudAPI(options) {
+    assert.object(options, 'options');
+    assert.string(options.url, 'options.url');
+    assert.string(options.user || options.account, 'options.user');
+    assert.func(options.sign, 'options.sign');
+    assert.optionalString(options.version, 'options.version');
+    assert.optionalObject(options.log, 'options.log');
+
+    this.url = options.url;
+    this.user = options.user || options.account;
+    this.sign = options.sign;
+    this.log = options.log || new BunyanNoopLogger();
+    if (!options.version) {
+        options.version = '*';
+    }
+    if (!options.userAgent) {
+        options.userAgent = sprintf('sdc/%s (%s-%s; node/%s)',
+            SDC_VERSION, OS_ARCH, OS_PLATFORM, process.versions.node);
+    }
+
+    // XXX relevant?
+    //options.retryCallback = function checkFor500(code) {
+    //    return (code === 500);
+    //};
+
+    // XXX relevant?
+    //this.token = options.token;
+
+    this.client = restify.createJsonClient(options);
+}
+
+
+CloudAPI.prototype._getAuthHeaders = function _getAuthHeaders(callback) {
+    assert.func(callback, 'callback');
+    var self = this;
+
+    var headers = {};
+    headers.date = new Date().toUTCString();
+    var sigstr = 'date: ' + headers.date;
+
+    //XXX
+    //if (this.token !== undefined) {
+    //    obj.headers['X-Auth-Token'] = this.token;
+    //}
+
+    self.sign(sigstr, function (err, sig) {
+        if (err || !sig) {
+            callback(new errors.SigningError(err));
+            return;
+        }
+
+        headers.authorization = sprintf(
+            'Signature keyId="/%s/keys/%s",algorithm="%s",signature="%s"',
+            self.user, sig.keyId, sig.algorithm, sig.signature);
+        callback(null, headers);
+    });
+};
+
+
+// ---- accounts
+
+/**
+ * Get the user's account data.
+ * <http://apidocs.joyent.com/cloudapi/#GetAccount>
+ *
+ * @param {Object} options (optional)
+ * @param {Function} callback of the form `function (err, user)`
+ */
+CloudAPI.prototype.getAccount = function (options, callback) {
+    var self = this;
+    if (callback === undefined) {
+        callback = options;
+        options = {};
+    }
+    assert.object(options, 'options');
+    assert.func(callback, 'callback');
+
+    var path = '/' + self.user;
+    self._getAuthHeaders(function (hErr, headers) {
+        if (hErr) {
+            callback(hErr);
+            return;
+        }
+        var opts = {
+            path: path,
+            headers: headers
+        };
+        self.client.get(opts, function (err, req, res, body) {
+            if (err) {
+                callback(err, null, res);
+            } else {
+                callback(null, body, res);
+            }
+        });
+    });
+};
+
+
+// ---- machines
+
+/**
+ * List the user's machines.
+ * <http://apidocs.joyent.com/cloudapi/#ListMachines>
+ *
+ * If no `offset` is given, then this will return all machines, calling
+ * multiple times if necessary. If `offset` is specified given, then just
+ * a single response will be made.
+ *
+ * @param {Object} options (optional)
+ *      - {Number} offset (optional) An offset number of machine at which to
+ *        return results.
+ *      - {Number} limit (optional) Max number of machines to return.
+ * @param {Function} callback of the form `function (err, machines, responses)`
+ *      where `responses` is an array of response objects in retrieving all
+ *      the machines. ListMachines has a max number of machines, so can require
+ *      multiple requests to list all of them.
+ */
+CloudAPI.prototype.listMachines = function (options, callback) {
+    var self = this;
+    if (callback === undefined) {
+        callback = options;
+        options = {};
+    }
+    assert.object(options, 'options');
+    assert.func(callback, 'callback');
+
+    var query = {
+        limit: options.limit
+    };
+
+    var paging = options.offset === undefined;
+    var offset = options.offset || 0;
+    var lastHeaders;
+    var responses = [];
+    var bodies = [];
+    async.doWhilst(
+        function getPage(next) {
+            self._getAuthHeaders(function (hErr, headers) {
+                if (hErr) {
+                    next(hErr);
+                    return;
+                }
+                query.offset = offset;
+                var path = sprintf('/%s/machines?%s', self.user,
+                    querystring.stringify(query));
+                var opts = {
+                    path: path,
+                    headers: headers
+                };
+                self.client.get(opts, function (err, req, res, body) {
+                    lastHeaders = res.headers;
+                    responses.push(res);
+                    bodies.push(body);
+                    next(err);
+                });
+            });
+        },
+        function testContinue() {
+            if (!paging) {
+                return false;
+            }
+            xQueryLimit = Number(lastHeaders['x-query-limit']);
+            xResourceCount = Number(lastHeaders['x-resource-count']);
+            assert.number(xQueryLimit, 'x-query-limit header');
+            assert.number(xResourceCount, 'x-resource-count header');
+            offset += Number(lastHeaders['x-resource-count']);
+            return xResourceCount >= xQueryLimit;
+        },
+        function doneMachines(err) {
+            if (err) {
+                callback(err, null, responses);
+            } else if (bodies.length === 1) {
+                callback(null, bodies[0], responses);
+            } else {
+                var machines = Array.prototype.concat.apply([], bodies);
+                callback(null, machines, responses);
+            }
+        }
+    )
+};
+
+
+
+// --- Exports
+
+module.exports = {
+    createClient: function (options) {
+        return new CloudAPI(options);
+    }
+};

lib/sdc.js

@@ -7,12 +7,14 @@
 var p = console.log;
 var assert = require('assert-plus');
 var async = require('async');
+var auth = require('smartdc-auth');
 var EventEmitter = require('events').EventEmitter;
 var format = require('util').format;
 var fs = require('fs');
 var path = require('path');
-var smartdc = require('smartdc');
+var restify = require('restify');
 
+var cloudapi = require('./cloudapi2');
 var common = require('./common');
 var loadConfigSync = require('./config').loadConfigSync;
 
@@ -33,7 +35,18 @@ function SDC(options) {
     assert.object(options.log, 'options.log');
     assert.optionalString(options.profile, 'options.profile');
 
-    this.log = options.log;
+    // Make sure a given bunyan logger has reasonable client_re[qs] serializers.
+    // Note: This was fixed in restify, then broken again in
+    // https://github.com/mcavage/node-restify/pull/501
+    if (options.log.serializers &&
+        (!options.log.serializers.client_req ||
+        !options.log.serializers.client_req)) {
+        this.log = options.log.child({
+            serializers: restify.bunyan.serializers
+        });
+    } else {
+        this.log = options.log;
+    }
     this.config = loadConfigSync();
     this.profiles = this.config.profiles;
     this.profile = this.getProfile(
@@ -125,24 +138,23 @@ SDC.prototype._clientFromDc = function _clientFromDc(dc) {
         var prof = this.profile;
         var sign;
         if (prof.privKey) {
-            sign = smartdc.privateKeySigner({
-                user: prof.account,
+            sign = auth.privateKeySigner({
+                user: prof.user,
                 keyId: prof.keyId,
                 key: prof.privKey
             });
         } else {
-            sign = smartdc.cliSigner({keyId: prof.keyId, user: prof.account});
+            sign = auth.cliSigner({
+                keyId: prof.keyId,
+                user: prof.user
+            });
         }
-        var client = smartdc.createClient({
+        var client = cloudapi.createClient({
             url: this.config.dcs[dc],
-            account: prof.account,
+            user: prof.user,
             version: '*',
-            noCache: true, //XXX
             rejectUnauthorized: Boolean(prof.rejectUnauthorized),
             sign: sign,
-            // XXX cloudapi.js stupidly uses its own logger, but takes logLevel
-            logLevel: this.log && this.log.level(),
-            // Pass our logger to underlying restify client.
             log: this.log
         });
         this._clientFromDcCache[dc] = client;

package.json

@@ -15,7 +15,8 @@
     "mkdirp": "0.3.5",
     "node-uuid": "1.4.1",
     "once": "1.3.0",
-    "smartdc": "git+ssh://git@github.com:joyent/node-smartdc.git#master",
+    "restify": "git+ssh://git@github.com:mcavage/node-restify.git#9bab8b7f",
+    "smartdc-auth": "git+ssh://git@github.com:joyent/node-smartdc-auth.git#9f21966",
     "verror": "1.3.7"
   },
   "engines": {