consume prometheus API to display graphs

spikes/graphs-matrix/chartjs-whiskers/client/chart/base.js

@@ -59,6 +59,7 @@ module.exports = React.createClass({
       datasets = [],
       labels = 0
     } = this.props;
+
     this._chart.data.datasets = datasets;
     this._chart.data.labels = buildArray(labels).map((v, i) => '');
     this._chart.update(0);

spikes/leak/Dockerfile

@@ -1,10 +1,14 @@
 FROM mhart/alpine-node:7
 
-WORKDIR /src
-ADD . .
-
 RUN npm install -g yarn
+
+WORKDIR /src
+COPY package.json package.json
+COPY yarn.lock yarn.lock
+
 RUN yarn install --production --pure-lockfile --prefer-offline
 
+COPY . .
+
 EXPOSE 8000
-CMD ["node", "start.js"]
+CMD ["node", "scripts/start.js"]

spikes/leak/package.json

@@ -3,24 +3,65 @@
   "version": "1.0.0",
   "private": true,
   "license": "private",
-  "main": "src/index.js",
+  "main": "src/server/index.js",
   "dependencies": {
     "artillery": "^1.5.0-17",
-    "clone": "^2.0.0",
+    "async": "^2.1.4",
+    "build-array": "^1.0.0",
+    "chart.js": "^2.4.0",
     "date.js": "^0.3.1",
     "epimetheus": "^1.0.46",
+    "force-array": "^3.1.0",
     "good": "^7.0.2",
     "good-console": "^6.3.1",
     "good-squeeze": "^5.0.1",
     "got": "^6.6.3",
     "hapi": "^15.2.0",
-    "hapi-webpack-dev-plugin": "^1.2.0",
+    "hapi-webpack-dev-plugin": "1.1.4",
+    "inert": "^4.0.2",
     "internet-timestamp": "^0.0.1",
+    "lodash.first": "^3.0.0",
     "lodash.get": "^4.4.2",
+    "lodash.take": "^4.1.1",
     "minimist": "^1.2.0",
+    "nes": "^6.3.1",
     "pretty-hrtime": "^1.0.3",
     "prom-client": "^6.1.2",
+    "qs": "^6.3.0",
+    "react": "^15.4.1",
+    "react-dom": "^15.4.1",
+    "react-hot-loader": "^3.0.0-beta.6",
+    "react-redux": "^4.4.6",
+    "redux": "^3.6.0",
+    "redux-logger": "^2.7.4",
+    "redux-promise-middleware": "^4.1.0",
+    "redux-thunk": "^2.1.0",
+    "relative-date": "^1.1.3",
     "require-dir": "^0.3.1",
-    "webpack": "^1.13.3"
+    "simple-statistics": "^2.2.0"
+  },
+  "devDependencies": {
+    "async": "^2.1.4",
+    "babel-core": "^6.18.2",
+    "babel-eslint": "^7.1.1",
+    "babel-loader": "^6.2.8",
+    "babel-plugin-add-module-exports": "^0.2.1",
+    "babel-plugin-transform-es2015-modules-commonjs": "^6.18.0",
+    "babel-plugin-transform-object-rest-spread": "^6.19.0",
+    "babel-preset-es2015": "^6.18.0",
+    "babel-preset-react": "^6.16.0",
+    "diskusage": "^0.1.5",
+    "eslint": "^3.10.2",
+    "eslint-config-semistandard": "^7.0.0",
+    "eslint-config-standard": "^6.2.1",
+    "eslint-plugin-babel": "^4.0.0",
+    "eslint-plugin-promise": "^3.4.0",
+    "eslint-plugin-react": "^6.7.1",
+    "eslint-plugin-standard": "^2.0.1",
+    "json-loader": "^0.5.4",
+    "os-utils": "^0.0.14",
+    "simple-statistics": "^2.2.0",
+    "webpack": "^1.13.3",
+    "webpack-dev-server": "^1.16.2"
   }
 }

spikes/leak/prometheus.yml

@@ -1,16 +1,17 @@
 scrape_configs:
   - job_name: 'leak-fast'
-    # Override the global default and scrape targets from this job every 5 seconds.
     scrape_interval: 1s
     static_configs:
       - targets: ['fast-node:8000', 'another-fast-node:8000']
   - job_name: 'leak-slow'
-    # Override the global default and scrape targets from this job every 5 seconds.
     scrape_interval: 1s
     static_configs:
       - targets: ['slow-node:8000']
   - job_name: 'no-leak'
-    # Override the global default and scrape targets from this job every 5 seconds.
     scrape_interval: 1s
     static_configs:
       - targets: ['plain-node:8000']
+  - job_name: 'leak'
+    scrape_interval: 1s
+    static_configs:
+      - targets: ['fast-node:8000', 'another-fast-node:8000', 'slow-node:8000', 'plain-node:8000']

spikes/leak/scripts/metrics.js

@@ -1,78 +0,0 @@
-process.on('unhandledRejection', (reason) => {
-  throw reason
-});
-
-const argv = require('minimist')(process.argv.slice(2));
-const get = require('lodash.get');
-const date = require('date.js');
-const timestamp = require('internet-timestamp');
-const got = require('got');
-const url = require('url');
-
-const conf = {
-  query: argv.query,
-  ago: argv.ago || '1h',
-  step: argv.step || '1s',
-  hostname: argv.hostname || 'localhost',
-};
-
-if (!conf.query) {
-  console.error(`
-Usage: node metrics.js --query={metric} --step={step} --ago={ago}
-       node metrics.js --query=node_memory_heap_used_bytes --query=node_memory_heap_total_bytes
-  `.trim());
-
-  process.exit(1);
-}
-
-// query=node_memory_heap_used_bytes&end=147989905368&step=15s
-const end = timestamp(new Date());
-const start = timestamp(date(`${conf.ago} ago`));
-
-Promise.all(conf.query.map((query) => {
-  return got(url.format({
-    protocol: 'http:',
-    slashes: true,
-    port: '9090',
-    hostname: conf.hostname,
-    pathname: '/api/v1/query_range',
-    query: {
-      query: query,
-      end: end,
-      step: conf.step,
-      start: start
-    }
-  }));
-})).then((res) => {
-  return res.reduce((sum, r) => {
-    const {
-      data: {
-        result
-      }
-    } = JSON.parse(r.body);
-
-    return result.reduce((sum, inst) => {
-      const {
-        values,
-        metric: {
-          instance,
-          job,
-          __name__
-        }
-      } = inst;
-
-      const oldJob = get(sum, job, {});
-      const oldQuery = get(sum, `${job}.${__name__}`, {});
-
-      return Object.assign(sum, {
-        [job]: Object.assign(oldJob, {
-          [__name__]: Object.assign(oldQuery, {
-            [instance]: values
-          })
-        })
-      })
-    }, sum);
-  }, {});
-}).then((res) => {
-  console.log(JSON.stringify(res, null, 2));
-});

spikes/leak/scripts/prometheus.js

@@ -0,0 +1,155 @@
+const forceArray = require('force-array');
+const get = require('lodash.get');
+const date = require('date.js');
+const timestamp = require('internet-timestamp');
+const got = require('got');
+const url = require('url');
+const qs = require('qs');
+
+const transform = (res) => {
+  return forceArray(res).reduce((sum, r) => {
+    const {
+      data
+    } = JSON.parse(r.body);
+
+    const result = !Array.isArray(data)
+      ? data.result
+      : data;
+
+    return result.reduce((sum, inst) => {
+      const metric = !inst.job
+        ? inst.metric
+        : inst;
+
+      const {
+        values = [],
+        value = []
+      } = inst;
+
+      const {
+        instance,
+        job,
+        __name__
+      } = metric;
+
+      const oldJob = get(sum, job, {});
+      const oldQuery = get(sum, `${job}.${__name__}`, {});
+      const _value = values.length ? values : value
+
+      return Object.assign(sum, {
+        [job]: Object.assign(oldJob, {
+          [instance]: Object.assign(oldQuery, {
+            [__name__]: _value
+          })
+        })
+      })
+    }, sum);
+  }, {});
+};
+
+const range = module.exports.range = ({
+  query = [],
+  ago = '1h ago',
+  step = '1s',
+  hostname = 'localhost'
+}) => {
+  const end = timestamp(new Date());
+  const start = timestamp(date(ago));
+
+  return Promise.all(query.map((query) => {
+    return got(url.format({
+      protocol: 'http:',
+      slashes: true,
+      port: '9090',
+      hostname: hostname,
+      pathname: '/api/v1/query_range',
+      query: {
+        query,
+        end,
+        step,
+        start
+      }
+    }));
+  }))
+  .then(transform);
+};
+
+const query = module.exports.query = ({
+  hostname = 'localhost',
+  query = []
+}) => {
+  return Promise.all(query.map((query) => {
+    return got(url.format({
+      protocol: 'http:',
+      slashes: true,
+      port: '9090',
+      hostname: hostname,
+      pathname: '/api/v1/query',
+      query: {
+        query: query
+      }
+    }));
+  }))
+  .then(transform);
+};
+
+const tree = module.exports.tree = ({
+  hostname = 'localhost',
+  query = []
+}) => {
+  return got(url.format({
+    protocol: 'http:',
+    slashes: true,
+    port: '9090',
+    hostname: hostname,
+    pathname: '/api/v1/series',
+    search: qs.stringify({
+      match: query
+    }, {
+      arrayFormat: 'brackets'
+    })
+  }))
+  .then(transform);
+};
+
+if (!module.parent) {
+  process.on('unhandledRejection', (reason) => {
+    throw reason
+  });
+
+  const usage = () => {
+    console.error(`
+      Usage: node metrics.js --type={type} --query={metric} --step={step} --ago={ago}
+             node metrics.js --type=range --query=node_memory_heap_used_bytes --query=node_memory_heap_total_bytes
+    `.trim());
+
+    return process.exit(1);
+  }
+
+  const argv = require('minimist')(process.argv.slice(2));
+
+  if (!argv.query || !argv.type) {
+    return usage();
+  }
+
+  const handlers = {
+    tree,
+    range,
+    query
+  };
+
+  if (!handlers[argv.type]) {
+    return usage();
+  }
+
+  const conf = {
+    query: argv.query,
+    ago: argv.ago,
+    step: argv.step,
+    hostname: argv.hostname
+  };
+
+  handlers[argv.type](conf).then((res) => {
+    console.log(JSON.stringify(res, null, 2));
+  });
+}

spikes/leak/scripts/start.js

@@ -15,13 +15,15 @@ Usage: TYPE={type} node start.js
 
 const handler = ({
   node: () => {
-    return cp.exec('node src/index.js', {
+    const root = path.join(__dirname, '../');
+    const script = path.join(root, 'src/server/index.js');
+    return cp.exec(`node ${script}`, {
       cwd: __dirname
-    })
+    });
   },
   artillery: () => {
     const conf = path.join(__dirname, '../artillery/artillery-${MODE}.yml');
-    return cp.exec(`../node_modules/.bin/artillery run ${conf}`)
+    return cp.exec(`../node_modules/.bin/artillery run ${conf}`);
   }
 })[TYPE];
 

spikes/leak/src/client/actions.js

@@ -1,8 +1,9 @@
 const take = require('lodash.take');
+const get = require('lodash.get');
 
-const actions =  {
+const actions = {
   'UPDATE_STATS': (state, action) => {
-    const data = (state[action.subscription] || {
+    const data = get(state, `data.${action.subscription}`, {
       cpu: [],
       mem: [],
       disk: []
@@ -24,7 +25,16 @@ const actions =  {
 
     return {
       ...state,
-      [action.subscription]: newData
+      data: {
+        ...state.data,
+        [action.subscription]: newData
+      }
+    };
+  },
+  'GET_JOB_TREE_FULFILLED': (state, action) => {
+    return {
+      ...state,
+      tree: action.payload
     };
   }
 };
@@ -79,4 +89,21 @@ module.exports.unsubscribe = (id) => (dispatch, getState) => {
     type: 'UNSUBSCRIBE',
     payload: p
   });
-};
+};
+
+module.exports.getTree = (id) => (dispatch, getState) => {
+  const {
+    ws
+  } = getState();
+
+  const p = new Promise((resolve, reject) => {
+    ws.request(`/job-tree`, (err, payload) => {
+      return err ? reject(err) : resolve(payload);
+    });
+  });
+
+  return dispatch({
+    type: 'GET_JOB_TREE',
+    payload: p
+  });
+};

spikes/leak/src/client/chart/base.js

@@ -1,6 +1,8 @@
 const buildArray = require('build-array');
 const Chart = require('chart.js');
 const React = require('react');
+const whisker = require('../whisker');
+whisker(Chart);
 
 module.exports = React.createClass({
   ref: function(name) {
@@ -17,7 +19,9 @@ module.exports = React.createClass({
       stacked = false,
       xAxe = false,
       yAxe = false,
-      legend = false
+      legend = false,
+      max = 100,
+      min = 0
     } = this.props;
 
     const _labels = !Array.isArray(labels)
@@ -25,26 +29,27 @@ module.exports = React.createClass({
       : labels;
 
     this._chart = new Chart(this._refs.component, {
-      type: 'bar',
-      stacked: stacked,
+      type: 'whisker',
       responsive: true,
       options: {
         scales: {
           xAxes: [{
-            display: xAxe,
-            stacked: stacked
+            barPercentage: 1.0,
+            categoryPercentage: 1.0
           }],
           yAxes: [{
-            display: yAxe,
-            stacked: stacked
+            ticks: {
+              min: min,
+              max: max
+            }
           }]
         },
         legend: {
-          display: legend
+          display: true
         }
       },
       data: {
-        labels:
+        labels: _labels,
         datasets: datasets
       }
     });
@@ -52,11 +57,15 @@ module.exports = React.createClass({
   componentWillReceiveProps: function(nextProps) {
     const {
       datasets = [],
-      labels = 0
+      labels = 0,
+      max,
+      min
     } = this.props;
 
     this._chart.data.datasets = datasets;
     this._chart.data.labels = buildArray(labels).map((v, i) => '');
+    this._chart.config.options.scales.yAxes[0].ticks.max = max;
+    this._chart.config.options.scales.yAxes[0].ticks.min = min;
     this._chart.update(0);
   },
   render: function() {
@@ -69,3 +78,8 @@ module.exports = React.createClass({
     );
   }
 });
+
+
+/*
+ * datasets[{altbackgr, back, data[{max, min, ...}, label]}]
+ */

spikes/leak/src/client/chart/cpu.js

@@ -4,18 +4,21 @@ const React = require('react');
 
 const colors = {
   user: 'rgb(255, 99, 132)',
-  sys: 'rgb(255, 159, 64)'
+  sys: 'rgb(255, 159, 64)',
+  perc: 'rgba(54, 74, 205, 0.2)',
+  alt: 'rgba(245, 93, 93, 0.2)'
 };
 
 module.exports = ({
   data = {},
   windowSize
 }) => {
-  const datasets = ['user', 'sys'].map((key) => {
+  const datasets = ['perc'].map((key) => {
     return {
       label: key,
       backgroundColor: colors[key],
-      data: buildArray(windowSize).map((v, i) => ((data[i] || {})[key] || 0))
+      altBackgroundColor: colors['alt'],
+      data: buildArray(windowSize).map((v, i) => ((data[i] || {})[key] || { firstQuartile: 0, thirdQuartile: 0, median: 0, max: 0, min: 0 })).reverse()
     };
   });
 

spikes/leak/src/client/chart/disk.js

@@ -1,11 +1,10 @@
-const pretty = require('prettysize');
 const buildArray = require('build-array');
 const Chart = require('./base');
 const React = require('react');
 
 const colors = {
-  user: 'rgb(255, 99, 132)',
-  sys: 'rgb(255, 159, 64)'
+  perc: 'rgba(54, 74, 205, 0.2)',
+  alt: 'rgba(245, 93, 93, 0.2)'
 };
 
 module.exports = ({
@@ -14,20 +13,15 @@ module.exports = ({
 }) => {
   const datasets = [{
     label: 'disk',
-    backgroundColor: 'rgb(255, 159, 64)',
-    data: buildArray(windowSize).map((v, i) => {
-      return data[i] ? (data[i].total - data[i].free) : 0;
-    })
+    backgroundColor: colors['perc'],
+    altBackgroundColor: colors['alt'],
+    data: buildArray(windowSize).map((v, i) => ((data[i] || {})['perc'] || { firstQuartile: 0, thirdQuartile: 0, median: 0, max: 0, min: 0 })).reverse()
   }];
 
-  const labels = buildArray(windowSize).map((v, i) => {
-    return data[i] ? pretty(datasets[0].data[i]) : '';
-  });
-
   return (
     <Chart
       datasets={datasets}
-      labels={labels}
+      labels={datasets[0].data.length}
       legend={true}
     />
   );

spikes/leak/src/client/chart/mem.js

@@ -1,27 +1,53 @@
+const first = require('lodash.first');
+const get = require('lodash.get');
 const buildArray = require('build-array');
 const Chart = require('./base');
 const React = require('react');
 
 const colors = {
-  user: 'rgb(255, 99, 132)',
-  sys: 'rgb(255, 159, 64)'
+  perc: 'rgba(54, 74, 205, 0.2)',
+  alt: 'rgba(245, 93, 93, 0.2)'
 };
 
 module.exports = ({
   data = [],
-  windowSize
+  windowSize,
+  aggregate = false,
+  name = 'mem',
+  max = 100
 }) => {
   const datasets = [{
-    label: 'mem',
-    backgroundColor: 'rgb(255, 99, 132)',
-    data: buildArray(windowSize).map((v, i) => ((data[i] || {}).used || 0))
+    label: name,
+    backgroundColor: colors.perc,
+    altBackgroundColor: colors.alt,
+    data: buildArray(windowSize).map((v, i) => {
+      const sample = get(data, `[${i}].perc`, {
+        firstQuartile: 0,
+        thirdQuartile: 0,
+        median: 0,
+        max: 0,
+        min: 0
+      });
+
+      return Object.keys(sample).reduce((sum, name) => {
+        // from bytes to mb
+        return {
+          ...sum,
+          [name]: (sample[name] > 0)
+            ?  sample[name] / 1000000
+            : sample[name]
+        };
+      }, {});
+    }).reverse()
   }];
 
   return (
     <Chart
       datasets={datasets}
-      labels={datasets[0].data.length}
+      stacked={aggregate}
+      labels={first(datasets).data.length}
       legend={true}
+      max={max/1000000}
     />
   );
 };

spikes/leak/src/client/element.whisker.js

@@ -0,0 +1,221 @@
+'use strict';
+
+module.exports = function(Chart) {
+
+  var globalOpts = Chart.defaults.global;
+
+  globalOpts.elements.rectangle = {
+    backgroundColor: globalOpts.defaultColor,
+    borderWidth: 0,
+    borderColor: globalOpts.defaultColor,
+    borderSkipped: 'bottom'
+  };
+
+  function isVertical(bar) {
+    return bar._view.width !== undefined;
+  }
+
+  /**
+   * Helper function to get the bounds of the bar regardless of the orientation
+   * @private
+   * @param bar {Chart.Element.Rectangle} the bar
+   * @return {Bounds} bounds of the bar
+   */
+  function getBarBounds(bar) {
+    var vm = bar._view;
+    var x1, x2, y1, y2;
+
+    if (isVertical(bar)) {
+      // vertical
+      var halfWidth = vm.width / 2;
+      x1 = vm.x - halfWidth;
+      x2 = vm.x + halfWidth;
+      y1 = Math.min(vm.y, vm.base);
+      y2 = Math.max(vm.y, vm.base);
+    } else {
+      // horizontal bar
+      var halfHeight = vm.height / 2;
+      x1 = Math.min(vm.x, vm.base);
+      x2 = Math.max(vm.x, vm.base);
+      y1 = vm.y - halfHeight;
+      y2 = vm.y + halfHeight;
+    }
+
+    return {
+      left: x1,
+      top: y1,
+      right: x2,
+      bottom: y2
+    };
+  }
+
+  Chart.elements.Whisker = Chart.Element.extend({
+    draw: function() {
+      var ctx = this._chart.ctx;
+      var vm = this._view;
+
+      var halfWidth = vm.width / 2,
+        leftX = vm.x - halfWidth,
+        rightX = vm.x + halfWidth,
+        top = vm.base - (vm.base - vm.y),
+        halfStroke = vm.borderWidth / 2;
+
+      // Canvas doesn't allow us to stroke inside the width so we can
+      // adjust the sizes to fit if we're setting a stroke on the line
+      if (vm.borderWidth) {
+        leftX += halfStroke;
+        rightX -= halfStroke;
+        top += halfStroke;
+      }
+
+      ctx.beginPath();
+      ctx.fillStyle = vm.backgroundColor;
+      ctx.strokeStyle = vm.borderColor;
+      ctx.lineWidth = vm.borderWidth;
+
+      // Corner points, from bottom-left to bottom-right clockwise
+      // | 1 2 |
+      // | 0 3 |
+      var corners = [
+        [leftX, vm.base],
+        [leftX, top],
+        [rightX, top],
+        [rightX, vm.base]
+      ];
+
+      // Find first (starting) corner with fallback to 'bottom'
+      var borders = ['bottom', 'left', 'top', 'right'];
+      var startCorner = borders.indexOf(vm.borderSkipped, 0);
+      if (startCorner === -1) {
+        startCorner = 0;
+      }
+
+      function cornerAt(index) {
+        return corners[(startCorner + index) % 4];
+      }
+
+      // Draw rectangle from 'startCorner'
+      var corner = cornerAt(0);
+      ctx.moveTo(corner[0], corner[1]);
+
+      for (var i = 1; i < 4; i++) {
+        corner = cornerAt(i);
+        ctx.lineTo(corner[0], corner[1]);
+      }
+
+      ctx.fill();
+      if (vm.borderWidth) {
+        ctx.stroke();
+      }
+      ctx.closePath();
+
+      // Median line
+      ctx.beginPath();
+
+      ctx.moveTo(leftX, vm.median);
+      ctx.lineTo(rightX, vm.median);
+      ctx.lineWidth = 2;
+
+      // set line color
+      ctx.strokeStyle = 'rgb(54, 74, 205)';
+      ctx.stroke();
+      ctx.closePath();
+
+      // Top Whisker
+      // if (smaller than 5px then do not draw)
+      if (vm.median - vm.maxV > 10) {
+        ctx.beginPath();
+        ctx.moveTo((rightX - leftX) / 2 + leftX, vm.median - 1);
+        ctx.lineTo((rightX - leftX) / 2 + leftX, vm.maxV);
+        ctx.lineWidth = 2;
+        ctx.strokeStyle = 'rgb(245, 93, 93)';
+        ctx.stroke();
+        ctx.closePath();
+        ctx.beginPath();
+        ctx.arc((rightX - leftX) / 2 + leftX, vm.maxV, 3, 0, 2 * Math.PI);
+        ctx.fillStyle = 'rgb(245, 93, 93)';
+        ctx.fill();
+      }
+
+      // Bottom Whisker
+      // if (smaller than 5px then do not draw)
+      if (vm.minV - vm.median > 10) {
+        ctx.beginPath();
+        ctx.moveTo((rightX - leftX) / 2 + leftX, vm.median + 1);
+        ctx.lineTo((rightX - leftX) / 2 + leftX, vm.minV);
+        ctx.lineWidth = 2;
+        ctx.strokeStyle = 'rgb(245, 93, 93)';
+        ctx.stroke();
+        ctx.closePath();
+        ctx.beginPath();
+        ctx.arc((rightX - leftX) / 2 + leftX, vm.minV, 3, 0, 2 * Math.PI);
+        ctx.fillStyle = 'rgb(245, 93, 93)';
+        ctx.fill();
+      }
+    },
+    height: function() {
+      var vm = this._view;
+      return vm.base - vm.y;
+    },
+    inRange: function(mouseX, mouseY) {
+      var inRange = false;
+
+      if (this._view) {
+        var bounds = getBarBounds(this);
+        inRange = mouseX >= bounds.left && mouseX <= bounds.right && mouseY >= bounds.top && mouseY <= bounds.bottom;
+      }
+
+      return inRange;
+    },
+    inLabelRange: function(mouseX, mouseY) {
+      var me = this;
+      if (!me._view) {
+        return false;
+      }
+
+      var inRange = false;
+      var bounds = getBarBounds(me);
+
+      if (isVertical(me)) {
+        inRange = mouseX >= bounds.left && mouseX <= bounds.right;
+      } else {
+        inRange = mouseY >= bounds.top && mouseY <= bounds.bottom;
+      }
+
+      return inRange;
+    },
+    inXRange: function(mouseX) {
+      var bounds = getBarBounds(this);
+      return mouseX >= bounds.left && mouseX <= bounds.right;
+    },
+    inYRange: function(mouseY) {
+      var bounds = getBarBounds(this);
+      return mouseY >= bounds.top && mouseY <= bounds.bottom;
+    },
+    getCenterPoint: function() {
+      var vm = this._view;
+      var x, y;
+      if (isVertical(this)) {
+        x = vm.x;
+        y = (vm.y + vm.base) / 2;
+      } else {
+        x = (vm.x + vm.base) / 2;
+        y = vm.y;
+      }
+
+      return {x: x, y: y};
+    },
+    getArea: function() {
+      var vm = this._view;
+      return vm.width * Math.abs(vm.y - vm.base);
+    },
+    tooltipPosition: function() {
+      var vm = this._view;
+      return {
+        x: vm.x,
+        y: vm.y
+      };
+    }
+  });
+
+};

spikes/leak/src/client/index.js

@@ -9,20 +9,28 @@ const {
 
 const client = new Client(`ws://${document.location.host}`);
 
+const store = Store({
+  windowSize: 20,
+  ws: client
+});
+
 client.connect((err) => {
   if (err) {
     throw err;
   }
-});
 
-const store = Store({
-  ws: client,
-  windowSize: 20
+  store.getState().wsReady = true;
+
+  render();
 });
 
 const render = () => {
   const Root = require('./root');
 
+  if (!store.getState().wsReady) {
+    return;
+  }
+
   ReactDOM.render(
     <Root store={store} />,
     document.getElementById('root')

spikes/leak/src/client/matrix.js

@@ -1,3 +1,4 @@
+const get = require('lodash.get');
 const React = require('react');
 const buildArray = require('build-array');
 const ReactRedux = require('react-redux');
@@ -10,80 +11,146 @@ const {
 
 const {
   subscribe,
-  unsubscribe
+  unsubscribe,
+  getTree
 } = actions;
 
-const mapStateToProps = (state, ownProps) => {
-  return {
-    windowSize: state.windowSize,
-    data: state[ownProps.id]
-  };
-};
-
-const mapDispatchToProps = (dispatch, ownProps) => {
-  return {
-    subscribe: () => {
-      return dispatch(subscribe(ownProps.id));
-    },
-    unsubscribe: () => {
-      return unsubscribe(ownProps.id);
-    }
-  }
-};
-
-const Row = connect(
-  mapStateToProps,
-  mapDispatchToProps,
-)(React.createClass({
+const Job = React.createClass({
   componentWillMount: function() {
-    this.props.subscribe();
+    this.props.subscribe(this.props.name);
   },
   componentWillUnmount: function() {
-    this.props.unsubscribe();
+    this.props.unsubscribe(this.props.name);
   },
   render: function() {
     const {
-      data = {},
+      data,
+      instances = [],
+      name,
       windowSize
     } = this.props;
 
-    const charts = Object.keys(data).map((key, i, arr) => {
-      if (!Chart[key]) {
-        return null;
-      }
+    if (!data) {
+      return null;
+    }
 
-      const chart = React.createElement(Chart[key], {
-        data: data[key],
+    if (instances.length < 2) {
+      return null;
+    }
+
+    let max = 0;
+
+    const charts = ['aggregate'].concat(instances.map((i) => {
+      return `instances.${i}`;
+    })).map((path) => {
+      const set = data.mem.map((sample) => {
+        const perc = get(sample, path);
+
+        if (perc.max > max) {
+          max = perc.max;
+        }
+
+        return {
+          perc: perc,
+          when: sample.when
+        };
+      });
+
+      return {
+        key: path,
+        data: set,
+        aggregate: path === 'aggregate',
         windowSize
+      };
+    }).map((ctx, i, arr) => {
+      const chart = React.createElement(Chart.mem, {
+        data: ctx.data,
+        aggregate: ctx.aggregate,
+        windowSize: ctx.windowSize,
+        max: max,
+        name: ctx.key
       });
 
       return (
-        <div key={key} className={`col-xs-${12/arr.length}`}>
+        <div
+          key={ctx.key}
+          className={`col-xs-${12 / arr.length}`}
+        >
           {chart}
         </div>
       );
     });
 
     return (
-      <div className='row'>
-        {charts}
+      <div>
+        <p>{name}</p>
+        <div className='row'>
+          {charts}
+        </div>
       </div>
     );
   }
-}));
+});
+
+const Jobs = React.createClass({
+  componentWillMount: function() {
+    this.props.getTree();
+  },
+  render: function() {
+    const {
+      subscribe,
+      unsubscribe,
+      tree = {},
+      data = {},
+      windowSize
+    } = this.props;
+
+
+    const jobs = Object.keys(tree).map((jobName) => {
+      return (
+        <Job
+          key={jobName}
+          windowSize={windowSize}
+          data={data[jobName]}
+          instances={Object.keys(tree[jobName])}
+          subscribe={subscribe}
+          unsubscribe={unsubscribe}
+          name={jobName}
+        />
+      );
+    })
 
-module.exports = ({
-  rows
-}) => {
-  const _rows = buildArray(rows).map((v, i) => {
     return (
-      <Row id={i} key={i} />
+      <div>
+        {jobs}
+      </div>
     );
-  });
+  }
+});
 
-  return (
-    <div>
-      {_rows}
-    </div>
-  );
+const mapStateToProps = (state) => {
+  return {
+    tree: state.tree,
+    data: state.data,
+    windowSize: state.windowSize
+  };
 };
+
+const mapDispatchToProps = (dispatch, ownProps) => {
+  return {
+    subscribe: (name) => {
+      return dispatch(subscribe(name));
+    },
+    unsubscribe: (name) => {
+      return dispatch(unsubscribe(name));
+    },
+    getTree: () => {
+      return dispatch(getTree());
+    }
+  }
+};
+
+module.exports = connect(
+  mapStateToProps,
+  mapDispatchToProps,
+)(Jobs);

spikes/leak/src/client/root.js

@@ -17,7 +17,7 @@ module.exports = ({
   return (
     <AppContainer>
       <Provider store={store}>
-        <Matrix rows={4} />
+        <Matrix />
       </Provider>
     </AppContainer>
   );

spikes/leak/src/client/whisker.js

@@ -0,0 +1,276 @@
+const whiskerElement = require('./element.whisker');
+
+module.exports = function(Chart) {
+  whiskerElement(Chart);
+
+  var helpers = Chart.helpers;
+
+  Chart.defaults.whisker = {
+    hover: {
+      mode: 'label'
+    },
+
+    scales: {
+      xAxes: [{
+        type: 'category',
+
+        // Specific to Bar Controller
+        categoryPercentage: 0.8,
+        barPercentage: 0.9,
+
+        // grid line settings
+        gridLines: {
+          offsetGridLines: true
+        }
+      }],
+      yAxes: [{
+        type: 'linear'
+      }]
+    }
+  };
+
+  Chart.controllers.whisker = Chart.DatasetController.extend({
+
+    dataElementType: Chart.elements.Whisker,
+
+    initialize: function(chart, datasetIndex) {
+      Chart.DatasetController.prototype.initialize.call(this, chart, datasetIndex);
+
+      // Use this to indicate that this is a bar dataset.
+      this.getMeta().bar = true;
+    },
+
+    // Get the number of datasets that display bars. We use this to correctly calculate the bar width
+    getBarCount: function() {
+      var me = this;
+      var barCount = 0;
+      helpers.each(me.chart.data.datasets, function(dataset, datasetIndex) {
+        var meta = me.chart.getDatasetMeta(datasetIndex);
+        if (meta.bar && me.chart.isDatasetVisible(datasetIndex)) {
+          ++barCount;
+        }
+      }, me);
+      return barCount;
+    },
+
+    update: function(reset) {
+      var me = this;
+      helpers.each(me.getMeta().data, function(rectangle, index) {
+        me.updateElement(rectangle, index, reset);
+      }, me);
+    },
+
+    updateElement: function(rectangle, index, reset) {
+      var me = this;
+      var meta = me.getMeta();
+      var xScale = me.getScaleForId(meta.xAxisID);
+      var yScale = me.getScaleForId(meta.yAxisID);
+      var scaleBase = yScale.getBasePixel();
+      var rectangleElementOptions = me.chart.options.elements.rectangle;
+      var custom = rectangle.custom || {};
+      var dataset = me.getDataset();
+
+      rectangle._xScale = xScale;
+      rectangle._yScale = yScale;
+      rectangle._datasetIndex = me.index;
+      rectangle._index = index;
+
+      var ruler = me.getRuler(index);
+      rectangle._model = {
+        x: me.calculateBarX(index, me.index, ruler),
+        y: reset ? scaleBase : me.boxTopValue(index, me.index),
+
+        // Tooltip
+        label: me.chart.data.labels[index],
+        datasetLabel: dataset.label,
+
+        // Appearance
+        median: reset ? scaleBase : me.medianValue(me.index, index),
+        maxV: reset ? scaleBase : me.maxValue(me.index, index),
+        minV: reset ? scaleBase : me.minValue(me.index, index),
+        base: reset ? scaleBase : me.boxBottomValue(me.index, index),
+        width: me.calculateBarWidth(ruler),
+        backgroundColor: custom.backgroundColor ? custom.backgroundColor : helpers.getValueAtIndexOrDefault(me.stddev(me.index, index) > 3 ? dataset.altBackgroundColor : dataset.backgroundColor, index, rectangleElementOptions.backgroundColor),
+        borderSkipped: custom.borderSkipped ? custom.borderSkipped : rectangleElementOptions.borderSkipped,
+        borderColor: custom.borderColor ? custom.borderColor : helpers.getValueAtIndexOrDefault(dataset.borderColor, index, rectangleElementOptions.borderColor),
+        borderWidth: custom.borderWidth ? custom.borderWidth : helpers.getValueAtIndexOrDefault(dataset.borderWidth, index, rectangleElementOptions.borderWidth)
+      };
+
+      rectangle.pivot();
+    },
+
+    stddev: function(datasetIndex, index) {
+      var me = this;
+      var obj = me.getDataset().data[index];
+      var value = Number(obj.stddev);
+
+      return value;
+    },
+
+    minValue: function(datasetIndex, index) {
+      var me = this;
+      var meta = me.getMeta();
+      var yScale = me.getScaleForId(meta.yAxisID);
+      var obj = me.getDataset().data[index];
+      var value = Number(obj.min);
+
+      return yScale.getPixelForValue(value);
+    },
+
+    maxValue: function(datasetIndex, index) {
+      var me = this;
+      var meta = me.getMeta();
+      var yScale = me.getScaleForId(meta.yAxisID);
+      var obj = me.getDataset().data[index];
+      var value = Number(obj.max);
+
+      return yScale.getPixelForValue(value);
+    },
+
+    medianValue: function(datasetIndex, index) {
+      var me = this;
+      var meta = me.getMeta();
+      var yScale = me.getScaleForId(meta.yAxisID);
+      var obj = me.getDataset().data[index];
+      var value = Number(obj.median);
+
+      return yScale.getPixelForValue(value);
+    },
+
+    boxBottomValue: function(datasetIndex, index) {
+      var me = this;
+      var meta = me.getMeta();
+      var yScale = me.getScaleForId(meta.yAxisID);
+      var obj = me.getDataset().data[index];
+      var value = Number(obj.firstQuartile);
+
+      return yScale.getPixelForValue(value);
+    },
+
+    boxTopValue: function(index, datasetIndex) {
+      var me = this;
+      var meta = me.getMeta();
+      var yScale = me.getScaleForId(meta.yAxisID);
+      var obj = me.getDataset().data[index];
+      var value = Number(obj.thirdQuartile);
+
+      return yScale.getPixelForValue(value);
+    },
+
+    getRuler: function(index) {
+      var me = this;
+      var meta = me.getMeta();
+      var xScale = me.getScaleForId(meta.xAxisID);
+      var datasetCount = me.getBarCount();
+
+      var tickWidth;
+
+      if (xScale.options.type === 'category') {
+        tickWidth = xScale.getPixelForTick(index + 1) - xScale.getPixelForTick(index);
+      } else {
+        // Average width
+        tickWidth = xScale.width / xScale.ticks.length;
+      }
+      var categoryWidth = tickWidth * xScale.options.categoryPercentage;
+      var categorySpacing = (tickWidth - (tickWidth * xScale.options.categoryPercentage)) / 2;
+      var fullBarWidth = categoryWidth / datasetCount;
+
+      if (xScale.ticks.length !== me.chart.data.labels.length) {
+        var perc = xScale.ticks.length / me.chart.data.labels.length;
+        fullBarWidth = fullBarWidth * perc;
+      }
+
+      var barWidth = fullBarWidth * xScale.options.barPercentage;
+      var barSpacing = fullBarWidth - (fullBarWidth * xScale.options.barPercentage);
+
+      return {
+        datasetCount: datasetCount,
+        tickWidth: tickWidth,
+        categoryWidth: categoryWidth,
+        categorySpacing: categorySpacing,
+        fullBarWidth: fullBarWidth,
+        barWidth: barWidth,
+        barSpacing: barSpacing
+      };
+    },
+
+    calculateBarWidth: function(ruler) {
+      var xScale = this.getScaleForId(this.getMeta().xAxisID);
+      if (xScale.options.barThickness) {
+        return xScale.options.barThickness;
+      }
+      return ruler.barWidth;
+    },
+
+    // Get bar index from the given dataset index accounting for the fact that not all bars are visible
+    getBarIndex: function(datasetIndex) {
+      var barIndex = 0;
+      var meta;
+      var j;
+
+      for (j = 0; j < datasetIndex; ++j) {
+        meta = this.chart.getDatasetMeta(j);
+        if (meta.bar && this.chart.isDatasetVisible(j)) {
+          ++barIndex;
+        }
+      }
+
+      return barIndex;
+    },
+
+    calculateBarX: function(index, datasetIndex, ruler) {
+      var me = this;
+      var meta = me.getMeta();
+      var xScale = me.getScaleForId(meta.xAxisID);
+      var barIndex = me.getBarIndex(datasetIndex);
+      var leftTick = xScale.getPixelForValue(null, index, datasetIndex, me.chart.isCombo);
+      leftTick -= me.chart.isCombo ? (ruler.tickWidth / 2) : 0;
+
+      return leftTick +
+        (ruler.barWidth / 2) +
+        ruler.categorySpacing +
+        (ruler.barWidth * barIndex) +
+        (ruler.barSpacing / 2) +
+        (ruler.barSpacing * barIndex);
+    },
+
+    draw: function(ease) {
+      var me = this;
+      var easingDecimal = ease || 1;
+      var metaData = me.getMeta().data;
+      var dataset = me.getDataset();
+      var i, len;
+
+      for (i = 0, len = metaData.length; i < len; ++i) {
+        var d = dataset.data[i];
+        if (d !== null && d !== undefined && typeof d === 'object' && !isNaN(d.median)) {
+          metaData[i].transition(easingDecimal).draw();
+        }
+      }
+    },
+
+    setHoverStyle: function(rectangle) {
+      var dataset = this.chart.data.datasets[rectangle._datasetIndex];
+      var index = rectangle._index;
+
+      var custom = rectangle.custom || {};
+      var model = rectangle._model;
+      model.backgroundColor = custom.hoverBackgroundColor ? custom.hoverBackgroundColor : helpers.getValueAtIndexOrDefault(dataset.hoverBackgroundColor, index, helpers.getHoverColor(model.backgroundColor));
+      model.borderColor = custom.hoverBorderColor ? custom.hoverBorderColor : helpers.getValueAtIndexOrDefault(dataset.hoverBorderColor, index, helpers.getHoverColor(model.borderColor));
+      model.borderWidth = custom.hoverBorderWidth ? custom.hoverBorderWidth : helpers.getValueAtIndexOrDefault(dataset.hoverBorderWidth, index, model.borderWidth);
+    },
+
+    removeHoverStyle: function(rectangle) {
+      var dataset = this.chart.data.datasets[rectangle._datasetIndex];
+      var index = rectangle._index;
+      var custom = rectangle.custom || {};
+      var model = rectangle._model;
+      var rectangleElementOptions = this.chart.options.elements.rectangle;
+
+      model.backgroundColor = custom.backgroundColor ? custom.backgroundColor : helpers.getValueAtIndexOrDefault(dataset.backgroundColor, index, rectangleElementOptions.backgroundColor);
+      model.borderColor = custom.borderColor ? custom.borderColor : helpers.getValueAtIndexOrDefault(dataset.borderColor, index, rectangleElementOptions.borderColor);
+      model.borderWidth = custom.borderWidth ? custom.borderWidth : helpers.getValueAtIndexOrDefault(dataset.borderWidth, index, rectangleElementOptions.borderWidth);
+    }
+
+  });
+};

spikes/leak/src/server/index.js

@@ -11,6 +11,8 @@ server.connection({
   port: 8000
 });
 
+epimetheus.instrument(server);
+
 server.register(plugins, (err) => {
   if (err) {
     throw err;
@@ -20,8 +22,6 @@ server.register(plugins, (err) => {
     routes[name](server);
   });
 
-  epimetheus.instrument(server);
-
   server.start((err) => {
     server.connections.forEach((conn) => {
       console.log(`started at: ${conn.info.uri}`);

spikes/leak/src/server/metric.js

@@ -0,0 +1,103 @@
+const relativeDate = require('relative-date');
+const statistics = require('simple-statistics');
+const prometheus = require('../../scripts/prometheus');
+const async = require('async');
+
+const cdm = {};
+
+const calc = (sample) => {
+  return {
+    firstQuartile: statistics.quantile(sample, 0.25),
+    median: statistics.median(sample),
+    thirdQuartile: statistics.quantile(sample, 0.75),
+    max: statistics.max(sample),
+    min: statistics.min(sample),
+    stddev: statistics.sampleStandardDeviation(sample)
+  };
+};
+
+const getMem = ({
+  job
+}, fn) => {
+  prometheus.query({
+    query: [`node_memory_heap_used_bytes{job="${job}"}`]
+  }).then((res) => {
+    if (!res || !res[job]) {
+      return null
+    }
+
+    const aggregate = calc(Object.keys(res[job]).map((inst) => {
+      return Number(res[job][inst].node_memory_heap_used_bytes[1]);
+    }));
+
+    const instances = Object.keys(res[job]).reduce((sum, inst) => {
+      return Object.assign(sum, {
+        [inst]: calc([Number(res[job][inst].node_memory_heap_used_bytes[1])])
+      })
+    }, {});
+
+    return {
+      raw: res[job],
+      aggregate,
+      instances
+    };
+  }).then((res) => {
+    return fn(null, res);
+  }).catch((err) => {
+    return fn(err);
+  });
+};
+
+const getStats = (ctx, fn) => {
+  async.parallel({
+    mem: async.apply(getMem, ctx)
+  }, fn);
+};
+
+module.exports = (server) => ({
+  on: (job) => {
+    console.log('on', job);
+
+    if (cdm[job] && (cdm[job].sockets > 0)) {
+      cdm[job].sockets += 1;
+      return;
+    }
+
+    let messageId = 0;
+
+    const update = () => {
+      console.log(`publishing /stats/${job}/${messageId += 1}`);
+
+      getStats({
+        job: job
+      }, (err, stats) => {
+        if (err) {
+          return console.error(err);
+        }
+
+        server.publish(`/stats/${job}`, {
+          when: new Date().getTime(),
+          stats
+        });
+      });
+    };
+
+    cdm[job] = {
+      interval: setInterval(update, 1000),
+      sockets: 1
+    };
+  },
+  off: (job) => {
+    console.log('off', job);
+
+    if (!(cdm[job].sockets -= 1)) {
+      clearInterval(cdm[job].interval);
+    }
+  }
+});
+
+module.exports.tree = (ctx) => {
+  return prometheus.tree({
+    query: ['node_memory_heap_used_bytes']
+  });
+};

spikes/leak/src/server/plugins.js

@@ -4,7 +4,8 @@ const path = require('path');
 const cfg = require('../webpack.config.js');
 
 module.exports = [
-  require('inert'), {
+  require('inert'),
+  require('nes'), {
   register: require('good'),
   options: {
     reporters: {

spikes/leak/src/server/routes/home.js

@@ -0,0 +1,12 @@
+
+const path = require('path');
+
+module.exports = (server) => {
+  server.route({
+    method: 'GET',
+    path: '/',
+    handler: (request, reply) => {
+      reply.file(path.join(__dirname, '../../../static/index.html'));
+    }
+  });
+};

spikes/leak/src/server/routes/leak.js

@@ -1,5 +1,4 @@
 const prettyHrtime = require('pretty-hrtime');
-const clone = require('clone');
 
 // leak example from https://auth0.com/blog/four-types-of-leaks-in-your-javascript-code-and-how-to-get-rid-of-them/
 let theLeak = null;

spikes/leak/src/server/routes/metrics.js

@@ -0,0 +1,26 @@
+const Metric = require('../metric');
+
+module.exports = (server) => {
+  const metric = Metric(server);
+
+  server.route({
+    method: 'GET',
+    path: '/job-tree',
+    config: {
+      handler: (request, reply) => reply(Metric.tree())
+    }
+  });
+
+  server.subscription('/stats/{id}', {
+    onSubscribe: (socket, path, params, next) => {
+      console.log('onSubscribe');
+      metric.on(params.id);
+      next();
+    },
+    onUnsubscribe: (socket, path, params, next) => {
+      console.log('onUnsubscribe');
+      metric.off(params.id);
+      next();
+    }
+  });
+};

spikes/leak/src/server/routes/version.js

@@ -1,4 +1,4 @@
-const Pkg = require('../../package.json');
+const Pkg = require('../../../package.json');
 
 const internals = {
   response: {

spikes/leak/src/webpack.config.js

@@ -31,7 +31,7 @@ const config = {
       include: [
         path.join(__dirname, './client')
       ],
-      loaders: ['babel']
+      loaders: ['babel-loader']
     }, {
       test: /\.json?$/,
       exclude: /node_modules/,