Browse Source

TRITON-1409 node-triton tests: improve skip messages, skip "known fails" by default

Reviewed by: Marsell Kukuljevic <marsell@joyent.com>
Approved by: Marsell Kukuljevic <marsell@joyent.com>
tags/7.1.1
Trent Mick 3 months ago
parent
commit
18b8fae335

+ 14
- 0
README.md View File

@@ -412,6 +412,20 @@ There are a few `TEST_...` vars that can tweak how the tests are run.
412 412
   those with "image" in the name, use `make test TEST_GLOB=*image*`, or
413 413
   to run a specific test file: `make test TEST_GLOB=metadataFromOpts`.
414 414
 
415
+- `TEST_KNOWN_FAIL=1` - At any given time there may be some known failures
416
+  in the test suite that are being worked on in specific tickets. Those
417
+  tests may be excluded from the default test run. These will show up in
418
+  test output like this:
419
+
420
+    ```
421
+    test/integration/cli-snapshots.test.js ................ 0/1 1s
422
+      Skipped: 1
423
+        triton instance snapshot known failure, see TRITON-1387
424
+    ```
425
+
426
+  Set the `TEST_KNOWN_FAIL=1` environment variable to include these tests
427
+  in the test run.
428
+
415 429
 - `TEST_JOBS=<number of test files to run concurrently>` - By default this is
416 430
   10. Set to 1 to run tests serially.
417 431
 

+ 3
- 1
test/integration/api-vlans.test.js View File

@@ -75,7 +75,9 @@ test('TritonApi vlan', function (suite) {
75 75
     });
76 76
 
77 77
 
78
-    suite.test(' TritonApi deleteFabricVlan', function (t) {
78
+    suite.test(' TritonApi deleteFabricVlan', {
79
+        skip: !h.CONFIG.allowWriteActions && 'requires config.allowWriteActions'
80
+    }, function (t) {
79 81
         function check(genId, idType, cb) {
80 82
             CLIENT.cloudapi.createFabricVlan({
81 83
                 vlan_id: 3291,

+ 1
- 1
test/integration/cli-account.test.js View File

@@ -21,7 +21,7 @@ var test = require('tap').test;
21 21
 // --- Globals
22 22
 
23 23
 var writeTestOpts = {
24
-    skip: !h.CONFIG.allowWriteActions
24
+    skip: !h.CONFIG.allowWriteActions && 'requires config.allowWriteActions'
25 25
 };
26 26
 
27 27
 

+ 4
- 2
test/integration/cli-affinity.test.js View File

@@ -30,10 +30,12 @@ var h = require('./helpers');
30 30
 var ALIAS_PREFIX = format('nodetritontest-affinity-%s', os.hostname());
31 31
 
32 32
 var testOpts = {
33
-    skip: !h.CONFIG.allowWriteActions || h.CONFIG.skipAffinityTests
33
+    skip: (
34
+        (!h.CONFIG.allowWriteActions || h.CONFIG.skipAffinityTests) &&
35
+        'requires config.allowWriteActions and config.skipAffinityTests=false'
36
+    )
34 37
 };
35 38
 
36
-
37 39
 // --- Tests
38 40
 
39 41
 test('affinity (triton create -a RULE ...)', testOpts, function (suite) {

+ 3
- 8
test/integration/cli-deletion-protection.test.js View File

@@ -22,8 +22,8 @@ var test = require('tap').test;
22 22
 
23 23
 var INST_ALIAS = f('nodetritontest-deletion-protection-%s', os.hostname());
24 24
 var INST;
25
-var OPTS = {
26
-    skip: !h.CONFIG.allowWriteActions
25
+var testOpts = {
26
+    skip: !h.CONFIG.allowWriteActions && 'requires config.allowWriteActions'
27 27
 };
28 28
 
29 29
 // --- Helpers
@@ -44,12 +44,7 @@ function cleanup(t) {
44 44
 
45 45
 // --- Tests
46 46
 
47
-if (OPTS.skip) {
48
-    console.error('** skipping %s tests', __filename);
49
-    console.error('** set "allowWriteActions" in test config to enable');
50
-}
51
-
52
-test('triton instance', OPTS, function (suite) {
47
+test('triton instance', testOpts, function (suite) {
53 48
     h.printConfig(suite);
54 49
 
55 50
     suite.test('  cleanup existing inst with alias ' + INST_ALIAS, cleanup);

+ 3
- 8
test/integration/cli-fwrules.test.js View File

@@ -25,18 +25,13 @@ var RULE2 = 'FROM any TO vm $id BLOCK tcp port 25';
25 25
 var INST;
26 26
 var ID;
27 27
 var INST_ALIAS = f('nodetritontest-fwrules-%s', os.hostname());
28
-var OPTS = {
29
-    skip: !h.CONFIG.allowWriteActions
28
+var testOpts = {
29
+    skip: !h.CONFIG.allowWriteActions && 'requires config.allowWriteActions'
30 30
 };
31 31
 
32 32
 // --- Tests
33 33
 
34
-if (OPTS.skip) {
35
-    console.error('** skipping %s tests', __filename);
36
-    console.error('** set "allowWriteActions" in test config to enable');
37
-}
38
-
39
-test('triton fwrule', OPTS, function (suite) {
34
+test('triton fwrule', testOpts, function (suite) {
40 35
     h.printConfig(suite);
41 36
 
42 37
     suite.test('  cleanup existing inst with alias ' + INST_ALIAS,

+ 2
- 1
test/integration/cli-image-create-kvm.test.js View File

@@ -33,7 +33,8 @@ var DERIVED_INST_ALIAS = _RESOURCE_NAME_PREFIX + '-derived';
33 33
 delete _RESOURCE_NAME_PREFIX;
34 34
 
35 35
 var testOpts = {
36
-    skip: !h.CONFIG.allowWriteActions || h.CONFIG.skipKvmTests
36
+    skip: (!h.CONFIG.allowWriteActions || h.CONFIG.skipKvmTests) &&
37
+        'requires config.allowWriteActions and config.skipKvmTests=false'
37 38
 };
38 39
 
39 40
 

+ 1
- 1
test/integration/cli-image-create.test.js View File

@@ -32,7 +32,7 @@ var IMAGE_DATA = {
32 32
 var DERIVED_ALIAS = format('nodetritontest-images-%s-derived', os.hostname());
33 33
 
34 34
 var testOpts = {
35
-    skip: !h.CONFIG.allowWriteActions
35
+    skip: !h.CONFIG.allowWriteActions && 'requires config.allowWriteActions'
36 36
 };
37 37
 
38 38
 

+ 3
- 8
test/integration/cli-instance-tag.test.js View File

@@ -26,19 +26,14 @@ var h = require('./helpers');
26 26
 // --- globals
27 27
 
28 28
 var INST_ALIAS = f('nodetritontest-insttag-%s', os.hostname());
29
-
30
-var opts = {
31
-    skip: !h.CONFIG.allowWriteActions
29
+var testOpts = {
30
+    skip: !h.CONFIG.allowWriteActions && 'requires config.allowWriteActions'
32 31
 };
33 32
 
34 33
 
35 34
 // --- Tests
36 35
 
37
-if (opts.skip) {
38
-    console.error('** skipping %s tests', __filename);
39
-    console.error('** set "allowWriteActions" in test config to enable');
40
-}
41
-test('triton inst tag ...', opts, function (suite) {
36
+test('triton inst tag ...', testOpts, function (suite) {
42 37
     h.printConfig(suite);
43 38
 
44 39
     var inst;

+ 3
- 7
test/integration/cli-manage-workflow.test.js View File

@@ -26,8 +26,8 @@ var h = require('./helpers');
26 26
 var INST_ALIAS = f('nodetritontest-managewf-%s', os.hostname());
27 27
 var INST_ALIAS_NEWNAME = INST_ALIAS + '-renamed';
28 28
 
29
-var opts = {
30
-    skip: !h.CONFIG.allowWriteActions
29
+var testOpts = {
30
+    skip: !h.CONFIG.allowWriteActions && 'requires config.allowWriteActions'
31 31
 };
32 32
 
33 33
 // global variable to hold vm instance JSON
@@ -36,11 +36,7 @@ var instance;
36 36
 
37 37
 // --- Tests
38 38
 
39
-if (opts.skip) {
40
-    console.error('** skipping %s tests', __filename);
41
-    console.error('** set "allowWriteActions" in test config to enable');
42
-}
43
-test('triton manage workflow', opts, function (suite) {
39
+test('triton manage workflow', testOpts, function (suite) {
44 40
     h.printConfig(suite);
45 41
 
46 42
     suite.test('  cleanup existing inst with alias ' + INST_ALIAS,

+ 12
- 12
test/integration/cli-networks.test.js View File

@@ -27,18 +27,9 @@ var NET_NAME = f('nodetritontest-network-%s', os.hostname());
27 27
 var networks;
28 28
 var vlan;
29 29
 
30
-var OPTS = {
31
-    skip: !h.CONFIG.allowWriteActions
32
-};
33
-
34 30
 
35 31
 // --- Tests
36 32
 
37
-if (OPTS.skip) {
38
-    console.error('** skipping some %s tests', __filename);
39
-    console.error('** set "allowWriteActions" in test config to enable');
40
-}
41
-
42 33
 test('triton networks', function (suite) {
43 34
 
44 35
     suite.test('  setup: find a test VLAN', function (t) {
@@ -238,7 +229,9 @@ test('triton network get', function (suite) {
238 229
 });
239 230
 
240 231
 
241
-test('triton network create', OPTS, function (suite) {
232
+test('triton network create', {
233
+    skip: !h.CONFIG.allowWriteActions && 'requires config.allowWriteActions'
234
+}, function (suite) {
242 235
 
243 236
     suite.test('  cleanup: rm network ' + NET_NAME + ' if exists',
244 237
     function (t) {
@@ -273,7 +266,9 @@ test('triton network create', OPTS, function (suite) {
273 266
         });
274 267
     });
275 268
 
276
-    suite.test(' triton network create VLAN', function (t) {
269
+    suite.test(' triton network create VLAN', {
270
+        skip: !process.env.TEST_KNOWN_FAIL && 'known failure, see TRITON-1389'
271
+    }, function (t) {
277 272
         h.triton('network create --name=' + NET_NAME +
278 273
                  ' --subnet=192.168.97.0/24 --start_ip=192.168.97.1' +
279 274
                  ' --end_ip=192.168.97.254 -j ' + vlan.vlan_id,
@@ -300,7 +295,12 @@ test('triton network create', OPTS, function (suite) {
300 295
 });
301 296
 
302 297
 
303
-test('triton network delete', OPTS, function (suite) {
298
+test('triton network delete', {
299
+    skip: (
300
+        (!process.env.TEST_KNOWN_FAIL && 'known failure, see TRITON-1389') ||
301
+        (!h.CONFIG.allowWriteActions && 'requires config.allowWriteActions')
302
+    )
303
+}, function (suite) {
304 304
 
305 305
     suite.test(' triton network delete -h', function (t) {
306 306
         h.triton('network delete -h', function (err, stdout, stderr) {

+ 6
- 8
test/integration/cli-nics.test.js View File

@@ -25,19 +25,17 @@ var INST;
25 25
 var NIC;
26 26
 var NIC2;
27 27
 
28
-var OPTS = {
29
-    skip: !h.CONFIG.allowWriteActions
28
+var testOpts = {
29
+    skip: (
30
+        (!process.env.TEST_KNOWN_FAIL && 'known failure, see TRITON-1398') ||
31
+        (!h.CONFIG.allowWriteActions && 'requires config.allowVolumesTests')
32
+    )
30 33
 };
31 34
 
32 35
 
33 36
 // --- Tests
34 37
 
35
-if (OPTS.skip) {
36
-    console.error('** skipping %s tests', __filename);
37
-    console.error('** set "allowWriteActions" in test config to enable');
38
-}
39
-
40
-test('triton instance nics', OPTS, function (suite) {
38
+test('triton instance nics', testOpts, function (suite) {
41 39
     h.printConfig(suite);
42 40
 
43 41
     suite.test('  cleanup existing inst with alias ' + INST_ALIAS,

+ 3
- 8
test/integration/cli-profiles.test.js View File

@@ -21,18 +21,13 @@ var test = require('tap').test;
21 21
 var PROFILE_FILE = path.join(__dirname, 'test-profile.json');
22 22
 var PROFILE_DATA = JSON.parse(fs.readFileSync(PROFILE_FILE, 'utf8'));
23 23
 
24
-var opts = {
25
-    skip: !h.CONFIG.allowWriteActions
24
+var testOpts = {
25
+    skip: !h.CONFIG.allowWriteActions && 'requires config.allowWriteActions'
26 26
 };
27 27
 
28 28
 
29 29
 // --- Tests
30 30
 
31
-if (opts.skip) {
32
-    console.error('** skipping %s tests', __filename);
33
-    console.error('** set "allowWriteActions" in test config to enable');
34
-}
35
-
36 31
 test('triton profiles (read only)', function (suite) {
37 32
     suite.test('  triton profile get env', function (t) {
38 33
         h.safeTriton(t, {json: true, args: ['profile', 'get', '-j', 'env']},
@@ -52,7 +47,7 @@ test('triton profiles (read only)', function (suite) {
52 47
     suite.end();
53 48
 });
54 49
 
55
-test('triton profiles (read/write)', opts, function (suite) {
50
+test('triton profiles (read/write)', testOpts, function (suite) {
56 51
     suite.test('  triton profile create', function (t) {
57 52
         /*
58 53
          * We need to skip the Docker setup with '--no-docker' because we are

+ 7
- 8
test/integration/cli-snapshots.test.js View File

@@ -22,18 +22,17 @@ var test = require('tap').test;
22 22
 var SNAP_NAME = 'test-snapshot';
23 23
 var INST_ALIAS = f('nodetritontest-snapshots-%s', os.hostname());
24 24
 var INST;
25
-var OPTS = {
26
-    skip: !h.CONFIG.allowWriteActions
25
+
26
+var testOpts = {
27
+    skip: (
28
+        (!process.env.TEST_KNOWN_FAIL && 'known failure, see TRITON-1387') ||
29
+        (!h.CONFIG.allowWriteActions && 'requires config.allowWriteActions')
30
+    )
27 31
 };
28 32
 
29 33
 // --- Tests
30 34
 
31
-if (OPTS.skip) {
32
-    console.error('** skipping %s tests', __filename);
33
-    console.error('** set "allowWriteActions" in test config to enable');
34
-}
35
-
36
-test('triton instance snapshot', OPTS, function (suite) {
35
+test('triton instance snapshot', testOpts, function (suite) {
37 36
     h.printConfig(suite);
38 37
 
39 38
     suite.test('  cleanup existing inst with alias ' + INST_ALIAS,

+ 4
- 9
test/integration/cli-vlans.test.js View File

@@ -27,18 +27,13 @@ var VLAN_ID   = 3197;
27 27
 
28 28
 var VLAN;
29 29
 
30
-var OPTS = {
31
-    skip: !h.CONFIG.allowWriteActions
30
+var writeTestOpts = {
31
+    skip: !h.CONFIG.allowWriteActions && 'requires config.allowWriteActions'
32 32
 };
33 33
 
34 34
 
35 35
 // --- Tests
36 36
 
37
-if (OPTS.skip) {
38
-    console.error('** skipping some %s tests', __filename);
39
-    console.error('** set "allowWriteActions" in test config to enable');
40
-}
41
-
42 37
 test('triton vlan list', function (suite) {
43 38
 
44 39
     suite.test(' triton vlan list -h', function (t) {
@@ -281,7 +276,7 @@ test('triton vlan networks', function (suite) {
281 276
 });
282 277
 
283 278
 
284
-test('triton vlan create', OPTS, function (suite) {
279
+test('triton vlan create', writeTestOpts, function (suite) {
285 280
 
286 281
     suite.test('  cleanup: rm vlan ' + VLAN_NAME + ' if exists', function (t) {
287 282
         h.triton('vlan delete ' + VLAN_NAME, function onTriton(err, stdout) {
@@ -348,7 +343,7 @@ test('triton vlan create', OPTS, function (suite) {
348 343
 });
349 344
 
350 345
 
351
-test('triton vlan delete', OPTS, function (suite) {
346
+test('triton vlan delete', writeTestOpts, function (suite) {
352 347
 
353 348
     suite.test(' triton vlan delete -h', function (t) {
354 349
         h.triton('vlan delete -h', function onTriton(err, stdout, stderr) {

+ 4
- 6
test/integration/cli-volumes-size.test.js View File

@@ -25,13 +25,11 @@ var FABRIC_NETWORKS = [];
25 25
 var MIBS_IN_GIB = 1024;
26 26
 
27 27
 var testOpts = {
28
-    skip: !(h.CONFIG.allowWriteActions && h.CONFIG.allowVolumesTests)
28
+    skip: (
29
+        !(h.CONFIG.allowWriteActions && h.CONFIG.allowVolumesTests) &&
30
+        'requires config.allowWriteActions and config.allowVolumesTests'
31
+    )
29 32
 };
30
-if (testOpts.skip) {
31
-    console.error('** skipping %s tests', __filename);
32
-    console.error('** set "allowWriteActions" and "allowVolumesTests" '
33
-        + 'in test config to enable');
34
-}
35 33
 
36 34
 test('triton volume create with non-default size...', testOpts,
37 35
 function (suite) {

+ 6
- 8
test/integration/cli-volumes.test.js View File

@@ -20,16 +20,14 @@ var vasync = require('vasync');
20 20
 var common = require('../../lib/common');
21 21
 var h = require('./helpers');
22 22
 
23
-var FABRIC_NETWORKS = [];
24
-
25 23
 var testOpts = {
26
-    skip: !(h.CONFIG.allowWriteActions && h.CONFIG.allowVolumesTests)
24
+    skip: (
25
+        !(h.CONFIG.allowWriteActions && h.CONFIG.allowVolumesTests) &&
26
+        'requires config.allowWriteActions and config.allowVolumesTests'
27
+    )
27 28
 };
28
-if (testOpts.skip) {
29
-    console.error('** skipping %s tests', __filename);
30
-    console.error('** set "allowWriteActions" and "allowVolumesTests" '
31
-        + 'in test config to enable');
32
-}
29
+var FABRIC_NETWORKS = [];
30
+
33 31
 
34 32
 test('triton volume create ...', testOpts, function (suite) {
35 33
     var currentVolume;

Loading…
Cancel
Save