[QTI-188] Update the UI tests to be able to run against a cluster deployed to AWS. Add build hooks (package.json/Makefile) to execute ui tests with a real backend. (#14396)

ui/package.json

@@ -26,6 +26,7 @@
     "start2": "ember server --proxy=http://localhost:8202 --port=4202",
     "start:mirage": "start () { MIRAGE_DEV_HANDLER=$1 yarn run start; }; start",
     "test": "npm-run-all lint:js:quiet lint:hbs:quiet && node scripts/start-vault.js",
+    "test:enos": "npm-run-all lint:js:quiet lint:hbs:quiet && node scripts/enos-test-ember.js",
     "test:oss": "yarn run test -f='!enterprise'",
     "test:browserstack": "export CI=true; node scripts/start-vault.js --browserstack",
     "test:quick": "node scripts/start-vault.js",

ui/scripts/enos-test-ember.js

@@ -0,0 +1,60 @@
+/* eslint-env node */
+/* eslint-disable no-console */
+
+const testHelper = require('./test-helper');
+
+(async function () {
+  try {
+    let unsealKeys = process.env.VAULT_UNSEAL_KEYS;
+    if (!unsealKeys) {
+      console.error(
+        'Cannot run ember tests without unseal keys, please make sure to export the keys, in an env ' +
+          'var named: VAULT_UNSEAL_KEYS'
+      );
+      process.exit(1);
+    } else {
+      unsealKeys = JSON.parse(unsealKeys);
+    }
+
+    const rootToken = process.env.VAULT_TOKEN;
+    if (!rootToken) {
+      console.error(
+        'Cannot run ember tests without root token, please make sure to export the root token, in an env ' +
+          'var named: VAULT_TOKEN'
+      );
+      process.exit(1);
+    }
+
+    testHelper.writeKeysFile(unsealKeys, rootToken);
+  } catch (error) {
+    console.log(error);
+    process.exit(1);
+  }
+
+  const vaultAddr = process.env.VAULT_ADDR;
+  if (!vaultAddr) {
+    console.error(
+      'Cannot run ember tests without the Vault Address, please make sure to export the vault address, in an env ' +
+        'var named: VAULT_ADDR'
+    );
+    process.exit(1);
+  }
+
+  console.log('VAULT_ADDR=' + vaultAddr);
+
+  try {
+    const testArgs = ['test', '-c', 'testem.enos.js'];
+
+    if (process.env.TEST_FILTERS) {
+      const filters = JSON.parse(process.env.TEST_FILTERS).map((filter) => '-f=' + filter);
+      testArgs.push(...filters);
+    }
+
+    await testHelper.run('ember', testArgs, false);
+  } catch (error) {
+    console.log(error);
+    process.exit(1);
+  } finally {
+    process.exit(0);
+  }
+})();

ui/scripts/start-vault.js

@@ -3,24 +3,8 @@
 /* eslint-disable no-process-exit */
 /* eslint-disable node/no-extraneous-require */
 
-var fs = require('fs');
-var path = require('path');
 var readline = require('readline');
-var execa = require('execa');
+const testHelper = require('./test-helper');
-var chalk = require('chalk');
-
-function run(command, args = [], shareStd = true) {
-  console.log(chalk.dim('$ ' + command + ' ' + args.join(' ')));
-  // cleanup means that execa will handle stopping the vault subprocess
-  // inherit all of the stdin/out/err so that testem still works as if you were running it directly
-  if (shareStd) {
-    return execa(command, args, { cleanup: true, stdin: 'inherit', stdout: 'inherit', stderr: 'inherit' });
-  }
-  let p = execa(command, args, { cleanup: true });
-  p.stdout.pipe(process.stdout);
-  p.stderr.pipe(process.stderr);
-  return p;
-}
 
 var output = '';
 var unseal, root, written, initError;
@@ -37,7 +21,7 @@ async function processLines(input, eachLine = () => {}) {
 
 (async function () {
   try {
-    let vault = run(
+    let vault = testHelper.run(
       'vault',
       [
         'server',
@@ -57,7 +41,7 @@ async function processLines(input, eachLine = () => {}) {
       output = output + line;
       var unsealMatch = output.match(/Unseal Key: (.+)$/m);
       if (unsealMatch && !unseal) {
-        unseal = unsealMatch[1];
+        unseal = [unsealMatch[1]];
       }
       var rootMatch = output.match(/Root Token: (.+)$/m);
       if (rootMatch && !root) {
@@ -68,13 +52,7 @@ async function processLines(input, eachLine = () => {}) {
         initError = errorMatch[1];
       }
       if (root && unseal && !written) {
-        fs.writeFile(
+        testHelper.writeKeysFile(unseal, root);
-          path.join(process.cwd(), 'tests/helpers/vault-keys.js'),
-          `export default ${JSON.stringify({ unseal, root }, null, 2)}`,
-          (err) => {
-            if (err) throw err;
-          }
-        );
         written = true;
         console.log('VAULT SERVER READY');
       } else if (initError) {
@@ -87,20 +65,20 @@ async function processLines(input, eachLine = () => {}) {
     });
     try {
       if (process.argv[2] === '--browserstack') {
-        await run('ember', ['browserstack:connect']);
+        await testHelper.run('ember', ['browserstack:connect']);
         try {
-          await run('ember', ['test', '-f=secrets/secret/create', '-c', 'testem.browserstack.js']);
+          await testHelper.run('ember', ['test', '-f=secrets/secret/create', '-c', 'testem.browserstack.js']);
 
           console.log('success');
           process.exit(0);
         } finally {
           if (process.env.CI === 'true') {
-            await run('ember', ['browserstack:results']);
+            await testHelper.run('ember', ['browserstack:results']);
           }
-          await run('ember', ['browserstack:disconnect']);
+          await testHelper.run('ember', ['browserstack:disconnect']);
         }
       } else {
-        await run('ember', ['test', ...process.argv.slice(2)]);
+        await testHelper.run('ember', ['test', ...process.argv.slice(2)]);
       }
     } catch (error) {
       console.log(error);

ui/scripts/test-helper.js

@@ -0,0 +1,54 @@
+/* eslint-env node */
+/* eslint-disable no-console */
+
+const fs = require('fs');
+const path = require('path');
+const chalk = require('chalk');
+const execa = require('execa');
+
+/**
+ * Writes a vault keys file that can be imported in other scripts, that includes the unseal keys and the root token.
+ * @param unsealKeys an array of unseal keys, must contain at least one key
+ * @param rootToken the root token
+ * @param filePath optional file path, if not provided the default path of <cwd>/tests/helpers/vault-keys.js
+ * will be used.
+ */
+function writeKeysFile(unsealKeys, rootToken, filePath) {
+  if (filePath === undefined) {
+    filePath = path.join(process.cwd(), 'tests/helpers/vault-keys.js');
+  }
+  let keys = {};
+  keys.unsealKeys = unsealKeys;
+  keys.rootToken = rootToken;
+
+  fs.writeFile(filePath, `export default ${JSON.stringify(keys, null, 2)}`, (err) => {
+    if (err) throw err;
+  });
+}
+
+/**
+ * Runs the provided command and pipes the processes stdout and stderr to the terminal. Upon completion with
+ * success or error the child process will be cleaned up.
+ * @param command some command to run
+ * @param args some arguments for the command to run
+ * @param shareStd if true the sub process created by the command will share the stdout and stderr of the parent
+ * process
+ * @returns {*} The child_process for the executed command which is also a Promise.
+ */
+function run(command, args = [], shareStd = true) {
+  console.log(chalk.dim('$ ' + command + ' ' + args.join(' ')));
+  // cleanup means that execa will handle stopping the subprocess
+  // inherit all of the stdin/out/err so that testem still works as if you were running it directly
+  if (shareStd) {
+    return execa(command, args, { cleanup: true, stdin: 'inherit', stdout: 'inherit', stderr: 'inherit' });
+  }
+  let p = execa(command, args, { cleanup: true });
+  p.stdout.pipe(process.stdout);
+  p.stderr.pipe(process.stderr);
+  return p;
+}
+
+module.exports = {
+  writeKeysFile: writeKeysFile,
+  run: run,
+};

ui/testem.enos.js

@@ -0,0 +1,40 @@
+/* eslint-env node */
+
+'use strict';
+
+const vault_addr = process.env.VAULT_ADDR;
+console.log('VAULT_ADDR=' + vault_addr);
+
+module.exports = {
+  test_page: 'tests/index.html?hidepassed',
+  tap_quiet_logs: true,
+  tap_failed_tests_only: true,
+  disable_watching: true,
+  launch_in_ci: ['Chrome'],
+  browser_start_timeout: 120,
+  browser_args: {
+    Chrome: {
+      ci: [
+        // --no-sandbox is needed when running Chrome inside a container
+        process.env.CI ? '--no-sandbox' : null,
+        '--headless',
+        '--disable-dev-shm-usage',
+        '--disable-software-rasterizer',
+        '--mute-audio',
+        '--remote-debugging-port=0',
+        '--window-size=1440,900',
+      ].filter(Boolean),
+    },
+  },
+  proxies: {
+    '/v1': {
+      target: vault_addr,
+    },
+  },
+};
+
+if (process.env.CI) {
+  module.exports.reporter = 'xunit';
+  module.exports.report_file = 'test-results/qunit/results.xml';
+  module.exports.xunit_intermediate_output = true;
+}

ui/tests/acceptance/unseal-test.js

@@ -1,4 +1,4 @@
-import { click, fillIn, currentURL, currentRouteName, visit, settled } from '@ember/test-helpers';
+import { click, currentRouteName, currentURL, fillIn, settled, visit } from '@ember/test-helpers';
 import { module, test } from 'qunit';
 import { setupApplicationTest } from 'ember-qunit';
 import VAULT_KEYS from 'vault/tests/helpers/vault-keys';
@@ -6,7 +6,7 @@ import authPage from 'vault/tests/pages/auth';
 import logout from 'vault/tests/pages/logout';
 import { pollCluster } from 'vault/tests/helpers/poll-cluster';
 
-const { unseal } = VAULT_KEYS;
+const { unsealKeys } = VAULT_KEYS;
 
 module('Acceptance | unseal', function (hooks) {
   setupApplicationTest(hooks);
@@ -34,12 +34,15 @@ module('Acceptance | unseal', function (hooks) {
     assert.equal(currentURL(), '/vault/unseal', 'vault is on the unseal page');
 
     // unseal
-    await fillIn('[data-test-shamir-input]', unseal);
+    for (const key of unsealKeys) {
+      await fillIn('[data-test-shamir-input]', key);
 
-    await click('button[type="submit"]');
+      await click('button[type="submit"]');
+
+      await pollCluster(this.owner);
+      await settled();
+    }
 
-    await pollCluster(this.owner);
-    await settled();
     assert.dom('[data-test-cluster-status]').doesNotExist('ui does not show sealed warning');
     assert.equal(currentRouteName(), 'vault.cluster.auth', 'vault is ready to authenticate');
   });

ui/tests/pages/auth.js

@@ -1,5 +1,8 @@
 import { create, visitable, fillable, clickable } from 'ember-cli-page-object';
 import { settled } from '@ember/test-helpers';
+import VAULT_KEYS from 'vault/tests/helpers/vault-keys';
+
+const { rootToken } = VAULT_KEYS;
 
 export default create({
   visit: visitable('/vault/auth'),
@@ -19,7 +22,7 @@ export default create({
       return;
     }
 
-    await this.tokenInput('root').submit();
+    await this.tokenInput(rootToken).submit();
     return;
   },
 });