UI: address test flakiness, especially kmip role edit form (#28262)

* absolute hail mary

* what about this?

* that was not right

* nope

* refactor problematic test

* remove all of the runloop stuff, just chasing flaky tests

* chasing authPage

* move away from page objects for runCmd

* replace existing runCmd function

* add line

* test if removing chrome version helps this time?

* rerun tests

* rerun tests

* Revert "test if removing chrome version helps this time?"

This reverts commit 0b189c4f6978d6c55c283e3fe9fddd03d28c4377.

* remove await

* add trace log

* change test:oss command

* remove log tracing
This commit is contained in:
claire bontempo
2024-09-04 14:16:09 -07:00
committed by GitHub
parent 690520ad1b
commit 1238a187df
8 changed files with 162 additions and 318 deletions

View File

@@ -10,7 +10,7 @@ import apiPath from 'vault/utils/api-path';
import lazyCapabilities from 'vault/macros/lazy-capabilities';
import { removeManyFromArray } from 'vault/helpers/remove-from-array';
export const COMPUTEDS = {
const COMPUTEDS = {
operationFields: computed('newFields', function () {
return this.newFields.filter((key) => key.startsWith('operation'));
}),

View File

@@ -31,7 +31,7 @@
"start:chroot": "ember server --proxy=http://127.0.0.1:8300 --port=4300",
"test": "concurrently --kill-others-on-fail -P -c \"auto\" -n lint:js,lint:hbs,vault \"yarn:lint:js:quiet\" \"yarn:lint:hbs:quiet\" \"node scripts/start-vault.js {@}\" --",
"test:enos": "concurrently --kill-others-on-fail -P -c \"auto\" -n lint:js,lint:hbs,enos \"yarn:lint:js:quiet\" \"yarn:lint:hbs:quiet\" \"node scripts/enos-test-ember.js {@}\" --",
"test:oss": "yarn run test -f='!enterprise' --split=8 --preserve-test-name --parallel",
"test:oss": "yarn run test -f='!enterprise'",
"test:ent": "node scripts/start-vault.js -f='enterprise'",
"test:quick": "node scripts/start-vault.js --split=8 --preserve-test-name --parallel",
"test:quick-oss": "node scripts/start-vault.js -f='!enterprise' --split=8 --preserve-test-name --parallel",

View File

@@ -3,7 +3,16 @@
* SPDX-License-Identifier: BUSL-1.1
*/
import { currentURL, currentRouteName, settled, fillIn, waitUntil, find, click } from '@ember/test-helpers';
import {
currentURL,
currentRouteName,
settled,
fillIn,
visit,
waitUntil,
find,
click,
} from '@ember/test-helpers';
import { module, test } from 'qunit';
import { setupApplicationTest } from 'ember-qunit';
@@ -14,7 +23,7 @@ import credentialsPage from 'vault/tests/pages/secrets/backend/kmip/credentials'
import mountSecrets from 'vault/tests/pages/settings/mount-secret-backend';
import { GENERAL } from 'vault/tests/helpers/general-selectors';
import { allEngines } from 'vault/helpers/mountable-secret-engines';
import { runCmd } from 'vault/tests/helpers/commands';
import { mountEngineCmd, runCmd } from 'vault/tests/helpers/commands';
import { v4 as uuidv4 } from 'uuid';
// port has a lower limit of 1024
@@ -76,6 +85,7 @@ const generateCreds = async (backend) => {
}
return { backend, scope, role, serial };
};
module('Acceptance | Enterprise | KMIP secrets', function (hooks) {
setupApplicationTest(hooks);
@@ -347,4 +357,81 @@ module('Acceptance | Enterprise | KMIP secrets', function (hooks) {
assert.strictEqual(credentialsPage.listItemLinks.length, 0, 'renders no credentials');
assert.ok(credentialsPage.isEmpty, 'renders empty');
});
// the kmip/role model relies on openApi so testing the form via an acceptance test
module('kmip role edit form', function (hooks) {
hooks.beforeEach(async function () {
this.store = this.owner.lookup('service:store');
this.scope = 'my-scope';
this.name = 'my-role';
await authPage.login();
await runCmd(mountEngineCmd('kmip', this.backend), false);
await runCmd([`write ${this.backend}/scope/${this.scope} -force`]);
await rolesPage.visit({ backend: this.backend, scope: this.scope });
this.setModel = async () => {
await click('[data-test-edit-form-submit]');
await visit(`/vault/secrets/${this.backend}/kmip/scopes/${this.scope}/roles/${this.name}`);
this.model = this.store.peekRecord('kmip/role', this.name);
};
});
// "operationNone" is the attr name for the 'Allow this role to perform KMIP operations' toggle
// operationNone = false => the toggle is ON and KMIP operations are allowed
// operationNone = true => the toggle is OFF and KMIP operations are not allowed
test('it submits when operationNone is toggled on', async function (assert) {
assert.expect(3);
await click('[data-test-role-create]');
await fillIn(GENERAL.inputByAttr('name'), this.name);
assert.dom(GENERAL.inputByAttr('operationAll')).isChecked('operationAll is checked by default');
await this.setModel();
assert.true(this.model.operationAll, 'operationAll is true');
assert.strictEqual(this.model.operationNone, undefined, 'operationNone is unset');
});
test('it submits when operationNone is toggled off', async function (assert) {
assert.expect(4);
await click('[data-test-role-create]');
await fillIn(GENERAL.inputByAttr('name'), this.name);
await click(GENERAL.inputByAttr('operationNone'));
assert
.dom(GENERAL.inputByAttr('operationNone'))
.isNotChecked('Allow this role to perform KMIP operations is toggled off');
assert
.dom(GENERAL.inputByAttr('operationAll'))
.doesNotExist('clicking the toggle hides KMIP operation checkboxes');
await this.setModel();
assert.strictEqual(this.model.operationAll, undefined, 'operationAll is unset');
assert.true(this.model.operationNone, 'operationNone is true');
});
test('it submits when operationAll is unchecked', async function (assert) {
assert.expect(2);
await click('[data-test-role-create]');
await fillIn(GENERAL.inputByAttr('name'), this.name);
await click(GENERAL.inputByAttr('operationAll'));
await this.setModel();
assert.strictEqual(this.model.operationAll, undefined, 'operationAll is unset');
assert.true(this.model.operationNone, 'operationNone is true');
});
test('it submits individually selected operations', async function (assert) {
assert.expect(6);
await click('[data-test-role-create]');
await fillIn(GENERAL.inputByAttr('name'), this.name);
await click(GENERAL.inputByAttr('operationAll'));
await click(GENERAL.inputByAttr('operationGet'));
await click(GENERAL.inputByAttr('operationGetAttributes'));
assert.dom(GENERAL.inputByAttr('operationAll')).isNotChecked();
assert.dom(GENERAL.inputByAttr('operationCreate')).isNotChecked(); // unchecking operationAll deselects the other checkboxes
await this.setModel();
assert.strictEqual(this.model.operationAll, undefined, 'operationAll is unset');
assert.strictEqual(this.model.operationNone, undefined, 'operationNone is unset');
assert.true(this.model.operationGet, 'operationGet is true');
assert.true(this.model.operationGetAttributes, 'operationGetAttributes is true');
});
});
});

View File

@@ -10,8 +10,7 @@ import { setupMirage } from 'ember-cli-mirage/test-support';
import { click, currentURL, fillIn, visit, settled, find, waitFor, waitUntil } from '@ember/test-helpers';
import { v4 as uuidv4 } from 'uuid';
import authPage from 'vault/tests/pages/auth';
import logout from 'vault/tests/pages/logout';
import { login, logout } from 'vault/tests/helpers/auth/auth-helpers';
import enablePage from 'vault/tests/pages/settings/mount-secret-backend';
import { runCmd } from 'vault/tests/helpers/commands';
import { GENERAL } from 'vault/tests/helpers/general-selectors';
@@ -29,17 +28,17 @@ module('Acceptance | pki configuration test', function (hooks) {
hooks.beforeEach(async function () {
this.pemBundle = issuerPemBundle;
await authPage.login();
await login();
// Setup PKI engine
const mountPath = `pki-workflow-${uuidv4()}`;
await enablePage.enable('pki', mountPath);
this.mountPath = mountPath;
await logout.visit();
await logout();
});
hooks.afterEach(async function () {
await logout.visit();
await authPage.login();
await logout();
await login();
// Cleanup engine
await runCmd([`delete sys/mounts/${this.mountPath}`]);
});
@@ -48,7 +47,7 @@ module('Acceptance | pki configuration test', function (hooks) {
setupMirage(hooks);
test('it shows the delete all issuers modal', async function (assert) {
await authPage.login(this.pkiAdminToken);
await login(this.pkiAdminToken);
await visit(`/vault/secrets/${this.mountPath}/pki/configuration`);
await click(PKI_CONFIGURE_CREATE.configureButton);
assert.strictEqual(currentURL(), `/vault/secrets/${this.mountPath}/pki/configuration/create`);
@@ -78,7 +77,7 @@ module('Acceptance | pki configuration test', function (hooks) {
});
test('it shows the correct empty state message if certificates exists after delete all issuers', async function (assert) {
await authPage.login(this.pkiAdminToken);
await login(this.pkiAdminToken);
await visit(`/vault/secrets/${this.mountPath}/pki/configuration`);
await click(PKI_CONFIGURE_CREATE.configureButton);
assert.strictEqual(
@@ -157,7 +156,7 @@ module('Acceptance | pki configuration test', function (hooks) {
});
test('it shows the correct empty state message if roles and certificates exists after delete all issuers', async function (assert) {
await authPage.login(this.pkiAdminToken);
await login(this.pkiAdminToken);
// Configure PKI
await visit(`/vault/secrets/${this.mountPath}/pki/configuration`);
await click(PKI_CONFIGURE_CREATE.configureButton);
@@ -231,7 +230,7 @@ module('Acceptance | pki configuration test', function (hooks) {
// test coverage for ed25519 certs not displaying because the verify() function errors
test('it generates and displays a root issuer of key type = ed25519', async function (assert) {
assert.expect(4);
await authPage.login(this.pkiAdminToken);
await login(this.pkiAdminToken);
await visit(`/vault/secrets/${this.mountPath}/pki/overview`);
await click(GENERAL.secretTab('Issuers'));
await click(PKI_ISSUER_LIST.generateIssuerDropdown);

View File

@@ -11,14 +11,14 @@ import { v4 as uuidv4 } from 'uuid';
import { GENERAL } from 'vault/tests/helpers/general-selectors';
import page from 'vault/tests/pages/settings/auth/enable';
import listPage from 'vault/tests/pages/access/methods';
import authPage from 'vault/tests/pages/auth';
import { login } from 'vault/tests/helpers/auth/auth-helpers';
module('Acceptance | settings/auth/enable', function (hooks) {
setupApplicationTest(hooks);
hooks.beforeEach(function () {
this.uid = uuidv4();
return authPage.login();
return login();
});
test('it mounts and redirects', async function (assert) {
@@ -29,7 +29,7 @@ module('Acceptance | settings/auth/enable', function (hooks) {
assert.strictEqual(currentRouteName(), 'vault.cluster.settings.auth.enable');
await page.enable(type, path);
await settled();
await assert.strictEqual(
assert.strictEqual(
page.flash.latestMessage,
`Successfully mounted the ${type} auth method at ${path}.`,
'success flash shows'

View File

@@ -3,8 +3,13 @@
* SPDX-License-Identifier: BUSL-1.1
*/
import consoleClass from 'vault/tests/pages/components/console/ui-panel';
import { create } from 'ember-cli-page-object';
import { click, fillIn, findAll, triggerKeyEvent } from '@ember/test-helpers';
const REPL = {
toggle: '[data-test-console-toggle]',
consoleInput: '[data-test-component="console/command-input"] input',
logOutputItems: '[data-test-component="console/output-log"] > div',
};
/**
* Helper functions to run common commands in the consoleComponent during tests.
@@ -31,30 +36,47 @@ import { create } from 'ember-cli-page-object';
* }
*/
const cc = create(consoleClass);
/**
* runCmd is used to run commands and throw an error if the output includes "Error"
* @param {string || string[]} commands array of commands that should run
* @param {boolean} throwErrors
* @returns the last log output. Throws an error if it includes an error
*/
export async function runCmd(commands, throwErrors = true) {
export const runCmd = async (commands, throwErrors = true) => {
if (!commands) {
throw new Error('runCmd requires commands array passed in');
}
if (!Array.isArray(commands)) {
commands = [commands];
}
await cc.toggle();
await cc.runCommands(commands, false);
const lastOutput = cc.lastLogOutput;
await cc.toggle();
await click(REPL.toggle);
await enterCommands(commands);
const lastOutput = await lastLogOutput();
await click(REPL.toggle);
if (throwErrors && lastOutput.includes('Error')) {
throw new Error(`Error occurred while running commands: "${commands.join('; ')}" - ${lastOutput}`);
}
return lastOutput;
}
};
export const enterCommands = async (commands) => {
const toExecute = Array.isArray(commands) ? commands : [commands];
for (const command of toExecute) {
await fillIn(REPL.consoleInput, command);
await triggerKeyEvent(REPL.consoleInput, 'keyup', 'Enter');
}
};
export const lastLogOutput = async () => {
const items = findAll(REPL.logOutputItems);
const count = items.length;
if (count === 0) {
// If no logOutput items are found, we can assume the response is empty
return '';
}
const outputItemText = items[count - 1].innerText;
return outputItemText;
};
// Common commands
export function mountEngineCmd(type, customName = '') {

View File

@@ -1,238 +0,0 @@
/**
* Copyright (c) HashiCorp, Inc.
* SPDX-License-Identifier: BUSL-1.1
*/
import { later, run, _cancelTimers as cancelTimers } from '@ember/runloop';
import { resolve } from 'rsvp';
import EmberObject, { computed } from '@ember/object';
import Service from '@ember/service';
import { module, test } from 'qunit';
import { setupRenderingTest } from 'ember-qunit';
import { click, render, settled } from '@ember/test-helpers';
import hbs from 'htmlbars-inline-precompile';
import sinon from 'sinon';
import { setupEngine } from 'ember-engines/test-support';
import { COMPUTEDS } from 'vault/models/kmip/role';
const flash = Service.extend({
success: sinon.stub(),
});
const namespace = Service.extend({});
const fieldToCheckbox = (field) => ({ name: field, type: 'boolean' });
const createModel = (options) => {
const model = EmberObject.extend(COMPUTEDS, {
/* eslint-disable ember/avoid-leaking-state-in-ember-objects */
newFields: [
'role',
'operationActivate',
'operationAddAttribute',
'operationAll',
'operationCreate',
'operationDestroy',
'operationDiscoverVersion',
'operationGet',
'operationGetAttributes',
'operationLocate',
'operationNone',
'operationRekey',
'operationRevoke',
'tlsClientKeyBits',
'tlsClientKeyType',
'tlsClientTtl',
],
fields: computed('operationFields', function () {
return this.operationFields.map(fieldToCheckbox);
}),
destroyRecord() {
return resolve();
},
save() {
return resolve();
},
rollbackAttributes() {},
});
return model.create({
...options,
});
};
module('Integration | Component | edit form kmip role', function (hooks) {
setupRenderingTest(hooks);
setupEngine(hooks, 'kmip');
hooks.beforeEach(function () {
this.context = { owner: this.engine }; // this.engine set by setupEngine
run(() => {
this.engine.unregister('service:flash-messages');
this.engine.register('service:flash-messages', flash);
this.engine.register('service:namespace', namespace);
});
});
test('it renders: new model', async function (assert) {
assert.expect(3);
const model = createModel({ isNew: true });
this.set('model', model);
this.onSave = ({ model }) => {
assert.false(model.operationNone, 'callback fires with operationNone as false');
assert.true(model.operationAll, 'callback fires with operationAll as true');
};
await render(hbs`<EditFormKmipRole @model={{this.model}} @onSave={{this.onSave}} />`, this.context);
assert.dom('[data-test-input="operationAll"]').isChecked('sets operationAll');
await click('[data-test-edit-form-submit]');
});
test('it renders: operationAll', async function (assert) {
assert.expect(3);
const model = createModel({ operationAll: true });
this.set('model', model);
this.onSave = ({ model }) => {
assert.false(model.operationNone, 'callback fires with operationNone as false');
assert.true(model.operationAll, 'callback fires with operationAll as true');
};
await render(hbs`<EditFormKmipRole @model={{this.model}} @onSave={{this.onSave}} />`, this.context);
assert.dom('[data-test-input="operationAll"]').isChecked('sets operationAll');
await click('[data-test-edit-form-submit]');
});
test('it renders: operationNone', async function (assert) {
assert.expect(2);
const model = createModel({ operationNone: true, operationAll: undefined });
this.set('model', model);
this.onSave = ({ model }) => {
assert.true(model.operationNone, 'callback fires with operationNone as true');
};
await render(hbs`<EditFormKmipRole @model={{this.model}} @onSave={{this.onSave}} />`, this.context);
assert.dom('[data-test-input="operationNone"]').isNotChecked('sets operationNone');
await click('[data-test-edit-form-submit]');
});
test('it renders: choose operations', async function (assert) {
assert.expect(3);
const model = createModel({ operationGet: true });
this.set('model', model);
this.onSave = ({ model }) => {
assert.false(model.operationNone, 'callback fires with operationNone as false');
};
await render(hbs`<EditFormKmipRole @model={{this.model}} @onSave={{this.onSave}} />`, this.context);
assert.dom('[data-test-input="operationNone"]').isChecked('sets operationNone');
assert.dom('[data-test-input="operationAll"]').isNotChecked('sets operationAll');
await click('[data-test-edit-form-submit]');
});
test('it saves operationNone=true when unchecking operationAll box', async function (assert) {
assert.expect(15);
const model = createModel({ isNew: true });
this.set('model', model);
this.onSave = ({ model }) => {
assert.true(model.operationNone, 'callback fires with operationNone as true');
assert.false(model.operationAll, 'callback fires with operationAll as false');
};
await render(hbs`<EditFormKmipRole @model={{this.model}} @onSave={{this.onSave}} />`, this.context);
await click('[data-test-input="operationAll"]');
for (const field of model.fields) {
const { name } = field;
if (name === 'operationNone') continue;
assert.dom(`[data-test-input="${name}"]`).isNotChecked(`${name} is unchecked`);
}
assert.dom('[data-test-input="operationAll"]').isNotChecked('sets operationAll');
assert
.dom('[data-test-input="operationNone"]')
.isChecked('operationNone toggle is true which means allow operations');
await click('[data-test-edit-form-submit]');
});
const savingTests = [
[
'setting operationAll',
{ operationNone: true, operationGet: true },
'operationNone',
{
operationAll: true,
operationNone: false,
operationGet: true,
},
{
operationGet: null,
operationNone: false,
},
5,
],
[
'setting operationNone',
{ operationAll: true, operationCreate: true },
'operationNone',
{
operationAll: false,
operationNone: true,
operationCreate: true,
},
{
operationNone: true,
operationCreate: null,
operationAll: false,
},
6,
],
[
'setting choose, and selecting an additional item',
{ operationAll: true, operationGet: true, operationCreate: true },
'operationAll,operationDestroy',
{
operationAll: false,
operationCreate: true,
operationGet: true,
},
{
operationGet: true,
operationCreate: true,
operationDestroy: true,
operationAll: false,
},
7,
],
];
for (const testCase of savingTests) {
const [name, initialState, displayClicks, stateBeforeSave, stateAfterSave, assertionCount] = testCase;
test(name, async function (assert) {
assert.expect(assertionCount);
const model = createModel(initialState);
this.set('model', model);
const clickTargets = displayClicks.split(',');
await render(hbs`<EditFormKmipRole @model={{this.model}} />`, this.context);
for (const clickTarget of clickTargets) {
await click(`label[for=${clickTarget}]`);
}
for (const beforeStateKey of Object.keys(stateBeforeSave)) {
assert.strictEqual(
model.get(beforeStateKey),
stateBeforeSave[beforeStateKey],
`sets ${beforeStateKey}`
);
}
await click('[data-test-edit-form-submit]');
later(() => cancelTimers(), 50);
await settled();
for (const afterStateKey of Object.keys(stateAfterSave)) {
assert.strictEqual(
model.get(afterStateKey),
stateAfterSave[afterStateKey],
`sets ${afterStateKey} on save`
);
}
});
}
});

View File

@@ -3,77 +3,51 @@
* SPDX-License-Identifier: BUSL-1.1
*/
import { later, run, _cancelTimers as cancelTimers } from '@ember/runloop';
import { resolve } from 'rsvp';
import EmberObject from '@ember/object';
import Service from '@ember/service';
import { module, test } from 'qunit';
import { setupRenderingTest } from 'ember-qunit';
import { render, settled } from '@ember/test-helpers';
import { click, render } from '@ember/test-helpers';
import hbs from 'htmlbars-inline-precompile';
import sinon from 'sinon';
import { create } from 'ember-cli-page-object';
import editForm from 'vault/tests/pages/components/edit-form';
const component = create(editForm);
const flash = Service.extend({
success: sinon.stub(),
});
const createModel = (canDelete = true) => {
return EmberObject.create({
fields: [
{ name: 'one', type: 'string' },
{ name: 'two', type: 'boolean' },
],
canDelete,
destroyRecord() {
return resolve();
},
save() {
return resolve();
},
rollbackAttributes() {},
});
};
import { GENERAL } from 'vault/tests/helpers/general-selectors';
module('Integration | Component | edit form', function (hooks) {
setupRenderingTest(hooks);
hooks.beforeEach(function () {
run(() => {
this.owner.unregister('service:flash-messages');
this.owner.register('service:flash-messages', flash);
this.model = EmberObject.create({
fields: [
{ name: 'one', type: 'string' },
{ name: 'two', type: 'boolean' },
],
destroyRecord() {},
save() {},
rollbackAttributes() {},
});
this.onSave = sinon.spy();
this.renderComponent = () =>
render(hbs`
<EditForm @model={{this.model}} @onSave={{this.onSave}} />
`);
});
test('it renders', async function (assert) {
this.set('model', createModel());
await render(hbs`{{edit-form model=this.model}}`);
assert.ok(component.fields.length, 2);
await this.renderComponent();
assert.dom(GENERAL.fieldByAttr('one')).exists();
assert.dom(GENERAL.fieldByAttr('two')).exists();
});
test('it calls flash message fns on save', async function (assert) {
assert.expect(4);
const onSave = () => {
return resolve();
};
this.set('model', createModel());
this.set('onSave', onSave);
const saveSpy = sinon.spy(this, 'onSave');
await render(hbs`{{edit-form model=this.model onSave=this.onSave}}`);
component.submit();
later(() => cancelTimers(), 50);
await settled();
assert.true(saveSpy.calledOnce, 'calls passed onSave');
assert.strictEqual(saveSpy.getCall(0).args[0].saveType, 'save');
assert.deepEqual(saveSpy.getCall(0).args[0].model, this.model, 'passes model to onSave');
const flash = this.owner.lookup('service:flash-messages');
assert.strictEqual(flash.success.callCount, 1, 'calls flash message success');
this.flashSuccessSpy = sinon.spy(flash, 'success');
await this.renderComponent();
await click('[data-test-edit-form-submit]');
const { saveType, model } = this.onSave.lastCall.args[0];
const [flashMessage] = this.flashSuccessSpy.lastCall.args;
assert.strictEqual(flashMessage, 'Saved!');
assert.strictEqual(saveType, 'save');
assert.strictEqual(saveType, 'save');
assert.deepEqual(model, this.model, 'passes model to onSave');
});
});