test(e2e): repair the harness, taking the browser suite from 96 failures to 11
Three separate reasons tests were failing, none of them a defect in the app. The calculators. e2e-harness.html loaded calculators.js and drugs-loader.js with `defer` after they were split into ES modules; index.html was updated at the time and this page was not. A module parsed as a classic script throws "Cannot use import statement outside a module" before a line runs, so no click handler was ever attached: the pills rendered from static HTML and did nothing. Only the first calculator appeared to pass, because it carries `active` in the markup and needs no click. That was 52 failures. Settings and FAQ. Both moved from the tab rail into the account-card menu; the helper still clicked button.tab-btn[data-tab=…] and timed out. Ten more. The AI mocks, which had stopped intercepting for two independent reasons and so were calling the real model on every run — spending credits and comparing genuine output against strings like "MOCK HPI from dictation". A '**/api/x' glob matches no URL on Playwright 1.50, and page.route fails silently when nothing matches; measured against a real URL, that glob and '*/**/api/x' both matched zero times where a regex matched. Fixing that alone was not enough: the app registers a service worker that answers every /api/ request with its own fetch(), and a request made inside a service worker never reaches page.route. Blocking registration in the config puts them back in the page. The mocked dictation test now finishes in 1.6s rather than 7.5s, which is what a real model call costs. Whole suite: 204 passed / 96 failed in 15.8 minutes, now 289 passed / 11 failed in 6.8. The remaining eleven are spread across nine specs with no shared cause and are not touched here. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01Dv6sqaY6Vq3ChZHMem3cnU
This commit is contained in:
parent
5577ec301c
commit
05dcd1146d
5 changed files with 90 additions and 4 deletions
|
|
@ -85,6 +85,21 @@ async function loginAs(context, request, email = TEST_EMAIL) {
|
||||||
}]);
|
}]);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// A '**/api/x' glob stopped matching any URL when Playwright went to 1.50, and
|
||||||
|
// page.route fails silently: no error, no warning, the request simply goes to
|
||||||
|
// the server. So every "mocked" AI test was calling the real model and
|
||||||
|
// comparing its genuine output against a canned string — spending real credits
|
||||||
|
// on every run and failing for a reason that looked like a UI bug. Measured:
|
||||||
|
// against http://127.0.0.1:3553/api/health, '**/api/health' and '*/**/api/health'
|
||||||
|
// both matched zero times; a regex matched.
|
||||||
|
//
|
||||||
|
// The patterns are kept as strings because they are also the keys callers pass
|
||||||
|
// in `overrides`, and turned into anchored regexes here.
|
||||||
|
function asMatcher(pattern) {
|
||||||
|
const path = pattern.replace(/^\*\*/, '');
|
||||||
|
return new RegExp(path.replace(/[.*+?^${}()|[\]\\]/g, '\\$&') + '(?:[?#]|$)');
|
||||||
|
}
|
||||||
|
|
||||||
// ── AI mock — intercepts generation endpoints ──────────────
|
// ── AI mock — intercepts generation endpoints ──────────────
|
||||||
// Canned response shape matches what each route's frontend expects.
|
// Canned response shape matches what each route's frontend expects.
|
||||||
// Override per-test by passing {pattern: responseFn} in overrides.
|
// Override per-test by passing {pattern: responseFn} in overrides.
|
||||||
|
|
@ -111,7 +126,7 @@ async function mockAI(page, overrides = {}) {
|
||||||
|
|
||||||
for (const { pattern, response } of routes) {
|
for (const { pattern, response } of routes) {
|
||||||
const override = overrides[pattern];
|
const override = overrides[pattern];
|
||||||
await page.route(pattern, async route => {
|
await page.route(asMatcher(pattern), async route => {
|
||||||
const resp = typeof override === 'function' ? await override(route.request()) : (override || response);
|
const resp = typeof override === 'function' ? await override(route.request()) : (override || response);
|
||||||
await route.fulfill({ status: 200, contentType: 'application/json', body: JSON.stringify(resp) });
|
await route.fulfill({ status: 200, contentType: 'application/json', body: JSON.stringify(resp) });
|
||||||
});
|
});
|
||||||
|
|
|
||||||
|
|
@ -28,6 +28,12 @@ module.exports = defineConfig({
|
||||||
reporter: [['list']],
|
reporter: [['list']],
|
||||||
use: {
|
use: {
|
||||||
baseURL: process.env.BASE_URL || 'http://127.0.0.1:3553',
|
baseURL: process.env.BASE_URL || 'http://127.0.0.1:3553',
|
||||||
|
// The app registers a service worker that answers every /api/ request with
|
||||||
|
// its own fetch(). A request made inside a service worker never reaches
|
||||||
|
// page.route, so mockAI could not intercept anything while one was running
|
||||||
|
// and the tests called the real model. Blocking registration puts the
|
||||||
|
// requests back in the page, where the mocks can see them.
|
||||||
|
serviceWorkers: 'block',
|
||||||
trace: 'retain-on-failure',
|
trace: 'retain-on-failure',
|
||||||
screenshot: 'only-on-failure',
|
screenshot: 'only-on-failure',
|
||||||
actionTimeout: 5_000,
|
actionTimeout: 5_000,
|
||||||
|
|
|
||||||
|
|
@ -5,6 +5,12 @@
|
||||||
|
|
||||||
const { test, expect, E2E_BASE, mockAI } = require('../fixtures');
|
const { test, expect, E2E_BASE, mockAI } = require('../fixtures');
|
||||||
|
|
||||||
|
// Settings and FAQ are not on the tab rail. They live in the account-card menu
|
||||||
|
// alongside Admin, and this helper used to click button.tab-btn[data-tab=…] for
|
||||||
|
// them, which simply timed out — the cause of ten of these failures. Dictation
|
||||||
|
// really is a rail tab, so both routes are needed.
|
||||||
|
const ACCOUNT_MENU = ['settings', 'faq'];
|
||||||
|
|
||||||
async function openTab(page, name) {
|
async function openTab(page, name) {
|
||||||
await page.goto(E2E_BASE + '/');
|
await page.goto(E2E_BASE + '/');
|
||||||
await page.waitForSelector('button.tab-btn', { timeout: 15000 });
|
await page.waitForSelector('button.tab-btn', { timeout: 15000 });
|
||||||
|
|
@ -12,7 +18,12 @@ async function openTab(page, name) {
|
||||||
if (vp && vp.width <= 768) {
|
if (vp && vp.width <= 768) {
|
||||||
await page.click('#btn-menu-toggle').catch(() => {});
|
await page.click('#btn-menu-toggle').catch(() => {});
|
||||||
}
|
}
|
||||||
|
if (ACCOUNT_MENU.includes(name)) {
|
||||||
|
await page.locator('.account-card-btn').first().click();
|
||||||
|
await page.locator(`[data-account-tab="${name}"]`).first().click();
|
||||||
|
} else {
|
||||||
await page.click(`button.tab-btn[data-tab="${name}"]`);
|
await page.click(`button.tab-btn[data-tab="${name}"]`);
|
||||||
|
}
|
||||||
await page.waitForFunction((t) => {
|
await page.waitForFunction((t) => {
|
||||||
const el = document.getElementById(t + '-tab');
|
const el = document.getElementById(t + '-tab');
|
||||||
return el && el.classList.contains('active') && el.innerHTML.trim().length > 100;
|
return el && el.classList.contains('active') && el.innerHTML.trim().length > 100;
|
||||||
|
|
|
||||||
|
|
@ -20,9 +20,17 @@
|
||||||
calc-nav-pill (which no longer exists). -->
|
calc-nav-pill (which no longer exists). -->
|
||||||
<section id="bedside-tab" class="tab-content active" data-component="bedside"></section>
|
<section id="bedside-tab" class="tab-content active" data-component="bedside"></section>
|
||||||
|
|
||||||
|
<!-- calculators.js and drugs-loader.js are ES modules and must be loaded as
|
||||||
|
modules. Loaded with `defer` instead, the browser parses them as classic
|
||||||
|
scripts, throws "Cannot use import statement outside a module" before a
|
||||||
|
single line runs, and no click handler is ever attached: the pills render
|
||||||
|
from static HTML and do nothing. index.html has always had this right;
|
||||||
|
this page was left behind when the calculators were split into modules,
|
||||||
|
which is what 52 of the e2e failures were. Keep these in step with
|
||||||
|
index.html. -->
|
||||||
<script defer src="/js/calc-math.js"></script>
|
<script defer src="/js/calc-math.js"></script>
|
||||||
<script defer src="/js/drugs-loader.js"></script>
|
<script type="module" src="/js/drugs-loader.js"></script>
|
||||||
<script defer src="/js/calculators.js"></script>
|
<script type="module" src="/js/calculators.js"></script>
|
||||||
<script type="module" src="/js/bedside/index.js"></script>
|
<script type="module" src="/js/bedside/index.js"></script>
|
||||||
<script defer src="/js/e2e-bootstrap.js"></script>
|
<script defer src="/js/e2e-bootstrap.js"></script>
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -58,3 +58,49 @@ test('the e2e accounts are seeded, and the seed cannot touch a real one', () =>
|
||||||
// And the runner seeds before it tests, so nobody has to remember to.
|
// And the runner seeds before it tests, so nobody has to remember to.
|
||||||
assert.match(read('scripts/e2e.sh'), /node e2e\/seed\.js/);
|
assert.match(read('scripts/e2e.sh'), /node e2e\/seed\.js/);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
test('the harness loads the calculators the way the app does', () => {
|
||||||
|
// calculators.js and drugs-loader.js are ES modules. Loaded with `defer` they
|
||||||
|
// are parsed as classic scripts and throw "Cannot use import statement outside
|
||||||
|
// a module" before a line runs, so no click handler is ever attached: the
|
||||||
|
// pills render from static HTML and do nothing. Measured in the browser, and
|
||||||
|
// it was 52 of the 96 e2e failures. Only the first calculator "passed",
|
||||||
|
// because it carries `active` in the markup and needs no click.
|
||||||
|
const harness = read('public/e2e-harness.html');
|
||||||
|
const app = read('public/index.html');
|
||||||
|
for (const file of ['calculators.js', 'drugs-loader.js']) {
|
||||||
|
const inApp = new RegExp('<script type="module" src="/js/' + file.replace('.', '\\.') + '"');
|
||||||
|
assert.match(app, inApp, file + ' is a module in the app');
|
||||||
|
assert.match(harness, inApp, 'so the harness must load it the same way');
|
||||||
|
}
|
||||||
|
assert.doesNotMatch(harness, /<script defer src="\/js\/(calculators|drugs-loader)\.js"/);
|
||||||
|
});
|
||||||
|
|
||||||
|
test('the AI mocks actually intercept, rather than calling the real model', () => {
|
||||||
|
const fixtures = read('e2e/fixtures.js');
|
||||||
|
// Two separate reasons the mocks were silently doing nothing, both measured.
|
||||||
|
//
|
||||||
|
// One: a '**/api/x' glob matches no URL on Playwright 1.50, and page.route
|
||||||
|
// fails silently when nothing matches.
|
||||||
|
assert.match(fixtures, /function asMatcher\(pattern\)/);
|
||||||
|
assert.match(fixtures, /await page\.route\(asMatcher\(pattern\)/);
|
||||||
|
|
||||||
|
// Two: the app registers a service worker that answers every /api/ request
|
||||||
|
// with its own fetch(), and a request made inside a service worker never
|
||||||
|
// reaches page.route. Blocking registration puts them back in the page.
|
||||||
|
assert.match(read('public/sw.js'), /url\.pathname\.startsWith\('\/api\/'\)/,
|
||||||
|
'if the worker stops handling /api this guard can be revisited');
|
||||||
|
assert.match(read('e2e/playwright.config.js'), /serviceWorkers: 'block'/);
|
||||||
|
});
|
||||||
|
|
||||||
|
test('Settings and FAQ are opened the way the app actually offers them', () => {
|
||||||
|
// Neither is on the tab rail; both live in the account-card menu with Admin.
|
||||||
|
const index = read('public/index.html');
|
||||||
|
for (const name of ['settings', 'faq']) {
|
||||||
|
assert.doesNotMatch(index, new RegExp('data-tab="' + name + '"'), name + ' is not a rail tab');
|
||||||
|
assert.match(index, new RegExp('data-account-tab="' + name + '"'));
|
||||||
|
}
|
||||||
|
const spec = read('e2e/tests/settings-faq-dictation.spec.js');
|
||||||
|
assert.match(spec, /const ACCOUNT_MENU = \['settings', 'faq'\];/);
|
||||||
|
assert.match(spec, /\[data-account-tab="\$\{name\}"\]/);
|
||||||
|
});
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue