test(e2e): repair the harness, taking the browser suite from 96 failures to 11

Three separate reasons tests were failing, none of them a defect in the app.

The calculators. e2e-harness.html loaded calculators.js and drugs-loader.js with
`defer` after they were split into ES modules; index.html was updated at the
time and this page was not. A module parsed as a classic script throws "Cannot
use import statement outside a module" before a line runs, so no click handler
was ever attached: the pills rendered from static HTML and did nothing. Only the
first calculator appeared to pass, because it carries `active` in the markup and
needs no click. That was 52 failures.

Settings and FAQ. Both moved from the tab rail into the account-card menu; the
helper still clicked button.tab-btn[data-tab=…] and timed out. Ten more.

The AI mocks, which had stopped intercepting for two independent reasons and so
were calling the real model on every run — spending credits and comparing
genuine output against strings like "MOCK HPI from dictation". A '**/api/x' glob
matches no URL on Playwright 1.50, and page.route fails silently when nothing
matches; measured against a real URL, that glob and '*/**/api/x' both matched
zero times where a regex matched. Fixing that alone was not enough: the app
registers a service worker that answers every /api/ request with its own
fetch(), and a request made inside a service worker never reaches page.route.
Blocking registration in the config puts them back in the page. The mocked
dictation test now finishes in 1.6s rather than 7.5s, which is what a real model
call costs.

Whole suite: 204 passed / 96 failed in 15.8 minutes, now 289 passed / 11 failed
in 6.8. The remaining eleven are spread across nine specs with no shared cause
and are not touched here.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01Dv6sqaY6Vq3ChZHMem3cnU
This commit is contained in:
Daniel 2026-09-11 18:54:03 +02:00
parent 5577ec301c
commit 05dcd1146d
5 changed files with 90 additions and 4 deletions

View file

@ -85,6 +85,21 @@ async function loginAs(context, request, email = TEST_EMAIL) {
}]);
}
// A '**/api/x' glob stopped matching any URL when Playwright went to 1.50, and
// page.route fails silently: no error, no warning, the request simply goes to
// the server. So every "mocked" AI test was calling the real model and
// comparing its genuine output against a canned string — spending real credits
// on every run and failing for a reason that looked like a UI bug. Measured:
// against http://127.0.0.1:3553/api/health, '**/api/health' and '*/**/api/health'
// both matched zero times; a regex matched.
//
// The patterns are kept as strings because they are also the keys callers pass
// in `overrides`, and turned into anchored regexes here.
function asMatcher(pattern) {
const path = pattern.replace(/^\*\*/, '');
return new RegExp(path.replace(/[.*+?^${}()|[\]\\]/g, '\\$&') + '(?:[?#]|$)');
}
// ── AI mock — intercepts generation endpoints ──────────────
// Canned response shape matches what each route's frontend expects.
// Override per-test by passing {pattern: responseFn} in overrides.
@ -111,7 +126,7 @@ async function mockAI(page, overrides = {}) {
for (const { pattern, response } of routes) {
const override = overrides[pattern];
await page.route(pattern, async route => {
await page.route(asMatcher(pattern), async route => {
const resp = typeof override === 'function' ? await override(route.request()) : (override || response);
await route.fulfill({ status: 200, contentType: 'application/json', body: JSON.stringify(resp) });
});

View file

@ -28,6 +28,12 @@ module.exports = defineConfig({
reporter: [['list']],
use: {
baseURL: process.env.BASE_URL || 'http://127.0.0.1:3553',
// The app registers a service worker that answers every /api/ request with
// its own fetch(). A request made inside a service worker never reaches
// page.route, so mockAI could not intercept anything while one was running
// and the tests called the real model. Blocking registration puts the
// requests back in the page, where the mocks can see them.
serviceWorkers: 'block',
trace: 'retain-on-failure',
screenshot: 'only-on-failure',
actionTimeout: 5_000,

View file

@ -5,6 +5,12 @@
const { test, expect, E2E_BASE, mockAI } = require('../fixtures');
// Settings and FAQ are not on the tab rail. They live in the account-card menu
// alongside Admin, and this helper used to click button.tab-btn[data-tab=…] for
// them, which simply timed out — the cause of ten of these failures. Dictation
// really is a rail tab, so both routes are needed.
const ACCOUNT_MENU = ['settings', 'faq'];
async function openTab(page, name) {
await page.goto(E2E_BASE + '/');
await page.waitForSelector('button.tab-btn', { timeout: 15000 });
@ -12,7 +18,12 @@ async function openTab(page, name) {
if (vp && vp.width <= 768) {
await page.click('#btn-menu-toggle').catch(() => {});
}
await page.click(`button.tab-btn[data-tab="${name}"]`);
if (ACCOUNT_MENU.includes(name)) {
await page.locator('.account-card-btn').first().click();
await page.locator(`[data-account-tab="${name}"]`).first().click();
} else {
await page.click(`button.tab-btn[data-tab="${name}"]`);
}
await page.waitForFunction((t) => {
const el = document.getElementById(t + '-tab');
return el && el.classList.contains('active') && el.innerHTML.trim().length > 100;

View file

@ -20,9 +20,17 @@
calc-nav-pill (which no longer exists). -->
<section id="bedside-tab" class="tab-content active" data-component="bedside"></section>
<!-- calculators.js and drugs-loader.js are ES modules and must be loaded as
modules. Loaded with `defer` instead, the browser parses them as classic
scripts, throws "Cannot use import statement outside a module" before a
single line runs, and no click handler is ever attached: the pills render
from static HTML and do nothing. index.html has always had this right;
this page was left behind when the calculators were split into modules,
which is what 52 of the e2e failures were. Keep these in step with
index.html. -->
<script defer src="/js/calc-math.js"></script>
<script defer src="/js/drugs-loader.js"></script>
<script defer src="/js/calculators.js"></script>
<script type="module" src="/js/drugs-loader.js"></script>
<script type="module" src="/js/calculators.js"></script>
<script type="module" src="/js/bedside/index.js"></script>
<script defer src="/js/e2e-bootstrap.js"></script>

View file

@ -58,3 +58,49 @@ test('the e2e accounts are seeded, and the seed cannot touch a real one', () =>
// And the runner seeds before it tests, so nobody has to remember to.
assert.match(read('scripts/e2e.sh'), /node e2e\/seed\.js/);
});
test('the harness loads the calculators the way the app does', () => {
// calculators.js and drugs-loader.js are ES modules. Loaded with `defer` they
// are parsed as classic scripts and throw "Cannot use import statement outside
// a module" before a line runs, so no click handler is ever attached: the
// pills render from static HTML and do nothing. Measured in the browser, and
// it was 52 of the 96 e2e failures. Only the first calculator "passed",
// because it carries `active` in the markup and needs no click.
const harness = read('public/e2e-harness.html');
const app = read('public/index.html');
for (const file of ['calculators.js', 'drugs-loader.js']) {
const inApp = new RegExp('<script type="module" src="/js/' + file.replace('.', '\\.') + '"');
assert.match(app, inApp, file + ' is a module in the app');
assert.match(harness, inApp, 'so the harness must load it the same way');
}
assert.doesNotMatch(harness, /<script defer src="\/js\/(calculators|drugs-loader)\.js"/);
});
test('the AI mocks actually intercept, rather than calling the real model', () => {
const fixtures = read('e2e/fixtures.js');
// Two separate reasons the mocks were silently doing nothing, both measured.
//
// One: a '**/api/x' glob matches no URL on Playwright 1.50, and page.route
// fails silently when nothing matches.
assert.match(fixtures, /function asMatcher\(pattern\)/);
assert.match(fixtures, /await page\.route\(asMatcher\(pattern\)/);
// Two: the app registers a service worker that answers every /api/ request
// with its own fetch(), and a request made inside a service worker never
// reaches page.route. Blocking registration puts them back in the page.
assert.match(read('public/sw.js'), /url\.pathname\.startsWith\('\/api\/'\)/,
'if the worker stops handling /api this guard can be revisited');
assert.match(read('e2e/playwright.config.js'), /serviceWorkers: 'block'/);
});
test('Settings and FAQ are opened the way the app actually offers them', () => {
// Neither is on the tab rail; both live in the account-card menu with Admin.
const index = read('public/index.html');
for (const name of ['settings', 'faq']) {
assert.doesNotMatch(index, new RegExp('data-tab="' + name + '"'), name + ' is not a rail tab');
assert.match(index, new RegExp('data-account-tab="' + name + '"'));
}
const spec = read('e2e/tests/settings-faq-dictation.spec.js');
assert.match(spec, /const ACCOUNT_MENU = \['settings', 'faq'\];/);
assert.match(spec, /\[data-account-tab="\$\{name\}"\]/);
});