From d9b493498edb8228a5fc4a399b28eec181069187 Mon Sep 17 00:00:00 2001 From: implementer Date: Sun, 30 Aug 2026 06:12:25 +0000 Subject: [PATCH] test: lock in the CI quality baseline stage order with the ci-stages suite (E00-S05-T01) tests/ci-stages.test.mjs asserts that .gitea/workflows/ci.yml declares the seven required PR stages (frozen-install, typecheck, formatting-lint, unit, architecture, postgres-integration, build-apps) in order, that every later stage gates on its predecessor through needs, that each stage runs its expected command (frozen install, pnpm typecheck, pnpm lint, the unit / architecture / postgres-integration node --test runs, the apps-group build with the server artifact check), and that every committed test suite is wired into exactly one stage. Mutation probes prove the assertions are non-vacuous. --- tests/ci-stages.test.mjs | 272 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 272 insertions(+) create mode 100644 tests/ci-stages.test.mjs diff --git a/tests/ci-stages.test.mjs b/tests/ci-stages.test.mjs new file mode 100644 index 0000000..4794284 --- /dev/null +++ b/tests/ci-stages.test.mjs @@ -0,0 +1,272 @@ +/** + * CI quality baseline test — locks in the [E00-S05-T01] required PR stages of + * the committed workflow (`.gitea/workflows/ci.yml`). + * + * The baseline (issue #187 acceptance criteria): + * - "CI runs frozen install before later stages" → `frozen-install` is the + * first job and every later stage declares `needs` on its predecessor, so + * the pipeline runs the required stages strictly in order and nothing + * proceeds past a failed stage. + * - "CI runs typecheck, formatting/lint, unit, architecture and PostgreSQL + * integration tests" → the five stage jobs exist with the expected + * commands: `pnpm typecheck`, `pnpm lint`, and the unit / architecture / + * postgres-integration `node --test` suite runs. + * - "CI builds the admin and server applications" → the `build-apps` stage + * compiles the whole apps group (`./apps/**` — apps/server today, the + * admin app when E06-S01 lands) and verifies the compiled server + * artifact. + * - every committed test suite under tests/ is wired into exactly one stage + * (`pnpm lint` runs the formatting-policy suite; every other suite is + * named by a `node --test` run in the unit, architecture or + * postgres-integration stage). + * + * Mutation probes prove the assertions are non-vacuous: removing a stage, + * breaking the `needs` chain, or dropping a suite from its stage all fail. + * + * Run: `node --test tests/ci-stages.test.mjs` + * (node:test — built into Node >= 18; no dependencies, lockfile untouched.) + */ + +import test from 'node:test'; +import assert from 'node:assert/strict'; +import { readFileSync, readdirSync } from 'node:fs'; +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; + +const REPO_ROOT = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..'); + +const read = (relPath) => readFileSync(path.join(REPO_ROOT, relPath), 'utf8'); + +/** The workflow file under test. */ +const WORKFLOW = '.gitea/workflows/ci.yml'; + +/** The required PR stages, in the order the pipeline must run them. */ +const REQUIRED_STAGES = [ + 'frozen-install', + 'typecheck', + 'formatting-lint', + 'unit', + 'architecture', + 'postgres-integration', + 'build-apps', +]; + +/** The root lint command the formatting-lint stage must run. */ +const LINT_SCRIPT = 'node --test tests/formatting-policy.test.mjs'; + +/** + * Parses the workflow's `jobs:` section (the committed file is 2-space + * indented YAML) into `{ order, jobs }` where `order` lists job keys in + * document order and each job carries its `needs` value and `run:` commands. + * Comments and blank lines are skipped; unknown keys under a job are ignored. + */ +function parseWorkflowJobs(yamlText) { + const lines = yamlText.split('\n'); + const jobsIndex = lines.findIndex((line) => line === 'jobs:'); + assert.ok(jobsIndex >= 0, 'the workflow must declare a top-level jobs: section'); + + const order = []; + const jobs = {}; + let current = null; + + for (let i = jobsIndex + 1; i < lines.length; i++) { + const line = lines[i]; + const trimmed = line.trim(); + if (trimmed === '' || trimmed.startsWith('#')) continue; + const indent = line.length - line.trimStart().length; + if (indent === 2) { + const key = /^([A-Za-z0-9_-]+):/.exec(trimmed); + assert.ok(key, `unexpected jobs: entry at indent 2: "${line}"`); + current = key[1]; + order.push(current); + jobs[current] = { needs: null, runs: [] }; + continue; + } + if (current && indent > 2) { + const needs = /^needs:\s*(.+)$/.exec(trimmed); + if (needs) jobs[current].needs = needs[1].trim().replace(/^\[|\]$/g, ''); + const run = /^run:\s*(.+)$/.exec(trimmed); + if (run) jobs[current].runs.push(run[1].trim()); + } + } + + return { order, jobs }; +} + +/** Extracts the tests/*.test.mjs suite names named by `node --test` runs. */ +function suitesNamedInRuns(runs) { + const out = []; + for (const run of runs) { + for (const token of run.split(/\s+/)) { + if (token.startsWith('tests/') && token.endsWith('.test.mjs')) { + out.push(token.slice('tests/'.length)); + } + } + } + return out; +} + +/** + * Asserts the whole E00-S05-T01 baseline for a parsed workflow. Throws an + * AssertionError describing the first violated invariant. + */ +function assertBaseline({ order, jobs }) { + // Every required stage exists. + for (const stage of REQUIRED_STAGES) { + assert.ok(jobs[stage], `required CI stage "${stage}" is missing from the workflow`); + } + + // The required stages run in order (their relative order is preserved). + const present = order.filter((name) => REQUIRED_STAGES.includes(name)); + assert.deepEqual( + present, + REQUIRED_STAGES, + `the required CI stages must run in order: ${REQUIRED_STAGES.join(' -> ')}`, + ); + + // Frozen install runs before later stages: every later stage gates on its + // predecessor, so the pipeline is strictly ordered. + for (let i = 1; i < REQUIRED_STAGES.length; i++) { + assert.equal( + jobs[REQUIRED_STAGES[i]].needs, + REQUIRED_STAGES[i - 1], + `stage "${REQUIRED_STAGES[i]}" must gate on the previous stage "${REQUIRED_STAGES[i - 1]}"`, + ); + } + + // Stage commands. + assert.ok( + jobs['frozen-install'].runs.some((run) => run.includes('pnpm install --frozen-lockfile')), + 'frozen-install must run the frozen lockfile install', + ); + assert.ok( + jobs['typecheck'].runs.some((run) => run.includes('pnpm typecheck')), + 'the typecheck stage must run pnpm typecheck', + ); + assert.ok( + jobs['formatting-lint'].runs.some((run) => run.includes('pnpm lint')), + 'the formatting-lint stage must run pnpm lint', + ); + + // The build stage compiles the apps group and verifies the server artifact. + const buildRuns = jobs['build-apps'].runs; + assert.ok( + buildRuns.some((run) => run.includes('run build') && run.includes('./apps/**')), + 'build-apps must build the apps group (pnpm --filter "./apps/**" run build)', + ); + assert.ok( + buildRuns.some((run) => run.includes('apps/server/dist/index.js')), + 'build-apps must verify the compiled server application artifact', + ); + + // Every committed test suite is wired into exactly one stage. + const staged = [ + ...suitesNamedInRuns(jobs['unit'].runs), + ...suitesNamedInRuns(jobs['architecture'].runs), + ...suitesNamedInRuns(jobs['postgres-integration'].runs), + ]; + const allSuites = readdirSync(path.join(REPO_ROOT, 'tests')) + .filter((file) => file.endsWith('.test.mjs')) + .sort(); + // The formatting-policy suite is run by the formatting-lint stage via the + // root lint script rather than by a node --test run in a test stage. + const expected = allSuites.filter((file) => file !== 'formatting-policy.test.mjs'); + assert.equal( + staged.length, + expected.length, + 'each test suite must be wired into exactly one CI stage run ' + + `(staged: ${staged.join(', ')}; expected: ${expected.join(', ')})`, + ); + assert.deepEqual( + [...new Set(staged)].sort(), + expected, + 'every committed test suite must be wired into exactly one CI stage ' + + `(staged: ${staged.join(', ')}; expected: ${expected.join(', ')})`, + ); +} + +/** Removes the whole ` :` block from a workflow text (probe helper). */ +function removeJobBlock(yamlText, jobName) { + const lines = yamlText.split('\n'); + const start = lines.findIndex((line) => line === ` ${jobName}:`); + assert.ok(start >= 0, `job ${jobName} must exist in the workflow text`); + let end = lines.length; + for (let i = start + 1; i < lines.length; i++) { + const trimmed = lines[i].trim(); + if ( + trimmed !== '' && + !trimmed.startsWith('#') && + lines[i].length - lines[i].trimStart().length === 2 && + /^[A-Za-z0-9_-]+:/.test(trimmed) + ) { + end = i; + break; + } + } + return [...lines.slice(0, start), ...lines.slice(end)].join('\n'); +} + +// --------------------------------------------------------------------------- +// Real-workflow baseline +// --------------------------------------------------------------------------- + +test('the committed CI workflow runs the required PR stages in order', () => { + assertBaseline(parseWorkflowJobs(read(WORKFLOW))); +}); + +test('the workflow triggers on pull requests and pushes to main', () => { + const text = read(WORKFLOW); + assert.match(text, /^on:$/m, 'the workflow must declare an on: trigger block'); + assert.match(text, /pull_request:/, 'PRs must trigger the CI workflow'); + assert.match(text, /push:/, 'pushes must trigger the CI workflow'); + assert.match(text, /branches:\s*\[main\]/, 'the push trigger must cover main'); +}); + +test('the root lint script runs the formatting-policy suite', () => { + const scripts = JSON.parse(read('package.json')).scripts ?? {}; + assert.equal( + scripts.lint, + LINT_SCRIPT, + `root package.json must declare scripts.lint exactly as "${LINT_SCRIPT}"`, + ); +}); + +// --------------------------------------------------------------------------- +// Mutation probes — the baseline assertions are non-vacuous +// --------------------------------------------------------------------------- + +test('removing a required stage fails the baseline (mutation probe)', () => { + const mutated = removeJobBlock(read(WORKFLOW), 'unit'); + assert.throws(() => assertBaseline(parseWorkflowJobs(mutated)), /required CI stage "unit"/); +}); + +test('breaking the needs chain fails the baseline (mutation probe)', () => { + const text = read(WORKFLOW); + const mutated = text.replace('needs: formatting-lint', 'needs: typecheck'); + assert.notEqual(mutated, text, 'the probe must mutate the workflow'); + assert.throws( + () => assertBaseline(parseWorkflowJobs(mutated)), + /must gate on the previous stage/, + ); +}); + +test('dropping a suite from its stage fails the coverage assertion (mutation probe)', () => { + const text = read(WORKFLOW); + const mutated = text.replace('tests/config-schema.test.mjs', ''); + assert.notEqual(mutated, text, 'the probe must mutate the workflow'); + assert.throws( + () => assertBaseline(parseWorkflowJobs(mutated)), + /must be wired into exactly one CI stage/, + ); +}); + +test('reordering the stages fails the baseline (mutation probe)', () => { + const text = read(WORKFLOW); + // Swap the typecheck and formatting-lint job blocks so their document order + // no longer matches the required stage order. + const typecheckBlock = text.slice(text.indexOf(' typecheck:'), text.indexOf(' formatting-lint:')); + const lintBlock = text.slice(text.indexOf(' formatting-lint:'), text.indexOf(' unit:')); + const mutated = text.replace(typecheckBlock + lintBlock, lintBlock + typecheckBlock); + assert.notEqual(mutated, text, 'the probe must mutate the workflow'); + assert.throws(() => assertBaseline(parseWorkflowJobs(mutated)), /must run in order/); +});