test: lock in the CI quality baseline stage order with the ci-stages suite (E00-S05-T01)

tests/ci-stages.test.mjs asserts that .gitea/workflows/ci.yml declares the seven required PR stages (frozen-install, typecheck, formatting-lint, unit, architecture, postgres-integration, build-apps) in order, that every later stage gates on its predecessor through needs, that each stage runs its expected command (frozen install, pnpm typecheck, pnpm lint, the unit / architecture / postgres-integration node --test runs, the apps-group build with the server artifact check), and that every committed test suite is wired into exactly one stage. Mutation probes prove the assertions are non-vacuous.
2026-08-30 06:12:25 +00:00
parent e6d28f94f0
commit d9b493498e
1 changed files with 272 additions and 0 deletions
@@ -0,0 +1,272 @@
+/**
+ * CI quality baseline test — locks in the [E00-S05-T01] required PR stages of
+ * the committed workflow (`.gitea/workflows/ci.yml`).
+ *
+ * The baseline (issue #187 acceptance criteria):
+ *   - "CI runs frozen install before later stages" → `frozen-install` is the
+ *     first job and every later stage declares `needs` on its predecessor, so
+ *     the pipeline runs the required stages strictly in order and nothing
+ *     proceeds past a failed stage.
+ *   - "CI runs typecheck, formatting/lint, unit, architecture and PostgreSQL
+ *     integration tests" → the five stage jobs exist with the expected
+ *     commands: `pnpm typecheck`, `pnpm lint`, and the unit / architecture /
+ *     postgres-integration `node --test` suite runs.
+ *   - "CI builds the admin and server applications" → the `build-apps` stage
+ *     compiles the whole apps group (`./apps/**` — apps/server today, the
+ *     admin app when E06-S01 lands) and verifies the compiled server
+ *     artifact.
+ *   - every committed test suite under tests/ is wired into exactly one stage
+ *     (`pnpm lint` runs the formatting-policy suite; every other suite is
+ *     named by a `node --test` run in the unit, architecture or
+ *     postgres-integration stage).
+ *
+ * Mutation probes prove the assertions are non-vacuous: removing a stage,
+ * breaking the `needs` chain, or dropping a suite from its stage all fail.
+ *
+ * Run: `node --test tests/ci-stages.test.mjs`
+ * (node:test — built into Node >= 18; no dependencies, lockfile untouched.)
+ */
+
+import test from 'node:test';
+import assert from 'node:assert/strict';
+import { readFileSync, readdirSync } from 'node:fs';
+import path from 'node:path';
+import { fileURLToPath } from 'node:url';
+
+const REPO_ROOT = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..');
+
+const read = (relPath) => readFileSync(path.join(REPO_ROOT, relPath), 'utf8');
+
+/** The workflow file under test. */
+const WORKFLOW = '.gitea/workflows/ci.yml';
+
+/** The required PR stages, in the order the pipeline must run them. */
+const REQUIRED_STAGES = [
+  'frozen-install',
+  'typecheck',
+  'formatting-lint',
+  'unit',
+  'architecture',
+  'postgres-integration',
+  'build-apps',
+];
+
+/** The root lint command the formatting-lint stage must run. */
+const LINT_SCRIPT = 'node --test tests/formatting-policy.test.mjs';
+
+/**
+ * Parses the workflow's `jobs:` section (the committed file is 2-space
+ * indented YAML) into `{ order, jobs }` where `order` lists job keys in
+ * document order and each job carries its `needs` value and `run:` commands.
+ * Comments and blank lines are skipped; unknown keys under a job are ignored.
+ */
+function parseWorkflowJobs(yamlText) {
+  const lines = yamlText.split('\n');
+  const jobsIndex = lines.findIndex((line) => line === 'jobs:');
+  assert.ok(jobsIndex >= 0, 'the workflow must declare a top-level jobs: section');
+
+  const order = [];
+  const jobs = {};
+  let current = null;
+
+  for (let i = jobsIndex + 1; i < lines.length; i++) {
+    const line = lines[i];
+    const trimmed = line.trim();
+    if (trimmed === '' || trimmed.startsWith('#')) continue;
+    const indent = line.length - line.trimStart().length;
+    if (indent === 2) {
+      const key = /^([A-Za-z0-9_-]+):/.exec(trimmed);
+      assert.ok(key, `unexpected jobs: entry at indent 2: "${line}"`);
+      current = key[1];
+      order.push(current);
+      jobs[current] = { needs: null, runs: [] };
+      continue;
+    }
+    if (current && indent > 2) {
+      const needs = /^needs:\s*(.+)$/.exec(trimmed);
+      if (needs) jobs[current].needs = needs[1].trim().replace(/^\[|\]$/g, '');
+      const run = /^run:\s*(.+)$/.exec(trimmed);
+      if (run) jobs[current].runs.push(run[1].trim());
+    }
+  }
+
+  return { order, jobs };
+}
+
+/** Extracts the tests/*.test.mjs suite names named by `node --test` runs. */
+function suitesNamedInRuns(runs) {
+  const out = [];
+  for (const run of runs) {
+    for (const token of run.split(/\s+/)) {
+      if (token.startsWith('tests/') && token.endsWith('.test.mjs')) {
+        out.push(token.slice('tests/'.length));
+      }
+    }
+  }
+  return out;
+}
+
+/**
+ * Asserts the whole E00-S05-T01 baseline for a parsed workflow. Throws an
+ * AssertionError describing the first violated invariant.
+ */
+function assertBaseline({ order, jobs }) {
+  // Every required stage exists.
+  for (const stage of REQUIRED_STAGES) {
+    assert.ok(jobs[stage], `required CI stage "${stage}" is missing from the workflow`);
+  }
+
+  // The required stages run in order (their relative order is preserved).
+  const present = order.filter((name) => REQUIRED_STAGES.includes(name));
+  assert.deepEqual(
+    present,
+    REQUIRED_STAGES,
+    `the required CI stages must run in order: ${REQUIRED_STAGES.join(' -> ')}`,
+  );
+
+  // Frozen install runs before later stages: every later stage gates on its
+  // predecessor, so the pipeline is strictly ordered.
+  for (let i = 1; i < REQUIRED_STAGES.length; i++) {
+    assert.equal(
+      jobs[REQUIRED_STAGES[i]].needs,
+      REQUIRED_STAGES[i - 1],
+      `stage "${REQUIRED_STAGES[i]}" must gate on the previous stage "${REQUIRED_STAGES[i - 1]}"`,
+    );
+  }
+
+  // Stage commands.
+  assert.ok(
+    jobs['frozen-install'].runs.some((run) => run.includes('pnpm install --frozen-lockfile')),
+    'frozen-install must run the frozen lockfile install',
+  );
+  assert.ok(
+    jobs['typecheck'].runs.some((run) => run.includes('pnpm typecheck')),
+    'the typecheck stage must run pnpm typecheck',
+  );
+  assert.ok(
+    jobs['formatting-lint'].runs.some((run) => run.includes('pnpm lint')),
+    'the formatting-lint stage must run pnpm lint',
+  );
+
+  // The build stage compiles the apps group and verifies the server artifact.
+  const buildRuns = jobs['build-apps'].runs;
+  assert.ok(
+    buildRuns.some((run) => run.includes('run build') && run.includes('./apps/**')),
+    'build-apps must build the apps group (pnpm --filter "./apps/**" run build)',
+  );
+  assert.ok(
+    buildRuns.some((run) => run.includes('apps/server/dist/index.js')),
+    'build-apps must verify the compiled server application artifact',
+  );
+
+  // Every committed test suite is wired into exactly one stage.
+  const staged = [
+    ...suitesNamedInRuns(jobs['unit'].runs),
+    ...suitesNamedInRuns(jobs['architecture'].runs),
+    ...suitesNamedInRuns(jobs['postgres-integration'].runs),
+  ];
+  const allSuites = readdirSync(path.join(REPO_ROOT, 'tests'))
+    .filter((file) => file.endsWith('.test.mjs'))
+    .sort();
+  // The formatting-policy suite is run by the formatting-lint stage via the
+  // root lint script rather than by a node --test run in a test stage.
+  const expected = allSuites.filter((file) => file !== 'formatting-policy.test.mjs');
+  assert.equal(
+    staged.length,
+    expected.length,
+    'each test suite must be wired into exactly one CI stage run ' +
+      `(staged: ${staged.join(', ')}; expected: ${expected.join(', ')})`,
+  );
+  assert.deepEqual(
+    [...new Set(staged)].sort(),
+    expected,
+    'every committed test suite must be wired into exactly one CI stage ' +
+      `(staged: ${staged.join(', ')}; expected: ${expected.join(', ')})`,
+  );
+}
+
+/** Removes the whole `  <jobName>:` block from a workflow text (probe helper). */
+function removeJobBlock(yamlText, jobName) {
+  const lines = yamlText.split('\n');
+  const start = lines.findIndex((line) => line === `  ${jobName}:`);
+  assert.ok(start >= 0, `job ${jobName} must exist in the workflow text`);
+  let end = lines.length;
+  for (let i = start + 1; i < lines.length; i++) {
+    const trimmed = lines[i].trim();
+    if (
+      trimmed !== '' &&
+      !trimmed.startsWith('#') &&
+      lines[i].length - lines[i].trimStart().length === 2 &&
+      /^[A-Za-z0-9_-]+:/.test(trimmed)
+    ) {
+      end = i;
+      break;
+    }
+  }
+  return [...lines.slice(0, start), ...lines.slice(end)].join('\n');
+}
+
+// ---------------------------------------------------------------------------
+// Real-workflow baseline
+// ---------------------------------------------------------------------------
+
+test('the committed CI workflow runs the required PR stages in order', () => {
+  assertBaseline(parseWorkflowJobs(read(WORKFLOW)));
+});
+
+test('the workflow triggers on pull requests and pushes to main', () => {
+  const text = read(WORKFLOW);
+  assert.match(text, /^on:$/m, 'the workflow must declare an on: trigger block');
+  assert.match(text, /pull_request:/, 'PRs must trigger the CI workflow');
+  assert.match(text, /push:/, 'pushes must trigger the CI workflow');
+  assert.match(text, /branches:\s*\[main\]/, 'the push trigger must cover main');
+});
+
+test('the root lint script runs the formatting-policy suite', () => {
+  const scripts = JSON.parse(read('package.json')).scripts ?? {};
+  assert.equal(
+    scripts.lint,
+    LINT_SCRIPT,
+    `root package.json must declare scripts.lint exactly as "${LINT_SCRIPT}"`,
+  );
+});
+
+// ---------------------------------------------------------------------------
+// Mutation probes — the baseline assertions are non-vacuous
+// ---------------------------------------------------------------------------
+
+test('removing a required stage fails the baseline (mutation probe)', () => {
+  const mutated = removeJobBlock(read(WORKFLOW), 'unit');
+  assert.throws(() => assertBaseline(parseWorkflowJobs(mutated)), /required CI stage "unit"/);
+});
+
+test('breaking the needs chain fails the baseline (mutation probe)', () => {
+  const text = read(WORKFLOW);
+  const mutated = text.replace('needs: formatting-lint', 'needs: typecheck');
+  assert.notEqual(mutated, text, 'the probe must mutate the workflow');
+  assert.throws(
+    () => assertBaseline(parseWorkflowJobs(mutated)),
+    /must gate on the previous stage/,
+  );
+});
+
+test('dropping a suite from its stage fails the coverage assertion (mutation probe)', () => {
+  const text = read(WORKFLOW);
+  const mutated = text.replace('tests/config-schema.test.mjs', '');
+  assert.notEqual(mutated, text, 'the probe must mutate the workflow');
+  assert.throws(
+    () => assertBaseline(parseWorkflowJobs(mutated)),
+    /must be wired into exactly one CI stage/,
+  );
+});
+
+test('reordering the stages fails the baseline (mutation probe)', () => {
+  const text = read(WORKFLOW);
+  // Swap the typecheck and formatting-lint job blocks so their document order
+  // no longer matches the required stage order.
+  const typecheckBlock = text.slice(text.indexOf('  typecheck:'), text.indexOf('  formatting-lint:'));
+  const lintBlock = text.slice(text.indexOf('  formatting-lint:'), text.indexOf('  unit:'));
+  const mutated = text.replace(typecheckBlock + lintBlock, lintBlock + typecheckBlock);
+  assert.notEqual(mutated, text, 'the probe must mutate the workflow');
+  assert.throws(() => assertBaseline(parseWorkflowJobs(mutated)), /must run in order/);
+});