/** * CI quality baseline test — locks in the [E00-S05-T01] required PR stages of * the committed workflow (`.gitea/workflows/ci.yml`). * * The baseline (issue #187 acceptance criteria): * - "CI runs frozen install before later stages" → `frozen-install` is the * first job and every later stage declares `needs` on its predecessor, so * the pipeline runs the required stages strictly in order and nothing * proceeds past a failed stage. * - "CI runs typecheck, formatting/lint, unit, architecture and PostgreSQL * integration tests" → the five stage jobs exist with the expected * commands: `pnpm typecheck`, `pnpm lint`, and the unit / architecture / * postgres-integration `node --test` suite runs. * - "CI builds the admin and server applications" → the `build-apps` stage * compiles the whole apps group (`./apps/**` — apps/server today, the * admin app when E06-S01 lands) and verifies the compiled server * artifact. * - "CI workflow pins third-party actions (actions/checkout, * actions/setup-node) to full commit SHAs, not floating tags" → every * `uses:` ref is a 40-char commit SHA pinned to the committed * PINNED_ACTIONS values — no `@v4`-style floating tags. * - "CI workflow declares minimal permissions (`permissions: contents: * read`) at the workflow level" → the workflow declares a top-level * `permissions:` block granting exactly `contents: read`. * - every committed test suite under tests/ is wired into exactly one stage * (`pnpm lint` runs the formatting-policy suite; every other suite is * named by a `node --test` run in the unit, architecture or * postgres-integration stage). * * Mutation probes prove the assertions are non-vacuous: removing a stage, * breaking the `needs` chain, or dropping a suite from its stage all fail. * * Run: `node --test tests/ci-stages.test.mjs` * (node:test — built into Node >= 18; no dependencies, lockfile untouched.) */ import test from 'node:test'; import assert from 'node:assert/strict'; import { readFileSync, readdirSync } from 'node:fs'; import path from 'node:path'; import { fileURLToPath } from 'node:url'; const REPO_ROOT = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..'); const read = (relPath) => readFileSync(path.join(REPO_ROOT, relPath), 'utf8'); /** The workflow file under test. */ const WORKFLOW = '.gitea/workflows/ci.yml'; /** The required PR stages, in the order the pipeline must run them. */ const REQUIRED_STAGES = [ 'frozen-install', 'typecheck', 'formatting-lint', 'unit', 'architecture', 'postgres-integration', 'build-apps', ]; /** The root lint command the formatting-lint stage must run. */ const LINT_SCRIPT = 'node --test tests/formatting-policy.test.mjs'; /** * The third-party actions the workflow may use, pinned to the full commit * SHA of a released version (E00-S05-T01 hardening). Updating a pin means * updating this table and the workflow together, in the same PR. */ const PINNED_ACTIONS = { 'actions/checkout': '11bd71901bbe5b1630ceea73d27597364c9af683', // v4.2.2 'actions/setup-node': '1d0ff469b7ec7b3cb9d8673fde0c81c44821de2a', // v4.2.0 }; /** * Parses the workflow's `jobs:` section (the committed file is 2-space * indented YAML) into `{ order, jobs }` where `order` lists job keys in * document order and each job carries its `needs` value and `run:` commands. * Comments and blank lines are skipped; unknown keys under a job are ignored. */ function parseWorkflowJobs(yamlText) { const lines = yamlText.split('\n'); const jobsIndex = lines.findIndex((line) => line === 'jobs:'); assert.ok(jobsIndex >= 0, 'the workflow must declare a top-level jobs: section'); const order = []; const jobs = {}; let current = null; for (let i = jobsIndex + 1; i < lines.length; i++) { const line = lines[i]; const trimmed = line.trim(); if (trimmed === '' || trimmed.startsWith('#')) continue; const indent = line.length - line.trimStart().length; if (indent === 2) { const key = /^([A-Za-z0-9_-]+):/.exec(trimmed); assert.ok(key, `unexpected jobs: entry at indent 2: "${line}"`); current = key[1]; order.push(current); jobs[current] = { needs: null, runs: [] }; continue; } if (current && indent > 2) { const needs = /^needs:\s*(.+)$/.exec(trimmed); if (needs) jobs[current].needs = needs[1].trim().replace(/^\[|\]$/g, ''); const run = /^run:\s*(.+)$/.exec(trimmed); if (run) jobs[current].runs.push(run[1].trim()); } } return { order, jobs }; } /** Extracts the tests/*.test.mjs suite names named by `node --test` runs. */ function suitesNamedInRuns(runs) { const out = []; for (const run of runs) { for (const token of run.split(/\s+/)) { if (token.startsWith('tests/') && token.endsWith('.test.mjs')) { out.push(token.slice('tests/'.length)); } } } return out; } /** Extracts the `uses:` refs from the workflow, e.g. "actions/checkout@". */ function usesRefs(yamlText) { const refs = []; for (const line of yamlText.split('\n')) { // `uses:` appears either as a bare key or as a sequence item ("- uses:"). const match = /^\s*(?:-\s+)?uses:\s*(\S+)/.exec(line); if (match) refs.push(match[1]); } return refs; } /** * Asserts the E00-S05-T01 hardening criteria: every third-party `uses:` ref * is pinned to a full 40-char commit SHA (exactly the committed PINNED_ACTIONS * values — no floating tags) and the workflow declares `permissions: * contents: read` at the top level. Throws an AssertionError describing the * first violated invariant. */ function assertHardening(yamlText) { const refs = usesRefs(yamlText); assert.ok(refs.length > 0, 'the workflow must use at least one third-party action'); for (const ref of refs) { const match = /^([\w.-]+\/[\w.-]+)@([0-9a-f]{40})$/.exec(ref); assert.ok( match, `every third-party action must be pinned to a full 40-char commit SHA, not a floating tag (got "${ref}")`, ); assert.ok( Object.hasOwn(PINNED_ACTIONS, match[1]), `unexpected third-party action "${match[1]}" — add it to the PINNED_ACTIONS policy table if it is approved`, ); assert.equal( match[2], PINNED_ACTIONS[match[1]], `"${match[1]}" must be pinned to the committed full commit SHA ${PINNED_ACTIONS[match[1]]} (got ${match[2]})`, ); } for (const action of Object.keys(PINNED_ACTIONS)) { assert.ok( refs.some((ref) => ref.startsWith(`${action}@`)), `the workflow must use "${action}" pinned to a full commit SHA`, ); } assert.match( yamlText, /^permissions:\n[ \t]+contents: read$/m, 'the workflow must declare a top-level "permissions: contents: read" block', ); } /** * Asserts the whole E00-S05-T01 baseline for a parsed workflow. Throws an * AssertionError describing the first violated invariant. */ function assertBaseline({ order, jobs }) { // Every required stage exists. for (const stage of REQUIRED_STAGES) { assert.ok(jobs[stage], `required CI stage "${stage}" is missing from the workflow`); } // The required stages run in order (their relative order is preserved). const present = order.filter((name) => REQUIRED_STAGES.includes(name)); assert.deepEqual( present, REQUIRED_STAGES, `the required CI stages must run in order: ${REQUIRED_STAGES.join(' -> ')}`, ); // Frozen install runs before later stages: every later stage gates on its // predecessor, so the pipeline is strictly ordered. for (let i = 1; i < REQUIRED_STAGES.length; i++) { assert.equal( jobs[REQUIRED_STAGES[i]].needs, REQUIRED_STAGES[i - 1], `stage "${REQUIRED_STAGES[i]}" must gate on the previous stage "${REQUIRED_STAGES[i - 1]}"`, ); } // Stage commands. assert.ok( jobs['frozen-install'].runs.some((run) => run.includes('pnpm install --frozen-lockfile')), 'frozen-install must run the frozen lockfile install', ); assert.ok( jobs['typecheck'].runs.some((run) => run.includes('pnpm typecheck')), 'the typecheck stage must run pnpm typecheck', ); assert.ok( jobs['formatting-lint'].runs.some((run) => run.includes('pnpm lint')), 'the formatting-lint stage must run pnpm lint', ); // The build stage compiles the apps group and verifies the server artifact. const buildRuns = jobs['build-apps'].runs; assert.ok( buildRuns.some((run) => run.includes('run build') && run.includes('./apps/**')), 'build-apps must build the apps group (pnpm --filter "./apps/**" run build)', ); assert.ok( buildRuns.some((run) => run.includes('apps/server/dist/index.js')), 'build-apps must verify the compiled server application artifact', ); // Every committed test suite is wired into exactly one stage. const staged = [ ...suitesNamedInRuns(jobs['unit'].runs), ...suitesNamedInRuns(jobs['architecture'].runs), ...suitesNamedInRuns(jobs['postgres-integration'].runs), ]; const allSuites = readdirSync(path.join(REPO_ROOT, 'tests')) .filter((file) => file.endsWith('.test.mjs')) .sort(); // The formatting-policy suite is run by the formatting-lint stage via the // root lint script rather than by a node --test run in a test stage. const expected = allSuites.filter((file) => file !== 'formatting-policy.test.mjs'); assert.equal( staged.length, expected.length, 'each test suite must be wired into exactly one CI stage run ' + `(staged: ${staged.join(', ')}; expected: ${expected.join(', ')})`, ); assert.deepEqual( [...new Set(staged)].sort(), expected, 'every committed test suite must be wired into exactly one CI stage ' + `(staged: ${staged.join(', ')}; expected: ${expected.join(', ')})`, ); } /** Removes the whole ` :` block from a workflow text (probe helper). */ function removeJobBlock(yamlText, jobName) { const lines = yamlText.split('\n'); const start = lines.findIndex((line) => line === ` ${jobName}:`); assert.ok(start >= 0, `job ${jobName} must exist in the workflow text`); let end = lines.length; for (let i = start + 1; i < lines.length; i++) { const trimmed = lines[i].trim(); if ( trimmed !== '' && !trimmed.startsWith('#') && lines[i].length - lines[i].trimStart().length === 2 && /^[A-Za-z0-9_-]+:/.test(trimmed) ) { end = i; break; } } return [...lines.slice(0, start), ...lines.slice(end)].join('\n'); } // --------------------------------------------------------------------------- // Real-workflow baseline // --------------------------------------------------------------------------- test('the committed CI workflow runs the required PR stages in order', () => { assertBaseline(parseWorkflowJobs(read(WORKFLOW))); }); test('the workflow triggers on pull requests and pushes to main', () => { const text = read(WORKFLOW); assert.match(text, /^on:$/m, 'the workflow must declare an on: trigger block'); assert.match(text, /pull_request:/, 'PRs must trigger the CI workflow'); assert.match(text, /push:/, 'pushes must trigger the CI workflow'); assert.match(text, /branches:\s*\[main\]/, 'the push trigger must cover main'); }); test('the root lint script runs the formatting-policy suite', () => { const scripts = JSON.parse(read('package.json')).scripts ?? {}; assert.equal( scripts.lint, LINT_SCRIPT, `root package.json must declare scripts.lint exactly as "${LINT_SCRIPT}"`, ); }); test('the workflow pins third-party actions to full commit SHAs and declares minimal permissions', () => { assertHardening(read(WORKFLOW)); }); // --------------------------------------------------------------------------- // Mutation probes — the baseline assertions are non-vacuous // --------------------------------------------------------------------------- test('removing a required stage fails the baseline (mutation probe)', () => { const mutated = removeJobBlock(read(WORKFLOW), 'unit'); assert.throws(() => assertBaseline(parseWorkflowJobs(mutated)), /required CI stage "unit"/); }); test('breaking the needs chain fails the baseline (mutation probe)', () => { const text = read(WORKFLOW); const mutated = text.replace('needs: formatting-lint', 'needs: typecheck'); assert.notEqual(mutated, text, 'the probe must mutate the workflow'); assert.throws( () => assertBaseline(parseWorkflowJobs(mutated)), /must gate on the previous stage/, ); }); test('dropping a suite from its stage fails the coverage assertion (mutation probe)', () => { const text = read(WORKFLOW); const mutated = text.replace('tests/config-schema.test.mjs', ''); assert.notEqual(mutated, text, 'the probe must mutate the workflow'); assert.throws( () => assertBaseline(parseWorkflowJobs(mutated)), /must be wired into exactly one CI stage/, ); }); test('reordering the stages fails the baseline (mutation probe)', () => { const text = read(WORKFLOW); // Swap the typecheck and formatting-lint job blocks so their document order // no longer matches the required stage order. const typecheckBlock = text.slice(text.indexOf(' typecheck:'), text.indexOf(' formatting-lint:')); const lintBlock = text.slice(text.indexOf(' formatting-lint:'), text.indexOf(' unit:')); const mutated = text.replace(typecheckBlock + lintBlock, lintBlock + typecheckBlock); assert.notEqual(mutated, text, 'the probe must mutate the workflow'); assert.throws(() => assertBaseline(parseWorkflowJobs(mutated)), /must run in order/); }); test('reverting an action pin to a floating tag fails the hardening assertion (mutation probe)', () => { const text = read(WORKFLOW); const mutated = text.replace( `actions/checkout@${PINNED_ACTIONS['actions/checkout']}`, 'actions/checkout@v4', ); assert.notEqual(mutated, text, 'the probe must mutate the workflow'); assert.throws(() => assertHardening(mutated), /full 40-char commit SHA/); }); test('changing a pinned action SHA fails the hardening assertion (mutation probe)', () => { const text = read(WORKFLOW); const mutated = text.replace( `actions/setup-node@${PINNED_ACTIONS['actions/setup-node']}`, `actions/setup-node@${'a'.repeat(40)}`, ); assert.notEqual(mutated, text, 'the probe must mutate the workflow'); assert.throws(() => assertHardening(mutated), /must be pinned to the committed full commit SHA/); }); test('removing the workflow-level permissions block fails the hardening assertion (mutation probe)', () => { const text = read(WORKFLOW); const mutated = text.replace(/^permissions:\n contents: read\n\n/m, ''); assert.notEqual(mutated, text, 'the probe must mutate the workflow'); assert.throws(() => assertHardening(mutated), /permissions: contents: read/); });