757 lines
32 KiB
JavaScript
757 lines
32 KiB
JavaScript
/**
|
|
* App readiness test — locks in the [E00-S03-T06] guarantee that the app does
|
|
* not report ready before migrations complete.
|
|
*
|
|
* Acceptance criteria covered (each test fails without the committed state):
|
|
* - "app does not report ready before migrations complete" → the committed
|
|
* `apps/server/src/index.ts` gates the health endpoint on a readiness
|
|
* flag (`migrationsComplete`) that starts `false` and flips to `true`
|
|
* only inside the startup migration run's success handler: while the run
|
|
* is in flight, `GET /health` answers HTTP 503 with `{"status":"not
|
|
* ready"}` (the app is up but not ready), never 200. Locked in
|
|
* statically (mutation probes prove non-vacuity: removing the 503
|
|
* branch, answering 200 in the not-ready state, replacing the gate with
|
|
* `if (true)`, or flipping the flag before the run all fail) and
|
|
* behaviorally by the deterministic probes and the docker-gated
|
|
* real-stack probe (the issue's test plan: "start with pending
|
|
* migrations and confirm readiness waits" — the app's migration run is
|
|
* blocked behind a held ACCESS EXCLUSIVE lock on the migration ledger,
|
|
* `/health` stays not-ready, then flips ready once the lock releases).
|
|
* - "readiness is reported only after migrations finish" → the committed
|
|
* source flips the flag only after `runner.run()` resolves (the
|
|
* `migrationsComplete = true` assignment sits inside the `.then` of
|
|
* `run()`, at a source index after the `runner.run()` call), and the
|
|
* real-stack probe confirms `/health` answers HTTP 200
|
|
* `{"status":"ok"}` exactly after the blocked run completes; the app
|
|
* logs the completed run (`applied`/`skipped`) so the flip is
|
|
* attributable to a real migration run.
|
|
* - the readiness gate runs the startup migrations through the driver
|
|
* boundary (`@personal-blog/database-postgres` — `Pool`,
|
|
* `MigrationLedger`, `MigrationRunner`; `Migration` type), never
|
|
* importing `pg`/Kysely directly (isolation E00-S03-T02 stays intact).
|
|
* - the no-migration path: when no `DATABASE_URL` is configured (the local
|
|
* non-container developer path, E00-S01-T06) there is no startup
|
|
* migration run to wait for, so the app reports ready immediately —
|
|
* locked in statically and by a deterministic probe (boot without
|
|
* `DATABASE_URL` → 200 `{"status":"ok"}`), keeping the E00-S02-T03
|
|
* health endpoint and the local `pnpm --filter @personal-blog/server
|
|
* start` path working.
|
|
*
|
|
* Run: `node --test tests/app-readiness.test.mjs`
|
|
* (node:test — built into Node >= 18; no dependencies, lockfile untouched.)
|
|
*/
|
|
|
|
import test from 'node:test';
|
|
import assert from 'node:assert/strict';
|
|
import { readFileSync, existsSync } from 'node:fs';
|
|
import { spawn, spawnSync } from 'node:child_process';
|
|
import { once } from 'node:events';
|
|
import { createServer as createNetServer } from 'node:net';
|
|
import path from 'node:path';
|
|
import { fileURLToPath } from 'node:url';
|
|
|
|
const REPO_ROOT = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..');
|
|
|
|
const read = (relPath) => readFileSync(path.join(REPO_ROOT, relPath), 'utf8');
|
|
|
|
/** The committed server entrypoint under test (the readiness gate lives here). */
|
|
const SERVER_SRC = 'apps/server/src/index.ts';
|
|
|
|
/** The driver boundary the server must run migrations through (E00-S03-T02). */
|
|
const DRIVER_BOUNDARY = '@personal-blog/database-postgres';
|
|
|
|
/** The root test glob (root `scripts.test`, E00-S01-T12) that runs every suite. */
|
|
const ROOT_TEST_GLOB = 'tests/**/*.test.mjs';
|
|
|
|
/** The CI job that gates the readiness criterion on every PR. */
|
|
const CI_JOB = 'app-readiness';
|
|
|
|
const delay = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Static assertions on the committed server entrypoint
|
|
// ---------------------------------------------------------------------------
|
|
|
|
/**
|
|
* Asserts the server runs the startup migrations through the driver boundary:
|
|
* it imports `Pool`, `MigrationLedger` and `MigrationRunner` (and the
|
|
* `Migration` type) from `@personal-blog/database-postgres`, and never
|
|
* imports `pg`/Kysely directly (isolation E00-S03-T02 stays intact). Fails
|
|
* fast on a missing/placeholder entrypoint; the mutation probes below prove
|
|
* the assertions are non-vacuous.
|
|
*/
|
|
function assertDriverBoundaryImports(src) {
|
|
assert.match(
|
|
src,
|
|
/from '@personal-blog\/database-postgres'/,
|
|
`the server must run migrations through the driver boundary (import from '${DRIVER_BOUNDARY}')`,
|
|
);
|
|
assert.match(
|
|
src,
|
|
/import \{ Pool \} from '@personal-blog\/database-postgres'/,
|
|
`the server must import the pool from the driver boundary (import { Pool } from '${DRIVER_BOUNDARY}')`,
|
|
);
|
|
assert.match(
|
|
src,
|
|
/import \{ MigrationLedger, MigrationRunner \} from '@personal-blog\/database-postgres'/,
|
|
`the server must import the ledger and runner from the driver boundary (import { MigrationLedger, MigrationRunner } from '${DRIVER_BOUNDARY}')`,
|
|
);
|
|
assert.match(
|
|
src,
|
|
/import type \{ Migration \} from '@personal-blog\/database-postgres'/,
|
|
`the server must import the Migration type from the driver boundary (import type { Migration } from '${DRIVER_BOUNDARY}')`,
|
|
);
|
|
assert.doesNotMatch(
|
|
src,
|
|
/from\s+'pg'/,
|
|
'the server must not import the pg driver directly (pg/Kysely imports are isolated to database-postgres, E00-S03-T02)',
|
|
);
|
|
assert.doesNotMatch(
|
|
src,
|
|
/from\s+'kysely'/,
|
|
'the server must not import Kysely directly (pg/Kysely imports are isolated to database-postgres, E00-S03-T02)',
|
|
);
|
|
}
|
|
|
|
/**
|
|
* Asserts the readiness gate exists in the /health route: the server answers
|
|
* HTTP 200 with the healthy payload only when `migrationsComplete` is true,
|
|
* and HTTP 503 with the not-ready payload otherwise — the app does not report
|
|
* ready before migrations complete.
|
|
*/
|
|
function assertReadinessGate(src) {
|
|
const route = src.slice(src.indexOf('function handleRequest'));
|
|
assert.match(
|
|
route,
|
|
/if \(migrationsComplete\) \{/,
|
|
'the /health route must gate the healthy answer on migrationsComplete (readiness reported only after migrations finish)',
|
|
);
|
|
assert.match(
|
|
route,
|
|
/sendJson\(res, 200, HEALTH_PAYLOAD\)/,
|
|
'the /health route must answer HTTP 200 with the healthy payload only when ready',
|
|
);
|
|
assert.match(
|
|
route,
|
|
/sendJson\(res, 503, NOT_READY_PAYLOAD\)/,
|
|
'the /health route must answer HTTP 503 with the not-ready payload while migrations are pending',
|
|
);
|
|
assert.match(
|
|
src,
|
|
/const NOT_READY_PAYLOAD = JSON\.stringify\(\{ status: 'not ready' \}\)/,
|
|
"the not-ready payload must report a not-ready application ({\"status\":\"not ready\"})",
|
|
);
|
|
assert.match(
|
|
src,
|
|
/let migrationsComplete = false;/,
|
|
'the readiness flag must start false (the app is not ready before the migration run completes)',
|
|
);
|
|
}
|
|
|
|
/**
|
|
* Asserts readiness flips only after the startup migration run finishes: in
|
|
* the DATABASE_URL-configured path the committed source creates a
|
|
* `MigrationRunner` and calls `run()`, and the `migrationsComplete = true`
|
|
* assignment sits inside the run's `.then` handler — at a source index
|
|
* strictly after the `runner.run()` call — so readiness is reported only
|
|
* after migrations finish.
|
|
*/
|
|
function assertReadyAfterRun(src) {
|
|
const dbUrlIndex = src.indexOf('const databaseUrl = process.env.DATABASE_URL');
|
|
assert.ok(dbUrlIndex !== -1, 'the server must read DATABASE_URL for the startup migration path');
|
|
const elseStart = src.indexOf('} else {', dbUrlIndex);
|
|
assert.ok(elseStart !== -1, 'the DATABASE_URL-configured startup path must exist (else branch)');
|
|
const catchIndex = src.indexOf('.catch(', elseStart);
|
|
assert.ok(catchIndex !== -1, 'the startup migration run must have a failure handler (.catch)');
|
|
const runPath = src.slice(elseStart, catchIndex);
|
|
|
|
assert.match(
|
|
runPath,
|
|
/const runner = new MigrationRunner\(pool, MIGRATIONS, new MigrationLedger\(pool\)\)/,
|
|
'the server must build the startup migration runner over the migration ledger (new MigrationRunner(pool, MIGRATIONS, new MigrationLedger(pool)))',
|
|
);
|
|
const runMatch = /runner\s*\n?\s*\.run\(\)/.exec(runPath);
|
|
assert.ok(runMatch, 'the server must run the startup migrations (runner.run())');
|
|
assert.match(
|
|
runPath,
|
|
/\.then\(\(result\) => \{/,
|
|
'the readiness flip must be attached to the run success path (runner.run().then(...))',
|
|
);
|
|
const flipMatch = /migrationsComplete = true;/.exec(runPath);
|
|
assert.ok(
|
|
flipMatch !== null && flipMatch.index > runMatch.index,
|
|
'migrationsComplete must flip to true only after the startup migration run (runner.run()) is invoked',
|
|
);
|
|
}
|
|
|
|
/**
|
|
* Asserts the no-DATABASE_URL path reports ready immediately: when no
|
|
* `DATABASE_URL` is configured there is no startup migration run to wait for,
|
|
* so the app marks migrations complete without a run (keeping the local
|
|
* non-container path and the E00-S02-T03 health endpoint working).
|
|
*/
|
|
function assertNoDatabaseUrlPath(src) {
|
|
const startup = src.slice(src.indexOf('const databaseUrl = process.env.DATABASE_URL'));
|
|
assert.match(
|
|
startup,
|
|
/if \(databaseUrl === undefined\) \{/,
|
|
'the server must branch on a missing DATABASE_URL (no database configured)',
|
|
);
|
|
const noDbBranch = startup.slice(startup.indexOf('if (databaseUrl === undefined) {'), startup.indexOf('} else {'));
|
|
assert.match(
|
|
noDbBranch,
|
|
/migrationsComplete = true;/,
|
|
'with no DATABASE_URL configured the server must report ready immediately (no migration run to wait for)',
|
|
);
|
|
}
|
|
|
|
/** Runs every static criterion assertion against the committed server entrypoint. */
|
|
function assertReadySource(src) {
|
|
assertDriverBoundaryImports(src);
|
|
assertReadinessGate(src);
|
|
assertReadyAfterRun(src);
|
|
assertNoDatabaseUrlPath(src);
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Boot helpers for the behavioral probes (Node type stripping, no build step)
|
|
// ---------------------------------------------------------------------------
|
|
|
|
/**
|
|
* How the current Node executes TypeScript sources: `default` (>= 23.6, type
|
|
* stripping on by default), `strip-types-flag` (>= 22.6 via
|
|
* `--experimental-strip-types`) or `null` (cannot run .ts at all). The
|
|
* workspace pins engines.node to 24.x, where type stripping is stable.
|
|
*/
|
|
function tsExecMode() {
|
|
const [major, minor] = process.versions.node.split('.').map(Number);
|
|
if (major > 23 || (major === 23 && minor >= 6)) return 'default';
|
|
if (major === 22 && minor >= 6) return 'strip-types-flag';
|
|
return null;
|
|
}
|
|
|
|
/** Reserves an ephemeral TCP port, then releases it for the child to bind. */
|
|
function reservePort() {
|
|
return new Promise((resolve, reject) => {
|
|
const probe = createNetServer();
|
|
probe.once('error', reject);
|
|
probe.listen(0, '127.0.0.1', () => {
|
|
const address = probe.address();
|
|
const port = typeof address === 'object' && address !== null ? address.port : 0;
|
|
probe.close(() => resolve(port));
|
|
});
|
|
});
|
|
}
|
|
|
|
/**
|
|
* Boots the committed server source on `port` with the given env overrides
|
|
* (merged over `process.env`; `DATABASE_URL` is stripped unless explicitly
|
|
* provided). Returns `{ child, stderr, stdout }`; the child writes its
|
|
* stdout/stderr into closures for diagnostics.
|
|
*/
|
|
function bootServer(port, envOverrides = {}) {
|
|
const args =
|
|
tsExecMode() === 'strip-types-flag'
|
|
? ['--experimental-strip-types', SERVER_SRC]
|
|
: [SERVER_SRC];
|
|
const env = { ...process.env, PORT: String(port), ...envOverrides };
|
|
const child = spawn(process.execPath, args, {
|
|
cwd: REPO_ROOT,
|
|
env,
|
|
stdio: ['ignore', 'pipe', 'pipe'],
|
|
});
|
|
let stdout = '';
|
|
let stderr = '';
|
|
child.stdout.on('data', (chunk) => {
|
|
stdout += String(chunk);
|
|
});
|
|
child.stderr.on('data', (chunk) => {
|
|
stderr += String(chunk);
|
|
});
|
|
return { child, stdout: () => stdout, stderr: () => stderr };
|
|
}
|
|
|
|
/**
|
|
* Polls `GET /health` until the server answers with ANY status, the child
|
|
* exits, or the deadline passes. Returns the fetch Response.
|
|
*/
|
|
async function waitForAnswer(port, child, stderr, deadlineMs = 10_000) {
|
|
const deadline = Date.now() + deadlineMs;
|
|
let lastError = '';
|
|
while (Date.now() < deadline) {
|
|
if (child.exitCode !== null) {
|
|
throw new Error(
|
|
`the server exited before answering GET /health (code ${child.exitCode}): ${stderr().trim()}`,
|
|
);
|
|
}
|
|
try {
|
|
return await fetch(`http://127.0.0.1:${port}/health`, {
|
|
signal: AbortSignal.timeout(1_000),
|
|
});
|
|
} catch (err) {
|
|
lastError = err instanceof Error ? err.message : String(err);
|
|
await delay(100);
|
|
}
|
|
}
|
|
throw new Error(
|
|
`GET /health did not answer within ${deadlineMs}ms (last error: ${lastError}; server stderr: ${stderr().trim()})`,
|
|
);
|
|
}
|
|
|
|
/** Kills a booted child (SIGTERM, then SIGKILL if needed) and waits for exit. */
|
|
async function stopChild(child) {
|
|
if (child.exitCode !== null || child.signalCode !== null) return;
|
|
child.kill('SIGTERM');
|
|
await Promise.race([once(child, 'exit'), delay(2_000)]);
|
|
if (child.exitCode === null && child.signalCode === null) child.kill('SIGKILL');
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Criterion tests — static assertions on the committed server entrypoint
|
|
// ---------------------------------------------------------------------------
|
|
|
|
test('the committed server entrypoint exists and gates readiness on the startup migration run', () => {
|
|
assert.ok(existsSync(path.join(REPO_ROOT, SERVER_SRC)), `committed ${SERVER_SRC} must exist`);
|
|
assertReadySource(read(SERVER_SRC));
|
|
});
|
|
|
|
test('the server runs migrations through the driver boundary, never importing pg/Kysely directly', () => {
|
|
assertDriverBoundaryImports(read(SERVER_SRC));
|
|
});
|
|
|
|
test('the /health route reports not-ready while migrations are pending and ready only after they finish', () => {
|
|
assertReadinessGate(read(SERVER_SRC));
|
|
});
|
|
|
|
test('the readiness flag flips to true only inside the startup migration run success handler', () => {
|
|
assertReadyAfterRun(read(SERVER_SRC));
|
|
});
|
|
|
|
test('with no DATABASE_URL configured the server reports ready immediately (no migration run to wait for)', () => {
|
|
assertNoDatabaseUrlPath(read(SERVER_SRC));
|
|
});
|
|
|
|
test('the app-readiness criterion is enforced in CI', () => {
|
|
// Picked up by the root test command (root `scripts.test` glob).
|
|
const scripts = JSON.parse(read('package.json')).scripts ?? {};
|
|
assert.equal(
|
|
scripts.test,
|
|
`node --test "${ROOT_TEST_GLOB}"`,
|
|
`root scripts.test must run the "${ROOT_TEST_GLOB}" glob so this suite runs with the rest`,
|
|
);
|
|
// And a dedicated CI job gates it on every PR.
|
|
const workflow = read('.gitea/workflows/ci.yml');
|
|
assert.ok(
|
|
workflow.includes(`node --test tests/app-readiness.test.mjs`),
|
|
`CI must run the app-readiness suite (job "${CI_JOB}") on every PR`,
|
|
);
|
|
});
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Deterministic behavioral probes (no database, no Docker) — the committed
|
|
// server is booted and probed over real HTTP
|
|
// ---------------------------------------------------------------------------
|
|
|
|
/** True when this Node can execute the committed `.ts` server source (>= 22.6, type stripping). */
|
|
const TS_STRIPPING = tsExecMode() !== null;
|
|
|
|
test('without DATABASE_URL the app reports ready immediately (deterministic probe)', { skip: !TS_STRIPPING }, async (t) => {
|
|
// The no-migration path (local non-container dev, E00-S01-T06): no
|
|
// DATABASE_URL is configured, so there is no startup migration run to wait
|
|
// for and GET /health must answer 200 {"status":"ok"} right away — keeping
|
|
// the E00-S02-T03 health endpoint working.
|
|
const port = await reservePort();
|
|
const { child, stdout, stderr } = bootServer(port); // DATABASE_URL stripped
|
|
try {
|
|
const response = await waitForAnswer(port, child, stderr);
|
|
assert.equal(
|
|
response.status,
|
|
200,
|
|
`GET /health without DATABASE_URL must answer 200 (got ${response.status}); server output: ${stdout().trim()} ${stderr().trim()}`,
|
|
);
|
|
assert.deepEqual(
|
|
await response.json(),
|
|
{ status: 'ok' },
|
|
'without DATABASE_URL the app must report a healthy application ({"status":"ok"})',
|
|
);
|
|
} finally {
|
|
await stopChild(child);
|
|
}
|
|
});
|
|
|
|
test('the app does not report ready while the migration run cannot complete (unreachable database)', { skip: !TS_STRIPPING }, async (t) => {
|
|
// "app does not report ready before migrations complete": with a
|
|
// DATABASE_URL that refuses connections, the startup migration run cannot
|
|
// complete, so the app must stay up but answer HTTP 503
|
|
// {"status":"not ready"} — never 200 — until the process is stopped.
|
|
const port = await reservePort();
|
|
const deadPort = await reservePort(); // reserved then released: nothing listens
|
|
const { child, stdout, stderr } = bootServer(port, {
|
|
DATABASE_URL: `postgres://eppp:eppp@127.0.0.1:${deadPort}/eppp`,
|
|
});
|
|
try {
|
|
// Every answer over a ~2s window must be 503 not-ready, never 200.
|
|
let answered = 0;
|
|
for (let attempt = 0; attempt < 6; attempt += 1) {
|
|
const response = await waitForAnswer(port, child, stderr);
|
|
answered += 1;
|
|
assert.equal(
|
|
response.status,
|
|
503,
|
|
`GET /health must answer 503 while the migration run cannot complete (got ${response.status}); server output: ${stdout().trim()} ${stderr().trim()}`,
|
|
);
|
|
assert.deepEqual(
|
|
await response.json(),
|
|
{ status: 'not ready' },
|
|
'the app must report not-ready ({"status":"not ready"}) while migrations are incomplete',
|
|
);
|
|
await delay(300);
|
|
}
|
|
assert.ok(answered >= 2, `the app must keep answering during the probe (answered ${answered} times)`);
|
|
assert.equal(
|
|
child.exitCode,
|
|
null,
|
|
'the app must stay up (not crash) while the migration run cannot complete — not-ready is a stable reported state',
|
|
);
|
|
assert.match(
|
|
stdout() + stderr(),
|
|
/startup migration run failed; app stays not-ready/,
|
|
'the app must log that the startup migration run failed and it stays not-ready',
|
|
);
|
|
} finally {
|
|
await stopChild(child);
|
|
}
|
|
});
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Docker probe helpers (the real-stack probe skips cleanly without Docker)
|
|
// ---------------------------------------------------------------------------
|
|
|
|
function run(cmd, args, opts = {}) {
|
|
return spawnSync(cmd, args, {
|
|
encoding: 'utf8',
|
|
timeout: 600_000,
|
|
...opts,
|
|
});
|
|
}
|
|
|
|
/** True when the `docker` CLI with the Compose plugin is on PATH. */
|
|
function dockerComposeAvailable() {
|
|
try {
|
|
return run('docker', ['compose', 'version'], { timeout: 15_000 }).status === 0;
|
|
} catch {
|
|
return false;
|
|
}
|
|
}
|
|
|
|
/** True when a reachable Docker daemon exists. */
|
|
function dockerDaemonAvailable() {
|
|
try {
|
|
return run('docker', ['info'], { timeout: 15_000 }).status === 0;
|
|
} catch {
|
|
return false;
|
|
}
|
|
}
|
|
|
|
/** Parses `docker compose ps --format json` (JSON array or one object per line). */
|
|
function parsePsJson(stdout) {
|
|
const text = String(stdout).trim();
|
|
if (!text) return [];
|
|
try {
|
|
const parsed = JSON.parse(text);
|
|
return Array.isArray(parsed) ? parsed : [parsed];
|
|
} catch {
|
|
return text
|
|
.split('\n')
|
|
.map((line) => line.trim())
|
|
.filter(Boolean)
|
|
.map((line) => JSON.parse(line));
|
|
}
|
|
}
|
|
|
|
/** Tolerant field lookup across compose ps JSON shapes. */
|
|
function field(container, ...names) {
|
|
for (const name of names) {
|
|
if (container[name] !== undefined) return container[name];
|
|
}
|
|
return undefined;
|
|
}
|
|
|
|
/**
|
|
* Polls `docker compose ps` until the db container of the given project
|
|
* reports healthy (or the deadline passes), so the probe never races a
|
|
* still-booting database.
|
|
*/
|
|
function waitForDbHealthy(project, deadlineMs = 60_000) {
|
|
const deadline = Date.now() + deadlineMs;
|
|
let last = '';
|
|
while (Date.now() < deadline) {
|
|
const ps = run('docker', ['compose', '-p', project, 'ps', '--format', 'json'], {
|
|
cwd: REPO_ROOT,
|
|
timeout: 15_000,
|
|
});
|
|
if (ps.status === 0) {
|
|
last = ps.stdout;
|
|
const db = parsePsJson(ps.stdout).find((c) => field(c, 'Service', 'service') === 'db');
|
|
if (db && /healthy/i.test(String(field(db, 'Health', 'health') ?? ''))) return;
|
|
}
|
|
run(process.execPath, ['-e', 'setTimeout(() => {}, 1000)']); // db still booting — retry
|
|
}
|
|
throw new Error(`the db container did not become healthy within ${deadlineMs}ms (last ps: "${last.trim()}")`);
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Real-stack probe — the issue's test plan: "start with pending migrations
|
|
// and confirm readiness waits", against a real database
|
|
// ---------------------------------------------------------------------------
|
|
|
|
const DOCKER_COMPOSE = dockerComposeAvailable();
|
|
const DOCKER_DAEMON = dockerDaemonAvailable();
|
|
|
|
// An isolated compose project + non-default host port so this probe never
|
|
// collides with the other suites' projects/ports (ledger 55432, lock 55433,
|
|
// diagnostic 55434, compose-config's default project/5432).
|
|
const COMPOSE_PROJECT = 'eppp-readiness-probe';
|
|
const POSTGRES_HOST_PORT = '55435';
|
|
const DATABASE_URL = `postgres://eppp:eppp@127.0.0.1:${POSTGRES_HOST_PORT}/eppp`;
|
|
|
|
test('the app reports not-ready while the migration run is pending and ready only after it finishes (real stack)', { skip: !DOCKER_COMPOSE || !DOCKER_DAEMON || !TS_STRIPPING }, async (t) => {
|
|
// The issue's test plan: "start with pending migrations and confirm
|
|
// readiness waits". The probe starts the committed compose `db` service
|
|
// (its own project + host port), creates the migration ledger, then holds
|
|
// an ACCESS EXCLUSIVE lock on the ledger from a background psql session so
|
|
// the app's startup migration run (ledger.ensure/applied) is genuinely
|
|
// pending. It boots the committed server against that database and asserts
|
|
// GET /health answers 503 {"status":"not ready"} while the run is blocked,
|
|
// then — once the lock releases and the run completes — answers 200
|
|
// {"status":"ok"} and logs the completed run. Rollback: drop the ledger
|
|
// and `docker compose -p eppp-readiness-probe down -v` (issue rollback
|
|
// note: revert the readiness gating logic).
|
|
const composeEnv = { ...process.env, POSTGRES_PORT: POSTGRES_HOST_PORT };
|
|
const compose = (args, opts = {}) =>
|
|
run('docker', ['compose', '-p', COMPOSE_PROJECT, ...args], { cwd: REPO_ROOT, env: composeEnv, ...opts });
|
|
const execPsql = (args, opts = {}) =>
|
|
compose(['exec', '-T', 'db', 'psql', '-U', 'eppp', '-d', 'eppp', ...args], opts);
|
|
|
|
let app;
|
|
let lockHolder;
|
|
|
|
try {
|
|
const up = compose(['up', '-d', 'db'], { timeout: 180_000 });
|
|
assert.equal(
|
|
up.status,
|
|
0,
|
|
`"docker compose up -d db" must exit 0:\n${(up.stdout || '')}\n${(up.stderr || '')}`.trim(),
|
|
);
|
|
waitForDbHealthy(COMPOSE_PROJECT);
|
|
|
|
// Clean slate (also recovers from a previously interrupted run): drop any
|
|
// leftover ledger, then create it so the app's migration run has a ledger
|
|
// to read (the run's ledger.ensure() is then a no-op and its
|
|
// ledger.applied() SELECT is what blocks behind the lock below).
|
|
const clean = execPsql([
|
|
'-v', 'ON_ERROR_STOP=1',
|
|
'-c',
|
|
'DROP TABLE IF EXISTS schema_migrations; ' +
|
|
'CREATE TABLE schema_migrations (version text PRIMARY KEY, applied_at timestamptz NOT NULL DEFAULT now());',
|
|
]);
|
|
assert.equal(
|
|
clean.status,
|
|
0,
|
|
`creating the migration ledger must succeed:\n${(clean.stdout || '')}\n${(clean.stderr || '')}`.trim(),
|
|
);
|
|
|
|
// Hold an ACCESS EXCLUSIVE lock on the ledger in a background psql
|
|
// session (BEGIN; LOCK; SELECT pg_sleep(30) keeps the transaction — and
|
|
// the lock — open). The app's migration run blocks on this lock, so the
|
|
// run is deterministically "pending" while we assert not-ready.
|
|
lockHolder = spawn(
|
|
'docker',
|
|
[
|
|
'compose', '-p', COMPOSE_PROJECT, 'exec', '-T', 'db',
|
|
'psql', '-U', 'eppp', '-d', 'eppp', '-v', 'ON_ERROR_STOP=1',
|
|
'-c', 'BEGIN; LOCK TABLE schema_migrations IN ACCESS EXCLUSIVE MODE; SELECT pg_sleep(30);',
|
|
],
|
|
{ cwd: REPO_ROOT, env: composeEnv, stdio: ['ignore', 'pipe', 'pipe'] },
|
|
);
|
|
let holderErr = '';
|
|
lockHolder.stderr.on('data', (chunk) => {
|
|
holderErr += String(chunk);
|
|
});
|
|
|
|
// Wait until the AccessExclusiveLock is actually held (deterministic, so
|
|
// the app boots into a genuinely blocked run).
|
|
const lockDeadline = Date.now() + 15_000;
|
|
let lockHeld = false;
|
|
while (Date.now() < lockDeadline) {
|
|
const held = execPsql([
|
|
'-tA',
|
|
'-c',
|
|
"SELECT count(*) FROM pg_locks WHERE locktype = 'relation' AND relation = 'schema_migrations'::regclass AND mode = 'AccessExclusiveLock';",
|
|
]);
|
|
if (held.status === 0 && held.stdout.trim() === '1') {
|
|
lockHeld = true;
|
|
break;
|
|
}
|
|
run(process.execPath, ['-e', 'setTimeout(() => {}, 500)']);
|
|
}
|
|
assert.ok(lockHeld, `the probe must hold the ACCESS EXCLUSIVE lock on the ledger (holder stderr: "${holderErr.trim()}")`);
|
|
|
|
// Boot the committed server against the probe database.
|
|
const port = await reservePort();
|
|
app = bootServer(port, { DATABASE_URL });
|
|
|
|
// "app does not report ready before migrations complete": while the run
|
|
// is blocked, every /health answer must be 503 {"status":"not ready"}.
|
|
for (let attempt = 0; attempt < 5; attempt += 1) {
|
|
const response = await waitForAnswer(port, app.child, app.stderr);
|
|
assert.equal(
|
|
response.status,
|
|
503,
|
|
`GET /health must answer 503 while the migration run is pending (got ${response.status}); server output: ${app.stdout().trim()} ${app.stderr().trim()}`,
|
|
);
|
|
assert.deepEqual(
|
|
await response.json(),
|
|
{ status: 'not ready' },
|
|
'the app must report not-ready ({"status":"not ready"}) while migrations are pending',
|
|
);
|
|
await delay(300);
|
|
}
|
|
assert.equal(
|
|
app.child.exitCode,
|
|
null,
|
|
'the app must stay up (not crash) while the migration run is pending',
|
|
);
|
|
|
|
// Release the lock: the psql session ends, its transaction rolls back and
|
|
// the ACCESS EXCLUSIVE lock is gone — the app's migration run completes.
|
|
lockHolder.kill('SIGTERM');
|
|
await Promise.race([once(lockHolder, 'exit'), delay(2_000)]);
|
|
if (lockHolder.exitCode === null && lockHolder.signalCode === null) lockHolder.kill('SIGKILL');
|
|
lockHolder = null;
|
|
|
|
// "readiness is reported only after migrations finish": once the run
|
|
// completes, /health must answer 200 {"status":"ok"}.
|
|
const readyDeadline = Date.now() + 15_000;
|
|
let ready = null;
|
|
while (Date.now() < readyDeadline) {
|
|
const response = await waitForAnswer(port, app.child, app.stderr);
|
|
if (response.status === 200) {
|
|
ready = await response.json();
|
|
break;
|
|
}
|
|
assert.equal(
|
|
response.status,
|
|
503,
|
|
`GET /health must stay 503 only while the run is pending (got ${response.status})`,
|
|
);
|
|
await delay(200);
|
|
}
|
|
assert.ok(ready, `the app must report ready after the migration run completes (server output: ${app.stdout().trim()} ${app.stderr().trim()})`);
|
|
assert.deepEqual(ready, { status: 'ok' }, 'the app must report a healthy application ({"status":"ok"}) after migrations finish');
|
|
|
|
// The readiness flip must be attributable to a completed migration run:
|
|
// the app logs the completed run, and the migration ledger still exists.
|
|
assert.match(
|
|
app.stdout() + app.stderr(),
|
|
/startup migration run complete \(applied 0, skipped 0\)/,
|
|
`the app must log the completed startup migration run (got: ${app.stdout().trim()} ${app.stderr().trim()})`,
|
|
);
|
|
const ledgerClass = execPsql(['-tA', '-c', "SELECT to_regclass('public.schema_migrations');"]);
|
|
assert.equal(
|
|
ledgerClass.status,
|
|
0,
|
|
`ledger existence query must succeed:\n${(ledgerClass.stdout || '')}\n${(ledgerClass.stderr || '')}`.trim(),
|
|
);
|
|
assert.match(
|
|
ledgerClass.stdout,
|
|
/schema_migrations/,
|
|
'the migration ledger must exist after the startup migration run',
|
|
);
|
|
} finally {
|
|
if (app) await stopChild(app.child);
|
|
if (lockHolder) {
|
|
lockHolder.kill('SIGKILL');
|
|
await Promise.race([once(lockHolder, 'exit'), delay(1_000)]);
|
|
}
|
|
// Rollback note from the issue: revert the readiness gating logic — here,
|
|
// reset migration state and tear down the isolated project.
|
|
try {
|
|
execPsql(['-c', 'DROP TABLE IF EXISTS schema_migrations;']);
|
|
} catch {
|
|
// container may already be gone — the compose down below still cleans up
|
|
}
|
|
compose(['down', '-v'], { timeout: 120_000 });
|
|
}
|
|
});
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Non-vacuous probes — the assertions above really do fail on violations
|
|
// ---------------------------------------------------------------------------
|
|
|
|
test('removing the 503 not-ready branch makes the readiness-gate criterion fail (mutation probe)', () => {
|
|
const src = read(SERVER_SRC);
|
|
const withoutNotReady = src.replace(/ } else \{\n sendJson\(res, 503, NOT_READY_PAYLOAD\);\n \}\n/, ' }\n');
|
|
assert.notEqual(withoutNotReady, src, 'the mutation must actually remove the 503 branch');
|
|
assert.throws(() => assertReadinessGate(withoutNotReady), /503/);
|
|
});
|
|
|
|
test('answering 200 in the not-ready state makes the gate criterion fail (mutation probe)', () => {
|
|
const src = read(SERVER_SRC);
|
|
const notReady200 = src.replace('sendJson(res, 503, NOT_READY_PAYLOAD)', 'sendJson(res, 200, NOT_READY_PAYLOAD)');
|
|
assert.notEqual(notReady200, src, 'the mutation must actually change the not-ready status code');
|
|
assert.throws(() => assertReadinessGate(notReady200), /503/);
|
|
});
|
|
|
|
test('replacing the gate with an unconditional healthy answer fails the gate criterion (mutation probe)', () => {
|
|
const src = read(SERVER_SRC);
|
|
const noGate = src.replace(/if \(migrationsComplete\) \{/, 'if (true) {');
|
|
assert.notEqual(noGate, src, 'the mutation must actually bypass the readiness flag');
|
|
assert.throws(() => assertReadinessGate(noGate), /migrationsComplete/);
|
|
});
|
|
|
|
test('removing the not-ready payload fails the not-ready-report criterion (mutation probe)', () => {
|
|
const src = read(SERVER_SRC);
|
|
const healthyNotReady = src.replace("status: 'not ready'", "status: 'ok'");
|
|
assert.notEqual(healthyNotReady, src, 'the mutation must actually change the not-ready payload');
|
|
assert.throws(() => assertReadinessGate(healthyNotReady), /not ready/);
|
|
});
|
|
|
|
test('flipping the readiness flag before the migration run fails the readiness-only-after criterion (mutation probe)', () => {
|
|
const src = read(SERVER_SRC);
|
|
const earlyFlip = src.replace(
|
|
/const runner = new MigrationRunner\(pool, MIGRATIONS, new MigrationLedger\(pool\)\);\n runner/,
|
|
'migrationsComplete = true;\n const runner = new MigrationRunner(pool, MIGRATIONS, new MigrationLedger(pool));\n runner',
|
|
);
|
|
assert.notEqual(earlyFlip, src, 'the mutation must actually flip the flag before the run');
|
|
assert.throws(() => assertReadyAfterRun(earlyFlip), /only after/);
|
|
});
|
|
|
|
test('dropping the migration run fails the readiness-after-run criterion (mutation probe)', () => {
|
|
const src = read(SERVER_SRC);
|
|
const noRun = src.replace(/ runner\n \.run\(\)\n/, '');
|
|
assert.notEqual(noRun, src, 'the mutation must actually remove the runner.run() call');
|
|
assert.throws(() => assertReadyAfterRun(noRun), /runner\.run\(\)/);
|
|
});
|
|
|
|
test('importing pg directly instead of the driver boundary fails the isolation criterion (mutation probe)', () => {
|
|
const src = read(SERVER_SRC);
|
|
const directPg = src.replace(
|
|
"import { Pool } from '@personal-blog/database-postgres';",
|
|
"import { Pool } from 'pg';",
|
|
);
|
|
assert.notEqual(directPg, src, 'the mutation must actually replace the boundary import');
|
|
assert.throws(() => assertDriverBoundaryImports(directPg), /must import the pool from the driver boundary/);
|
|
});
|
|
|
|
test('removing the no-DATABASE_URL ready path fails the no-migration criterion (mutation probe)', () => {
|
|
const src = read(SERVER_SRC);
|
|
const withoutNoDb = src.replace(/if \(databaseUrl === undefined\) \{\n[\s\S]*?\} else \{/, '} else {');
|
|
assert.notEqual(withoutNoDb, src, 'the mutation must actually remove the no-DATABASE_URL branch');
|
|
assert.throws(() => assertNoDatabaseUrlPath(withoutNoDb), /must branch on a missing DATABASE_URL/);
|
|
});
|
|
|
|
test('a placeholder entrypoint (no server at all) fails the readiness-source criterion (mutation probe)', () => {
|
|
assert.throws(() => assertReadySource('export {};\n'), /driver boundary/);
|
|
});
|