test: lock in the app readiness gate with static, deterministic and real-stack probes (E00-S03-T06)
This commit is contained in:
@@ -0,0 +1,756 @@
|
||||
/**
|
||||
* App readiness test — locks in the [E00-S03-T06] guarantee that the app does
|
||||
* not report ready before migrations complete.
|
||||
*
|
||||
* Acceptance criteria covered (each test fails without the committed state):
|
||||
* - "app does not report ready before migrations complete" → the committed
|
||||
* `apps/server/src/index.ts` gates the health endpoint on a readiness
|
||||
* flag (`migrationsComplete`) that starts `false` and flips to `true`
|
||||
* only inside the startup migration run's success handler: while the run
|
||||
* is in flight, `GET /health` answers HTTP 503 with `{"status":"not
|
||||
* ready"}` (the app is up but not ready), never 200. Locked in
|
||||
* statically (mutation probes prove non-vacuity: removing the 503
|
||||
* branch, answering 200 in the not-ready state, replacing the gate with
|
||||
* `if (true)`, or flipping the flag before the run all fail) and
|
||||
* behaviorally by the deterministic probes and the docker-gated
|
||||
* real-stack probe (the issue's test plan: "start with pending
|
||||
* migrations and confirm readiness waits" — the app's migration run is
|
||||
* blocked behind a held ACCESS EXCLUSIVE lock on the migration ledger,
|
||||
* `/health` stays not-ready, then flips ready once the lock releases).
|
||||
* - "readiness is reported only after migrations finish" → the committed
|
||||
* source flips the flag only after `runner.run()` resolves (the
|
||||
* `migrationsComplete = true` assignment sits inside the `.then` of
|
||||
* `run()`, at a source index after the `runner.run()` call), and the
|
||||
* real-stack probe confirms `/health` answers HTTP 200
|
||||
* `{"status":"ok"}` exactly after the blocked run completes; the app
|
||||
* logs the completed run (`applied`/`skipped`) so the flip is
|
||||
* attributable to a real migration run.
|
||||
* - the readiness gate runs the startup migrations through the driver
|
||||
* boundary (`@personal-blog/database-postgres` — `Pool`,
|
||||
* `MigrationLedger`, `MigrationRunner`; `Migration` type), never
|
||||
* importing `pg`/Kysely directly (isolation E00-S03-T02 stays intact).
|
||||
* - the no-migration path: when no `DATABASE_URL` is configured (the local
|
||||
* non-container developer path, E00-S01-T06) there is no startup
|
||||
* migration run to wait for, so the app reports ready immediately —
|
||||
* locked in statically and by a deterministic probe (boot without
|
||||
* `DATABASE_URL` → 200 `{"status":"ok"}`), keeping the E00-S02-T03
|
||||
* health endpoint and the local `pnpm --filter @personal-blog/server
|
||||
* start` path working.
|
||||
*
|
||||
* Run: `node --test tests/app-readiness.test.mjs`
|
||||
* (node:test — built into Node >= 18; no dependencies, lockfile untouched.)
|
||||
*/
|
||||
|
||||
import test from 'node:test';
|
||||
import assert from 'node:assert/strict';
|
||||
import { readFileSync, existsSync } from 'node:fs';
|
||||
import { spawn, spawnSync } from 'node:child_process';
|
||||
import { once } from 'node:events';
|
||||
import { createServer as createNetServer } from 'node:net';
|
||||
import path from 'node:path';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
|
||||
const REPO_ROOT = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..');
|
||||
|
||||
const read = (relPath) => readFileSync(path.join(REPO_ROOT, relPath), 'utf8');
|
||||
|
||||
/** The committed server entrypoint under test (the readiness gate lives here). */
|
||||
const SERVER_SRC = 'apps/server/src/index.ts';
|
||||
|
||||
/** The driver boundary the server must run migrations through (E00-S03-T02). */
|
||||
const DRIVER_BOUNDARY = '@personal-blog/database-postgres';
|
||||
|
||||
/** The root test glob (root `scripts.test`, E00-S01-T12) that runs every suite. */
|
||||
const ROOT_TEST_GLOB = 'tests/**/*.test.mjs';
|
||||
|
||||
/** The CI job that gates the readiness criterion on every PR. */
|
||||
const CI_JOB = 'app-readiness';
|
||||
|
||||
const delay = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Static assertions on the committed server entrypoint
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/**
|
||||
* Asserts the server runs the startup migrations through the driver boundary:
|
||||
* it imports `Pool`, `MigrationLedger` and `MigrationRunner` (and the
|
||||
* `Migration` type) from `@personal-blog/database-postgres`, and never
|
||||
* imports `pg`/Kysely directly (isolation E00-S03-T02 stays intact). Fails
|
||||
* fast on a missing/placeholder entrypoint; the mutation probes below prove
|
||||
* the assertions are non-vacuous.
|
||||
*/
|
||||
function assertDriverBoundaryImports(src) {
|
||||
assert.match(
|
||||
src,
|
||||
/from '@personal-blog\/database-postgres'/,
|
||||
`the server must run migrations through the driver boundary (import from '${DRIVER_BOUNDARY}')`,
|
||||
);
|
||||
assert.match(
|
||||
src,
|
||||
/import \{ Pool \} from '@personal-blog\/database-postgres'/,
|
||||
`the server must import the pool from the driver boundary (import { Pool } from '${DRIVER_BOUNDARY}')`,
|
||||
);
|
||||
assert.match(
|
||||
src,
|
||||
/import \{ MigrationLedger, MigrationRunner \} from '@personal-blog\/database-postgres'/,
|
||||
`the server must import the ledger and runner from the driver boundary (import { MigrationLedger, MigrationRunner } from '${DRIVER_BOUNDARY}')`,
|
||||
);
|
||||
assert.match(
|
||||
src,
|
||||
/import type \{ Migration \} from '@personal-blog\/database-postgres'/,
|
||||
`the server must import the Migration type from the driver boundary (import type { Migration } from '${DRIVER_BOUNDARY}')`,
|
||||
);
|
||||
assert.doesNotMatch(
|
||||
src,
|
||||
/from\s+'pg'/,
|
||||
'the server must not import the pg driver directly (pg/Kysely imports are isolated to database-postgres, E00-S03-T02)',
|
||||
);
|
||||
assert.doesNotMatch(
|
||||
src,
|
||||
/from\s+'kysely'/,
|
||||
'the server must not import Kysely directly (pg/Kysely imports are isolated to database-postgres, E00-S03-T02)',
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Asserts the readiness gate exists in the /health route: the server answers
|
||||
* HTTP 200 with the healthy payload only when `migrationsComplete` is true,
|
||||
* and HTTP 503 with the not-ready payload otherwise — the app does not report
|
||||
* ready before migrations complete.
|
||||
*/
|
||||
function assertReadinessGate(src) {
|
||||
const route = src.slice(src.indexOf('function handleRequest'));
|
||||
assert.match(
|
||||
route,
|
||||
/if \(migrationsComplete\) \{/,
|
||||
'the /health route must gate the healthy answer on migrationsComplete (readiness reported only after migrations finish)',
|
||||
);
|
||||
assert.match(
|
||||
route,
|
||||
/sendJson\(res, 200, HEALTH_PAYLOAD\)/,
|
||||
'the /health route must answer HTTP 200 with the healthy payload only when ready',
|
||||
);
|
||||
assert.match(
|
||||
route,
|
||||
/sendJson\(res, 503, NOT_READY_PAYLOAD\)/,
|
||||
'the /health route must answer HTTP 503 with the not-ready payload while migrations are pending',
|
||||
);
|
||||
assert.match(
|
||||
src,
|
||||
/const NOT_READY_PAYLOAD = JSON\.stringify\(\{ status: 'not ready' \}\)/,
|
||||
"the not-ready payload must report a not-ready application ({\"status\":\"not ready\"})",
|
||||
);
|
||||
assert.match(
|
||||
src,
|
||||
/let migrationsComplete = false;/,
|
||||
'the readiness flag must start false (the app is not ready before the migration run completes)',
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Asserts readiness flips only after the startup migration run finishes: in
|
||||
* the DATABASE_URL-configured path the committed source creates a
|
||||
* `MigrationRunner` and calls `run()`, and the `migrationsComplete = true`
|
||||
* assignment sits inside the run's `.then` handler — at a source index
|
||||
* strictly after the `runner.run()` call — so readiness is reported only
|
||||
* after migrations finish.
|
||||
*/
|
||||
function assertReadyAfterRun(src) {
|
||||
const dbUrlIndex = src.indexOf('const databaseUrl = process.env.DATABASE_URL');
|
||||
assert.ok(dbUrlIndex !== -1, 'the server must read DATABASE_URL for the startup migration path');
|
||||
const elseStart = src.indexOf('} else {', dbUrlIndex);
|
||||
assert.ok(elseStart !== -1, 'the DATABASE_URL-configured startup path must exist (else branch)');
|
||||
const catchIndex = src.indexOf('.catch(', elseStart);
|
||||
assert.ok(catchIndex !== -1, 'the startup migration run must have a failure handler (.catch)');
|
||||
const runPath = src.slice(elseStart, catchIndex);
|
||||
|
||||
assert.match(
|
||||
runPath,
|
||||
/const runner = new MigrationRunner\(pool, MIGRATIONS, new MigrationLedger\(pool\)\)/,
|
||||
'the server must build the startup migration runner over the migration ledger (new MigrationRunner(pool, MIGRATIONS, new MigrationLedger(pool)))',
|
||||
);
|
||||
const runMatch = /runner\s*\n?\s*\.run\(\)/.exec(runPath);
|
||||
assert.ok(runMatch, 'the server must run the startup migrations (runner.run())');
|
||||
assert.match(
|
||||
runPath,
|
||||
/\.then\(\(result\) => \{/,
|
||||
'the readiness flip must be attached to the run success path (runner.run().then(...))',
|
||||
);
|
||||
const flipMatch = /migrationsComplete = true;/.exec(runPath);
|
||||
assert.ok(
|
||||
flipMatch !== null && flipMatch.index > runMatch.index,
|
||||
'migrationsComplete must flip to true only after the startup migration run (runner.run()) is invoked',
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Asserts the no-DATABASE_URL path reports ready immediately: when no
|
||||
* `DATABASE_URL` is configured there is no startup migration run to wait for,
|
||||
* so the app marks migrations complete without a run (keeping the local
|
||||
* non-container path and the E00-S02-T03 health endpoint working).
|
||||
*/
|
||||
function assertNoDatabaseUrlPath(src) {
|
||||
const startup = src.slice(src.indexOf('const databaseUrl = process.env.DATABASE_URL'));
|
||||
assert.match(
|
||||
startup,
|
||||
/if \(databaseUrl === undefined\) \{/,
|
||||
'the server must branch on a missing DATABASE_URL (no database configured)',
|
||||
);
|
||||
const noDbBranch = startup.slice(startup.indexOf('if (databaseUrl === undefined) {'), startup.indexOf('} else {'));
|
||||
assert.match(
|
||||
noDbBranch,
|
||||
/migrationsComplete = true;/,
|
||||
'with no DATABASE_URL configured the server must report ready immediately (no migration run to wait for)',
|
||||
);
|
||||
}
|
||||
|
||||
/** Runs every static criterion assertion against the committed server entrypoint. */
|
||||
function assertReadySource(src) {
|
||||
assertDriverBoundaryImports(src);
|
||||
assertReadinessGate(src);
|
||||
assertReadyAfterRun(src);
|
||||
assertNoDatabaseUrlPath(src);
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Boot helpers for the behavioral probes (Node type stripping, no build step)
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/**
|
||||
* How the current Node executes TypeScript sources: `default` (>= 23.6, type
|
||||
* stripping on by default), `strip-types-flag` (>= 22.6 via
|
||||
* `--experimental-strip-types`) or `null` (cannot run .ts at all). The
|
||||
* workspace pins engines.node to 24.x, where type stripping is stable.
|
||||
*/
|
||||
function tsExecMode() {
|
||||
const [major, minor] = process.versions.node.split('.').map(Number);
|
||||
if (major > 23 || (major === 23 && minor >= 6)) return 'default';
|
||||
if (major === 22 && minor >= 6) return 'strip-types-flag';
|
||||
return null;
|
||||
}
|
||||
|
||||
/** Reserves an ephemeral TCP port, then releases it for the child to bind. */
|
||||
function reservePort() {
|
||||
return new Promise((resolve, reject) => {
|
||||
const probe = createNetServer();
|
||||
probe.once('error', reject);
|
||||
probe.listen(0, '127.0.0.1', () => {
|
||||
const address = probe.address();
|
||||
const port = typeof address === 'object' && address !== null ? address.port : 0;
|
||||
probe.close(() => resolve(port));
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Boots the committed server source on `port` with the given env overrides
|
||||
* (merged over `process.env`; `DATABASE_URL` is stripped unless explicitly
|
||||
* provided). Returns `{ child, stderr, stdout }`; the child writes its
|
||||
* stdout/stderr into closures for diagnostics.
|
||||
*/
|
||||
function bootServer(port, envOverrides = {}) {
|
||||
const args =
|
||||
tsExecMode() === 'strip-types-flag'
|
||||
? ['--experimental-strip-types', SERVER_SRC]
|
||||
: [SERVER_SRC];
|
||||
const env = { ...process.env, PORT: String(port), ...envOverrides };
|
||||
const child = spawn(process.execPath, args, {
|
||||
cwd: REPO_ROOT,
|
||||
env,
|
||||
stdio: ['ignore', 'pipe', 'pipe'],
|
||||
});
|
||||
let stdout = '';
|
||||
let stderr = '';
|
||||
child.stdout.on('data', (chunk) => {
|
||||
stdout += String(chunk);
|
||||
});
|
||||
child.stderr.on('data', (chunk) => {
|
||||
stderr += String(chunk);
|
||||
});
|
||||
return { child, stdout: () => stdout, stderr: () => stderr };
|
||||
}
|
||||
|
||||
/**
|
||||
* Polls `GET /health` until the server answers with ANY status, the child
|
||||
* exits, or the deadline passes. Returns the fetch Response.
|
||||
*/
|
||||
async function waitForAnswer(port, child, stderr, deadlineMs = 10_000) {
|
||||
const deadline = Date.now() + deadlineMs;
|
||||
let lastError = '';
|
||||
while (Date.now() < deadline) {
|
||||
if (child.exitCode !== null) {
|
||||
throw new Error(
|
||||
`the server exited before answering GET /health (code ${child.exitCode}): ${stderr().trim()}`,
|
||||
);
|
||||
}
|
||||
try {
|
||||
return await fetch(`http://127.0.0.1:${port}/health`, {
|
||||
signal: AbortSignal.timeout(1_000),
|
||||
});
|
||||
} catch (err) {
|
||||
lastError = err instanceof Error ? err.message : String(err);
|
||||
await delay(100);
|
||||
}
|
||||
}
|
||||
throw new Error(
|
||||
`GET /health did not answer within ${deadlineMs}ms (last error: ${lastError}; server stderr: ${stderr().trim()})`,
|
||||
);
|
||||
}
|
||||
|
||||
/** Kills a booted child (SIGTERM, then SIGKILL if needed) and waits for exit. */
|
||||
async function stopChild(child) {
|
||||
if (child.exitCode !== null || child.signalCode !== null) return;
|
||||
child.kill('SIGTERM');
|
||||
await Promise.race([once(child, 'exit'), delay(2_000)]);
|
||||
if (child.exitCode === null && child.signalCode === null) child.kill('SIGKILL');
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Criterion tests — static assertions on the committed server entrypoint
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
test('the committed server entrypoint exists and gates readiness on the startup migration run', () => {
|
||||
assert.ok(existsSync(path.join(REPO_ROOT, SERVER_SRC)), `committed ${SERVER_SRC} must exist`);
|
||||
assertReadySource(read(SERVER_SRC));
|
||||
});
|
||||
|
||||
test('the server runs migrations through the driver boundary, never importing pg/Kysely directly', () => {
|
||||
assertDriverBoundaryImports(read(SERVER_SRC));
|
||||
});
|
||||
|
||||
test('the /health route reports not-ready while migrations are pending and ready only after they finish', () => {
|
||||
assertReadinessGate(read(SERVER_SRC));
|
||||
});
|
||||
|
||||
test('the readiness flag flips to true only inside the startup migration run success handler', () => {
|
||||
assertReadyAfterRun(read(SERVER_SRC));
|
||||
});
|
||||
|
||||
test('with no DATABASE_URL configured the server reports ready immediately (no migration run to wait for)', () => {
|
||||
assertNoDatabaseUrlPath(read(SERVER_SRC));
|
||||
});
|
||||
|
||||
test('the app-readiness criterion is enforced in CI', () => {
|
||||
// Picked up by the root test command (root `scripts.test` glob).
|
||||
const scripts = JSON.parse(read('package.json')).scripts ?? {};
|
||||
assert.equal(
|
||||
scripts.test,
|
||||
`node --test "${ROOT_TEST_GLOB}"`,
|
||||
`root scripts.test must run the "${ROOT_TEST_GLOB}" glob so this suite runs with the rest`,
|
||||
);
|
||||
// And a dedicated CI job gates it on every PR.
|
||||
const workflow = read('.gitea/workflows/ci.yml');
|
||||
assert.ok(
|
||||
workflow.includes(`node --test tests/app-readiness.test.mjs`),
|
||||
`CI must run the app-readiness suite (job "${CI_JOB}") on every PR`,
|
||||
);
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Deterministic behavioral probes (no database, no Docker) — the committed
|
||||
// server is booted and probed over real HTTP
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/** True when this Node can execute the committed `.ts` server source (>= 22.6, type stripping). */
|
||||
const TS_STRIPPING = tsExecMode() !== null;
|
||||
|
||||
test('without DATABASE_URL the app reports ready immediately (deterministic probe)', { skip: !TS_STRIPPING }, async (t) => {
|
||||
// The no-migration path (local non-container dev, E00-S01-T06): no
|
||||
// DATABASE_URL is configured, so there is no startup migration run to wait
|
||||
// for and GET /health must answer 200 {"status":"ok"} right away — keeping
|
||||
// the E00-S02-T03 health endpoint working.
|
||||
const port = await reservePort();
|
||||
const { child, stdout, stderr } = bootServer(port); // DATABASE_URL stripped
|
||||
try {
|
||||
const response = await waitForAnswer(port, child, stderr);
|
||||
assert.equal(
|
||||
response.status,
|
||||
200,
|
||||
`GET /health without DATABASE_URL must answer 200 (got ${response.status}); server output: ${stdout().trim()} ${stderr().trim()}`,
|
||||
);
|
||||
assert.deepEqual(
|
||||
await response.json(),
|
||||
{ status: 'ok' },
|
||||
'without DATABASE_URL the app must report a healthy application ({"status":"ok"})',
|
||||
);
|
||||
} finally {
|
||||
await stopChild(child);
|
||||
}
|
||||
});
|
||||
|
||||
test('the app does not report ready while the migration run cannot complete (unreachable database)', { skip: !TS_STRIPPING }, async (t) => {
|
||||
// "app does not report ready before migrations complete": with a
|
||||
// DATABASE_URL that refuses connections, the startup migration run cannot
|
||||
// complete, so the app must stay up but answer HTTP 503
|
||||
// {"status":"not ready"} — never 200 — until the process is stopped.
|
||||
const port = await reservePort();
|
||||
const deadPort = await reservePort(); // reserved then released: nothing listens
|
||||
const { child, stdout, stderr } = bootServer(port, {
|
||||
DATABASE_URL: `postgres://eppp:eppp@127.0.0.1:${deadPort}/eppp`,
|
||||
});
|
||||
try {
|
||||
// Every answer over a ~2s window must be 503 not-ready, never 200.
|
||||
let answered = 0;
|
||||
for (let attempt = 0; attempt < 6; attempt += 1) {
|
||||
const response = await waitForAnswer(port, child, stderr);
|
||||
answered += 1;
|
||||
assert.equal(
|
||||
response.status,
|
||||
503,
|
||||
`GET /health must answer 503 while the migration run cannot complete (got ${response.status}); server output: ${stdout().trim()} ${stderr().trim()}`,
|
||||
);
|
||||
assert.deepEqual(
|
||||
await response.json(),
|
||||
{ status: 'not ready' },
|
||||
'the app must report not-ready ({"status":"not ready"}) while migrations are incomplete',
|
||||
);
|
||||
await delay(300);
|
||||
}
|
||||
assert.ok(answered >= 2, `the app must keep answering during the probe (answered ${answered} times)`);
|
||||
assert.equal(
|
||||
child.exitCode,
|
||||
null,
|
||||
'the app must stay up (not crash) while the migration run cannot complete — not-ready is a stable reported state',
|
||||
);
|
||||
assert.match(
|
||||
stdout() + stderr(),
|
||||
/startup migration run failed; app stays not-ready/,
|
||||
'the app must log that the startup migration run failed and it stays not-ready',
|
||||
);
|
||||
} finally {
|
||||
await stopChild(child);
|
||||
}
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Docker probe helpers (the real-stack probe skips cleanly without Docker)
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
function run(cmd, args, opts = {}) {
|
||||
return spawnSync(cmd, args, {
|
||||
encoding: 'utf8',
|
||||
timeout: 600_000,
|
||||
...opts,
|
||||
});
|
||||
}
|
||||
|
||||
/** True when the `docker` CLI with the Compose plugin is on PATH. */
|
||||
function dockerComposeAvailable() {
|
||||
try {
|
||||
return run('docker', ['compose', 'version'], { timeout: 15_000 }).status === 0;
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
/** True when a reachable Docker daemon exists. */
|
||||
function dockerDaemonAvailable() {
|
||||
try {
|
||||
return run('docker', ['info'], { timeout: 15_000 }).status === 0;
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
/** Parses `docker compose ps --format json` (JSON array or one object per line). */
|
||||
function parsePsJson(stdout) {
|
||||
const text = String(stdout).trim();
|
||||
if (!text) return [];
|
||||
try {
|
||||
const parsed = JSON.parse(text);
|
||||
return Array.isArray(parsed) ? parsed : [parsed];
|
||||
} catch {
|
||||
return text
|
||||
.split('\n')
|
||||
.map((line) => line.trim())
|
||||
.filter(Boolean)
|
||||
.map((line) => JSON.parse(line));
|
||||
}
|
||||
}
|
||||
|
||||
/** Tolerant field lookup across compose ps JSON shapes. */
|
||||
function field(container, ...names) {
|
||||
for (const name of names) {
|
||||
if (container[name] !== undefined) return container[name];
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
/**
|
||||
* Polls `docker compose ps` until the db container of the given project
|
||||
* reports healthy (or the deadline passes), so the probe never races a
|
||||
* still-booting database.
|
||||
*/
|
||||
function waitForDbHealthy(project, deadlineMs = 60_000) {
|
||||
const deadline = Date.now() + deadlineMs;
|
||||
let last = '';
|
||||
while (Date.now() < deadline) {
|
||||
const ps = run('docker', ['compose', '-p', project, 'ps', '--format', 'json'], {
|
||||
cwd: REPO_ROOT,
|
||||
timeout: 15_000,
|
||||
});
|
||||
if (ps.status === 0) {
|
||||
last = ps.stdout;
|
||||
const db = parsePsJson(ps.stdout).find((c) => field(c, 'Service', 'service') === 'db');
|
||||
if (db && /healthy/i.test(String(field(db, 'Health', 'health') ?? ''))) return;
|
||||
}
|
||||
run(process.execPath, ['-e', 'setTimeout(() => {}, 1000)']); // db still booting — retry
|
||||
}
|
||||
throw new Error(`the db container did not become healthy within ${deadlineMs}ms (last ps: "${last.trim()}")`);
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Real-stack probe — the issue's test plan: "start with pending migrations
|
||||
// and confirm readiness waits", against a real database
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
const DOCKER_COMPOSE = dockerComposeAvailable();
|
||||
const DOCKER_DAEMON = dockerDaemonAvailable();
|
||||
|
||||
// An isolated compose project + non-default host port so this probe never
|
||||
// collides with the other suites' projects/ports (ledger 55432, lock 55433,
|
||||
// diagnostic 55434, compose-config's default project/5432).
|
||||
const COMPOSE_PROJECT = 'eppp-readiness-probe';
|
||||
const POSTGRES_HOST_PORT = '55435';
|
||||
const DATABASE_URL = `postgres://eppp:eppp@127.0.0.1:${POSTGRES_HOST_PORT}/eppp`;
|
||||
|
||||
test('the app reports not-ready while the migration run is pending and ready only after it finishes (real stack)', { skip: !DOCKER_COMPOSE || !DOCKER_DAEMON || !TS_STRIPPING }, async (t) => {
|
||||
// The issue's test plan: "start with pending migrations and confirm
|
||||
// readiness waits". The probe starts the committed compose `db` service
|
||||
// (its own project + host port), creates the migration ledger, then holds
|
||||
// an ACCESS EXCLUSIVE lock on the ledger from a background psql session so
|
||||
// the app's startup migration run (ledger.ensure/applied) is genuinely
|
||||
// pending. It boots the committed server against that database and asserts
|
||||
// GET /health answers 503 {"status":"not ready"} while the run is blocked,
|
||||
// then — once the lock releases and the run completes — answers 200
|
||||
// {"status":"ok"} and logs the completed run. Rollback: drop the ledger
|
||||
// and `docker compose -p eppp-readiness-probe down -v` (issue rollback
|
||||
// note: revert the readiness gating logic).
|
||||
const composeEnv = { ...process.env, POSTGRES_PORT: POSTGRES_HOST_PORT };
|
||||
const compose = (args, opts = {}) =>
|
||||
run('docker', ['compose', '-p', COMPOSE_PROJECT, ...args], { cwd: REPO_ROOT, env: composeEnv, ...opts });
|
||||
const execPsql = (args, opts = {}) =>
|
||||
compose(['exec', '-T', 'db', 'psql', '-U', 'eppp', '-d', 'eppp', ...args], opts);
|
||||
|
||||
let app;
|
||||
let lockHolder;
|
||||
|
||||
try {
|
||||
const up = compose(['up', '-d', 'db'], { timeout: 180_000 });
|
||||
assert.equal(
|
||||
up.status,
|
||||
0,
|
||||
`"docker compose up -d db" must exit 0:\n${(up.stdout || '')}\n${(up.stderr || '')}`.trim(),
|
||||
);
|
||||
waitForDbHealthy(COMPOSE_PROJECT);
|
||||
|
||||
// Clean slate (also recovers from a previously interrupted run): drop any
|
||||
// leftover ledger, then create it so the app's migration run has a ledger
|
||||
// to read (the run's ledger.ensure() is then a no-op and its
|
||||
// ledger.applied() SELECT is what blocks behind the lock below).
|
||||
const clean = execPsql([
|
||||
'-v', 'ON_ERROR_STOP=1',
|
||||
'-c',
|
||||
'DROP TABLE IF EXISTS schema_migrations; ' +
|
||||
'CREATE TABLE schema_migrations (version text PRIMARY KEY, applied_at timestamptz NOT NULL DEFAULT now());',
|
||||
]);
|
||||
assert.equal(
|
||||
clean.status,
|
||||
0,
|
||||
`creating the migration ledger must succeed:\n${(clean.stdout || '')}\n${(clean.stderr || '')}`.trim(),
|
||||
);
|
||||
|
||||
// Hold an ACCESS EXCLUSIVE lock on the ledger in a background psql
|
||||
// session (BEGIN; LOCK; SELECT pg_sleep(30) keeps the transaction — and
|
||||
// the lock — open). The app's migration run blocks on this lock, so the
|
||||
// run is deterministically "pending" while we assert not-ready.
|
||||
lockHolder = spawn(
|
||||
'docker',
|
||||
[
|
||||
'compose', '-p', COMPOSE_PROJECT, 'exec', '-T', 'db',
|
||||
'psql', '-U', 'eppp', '-d', 'eppp', '-v', 'ON_ERROR_STOP=1',
|
||||
'-c', 'BEGIN; LOCK TABLE schema_migrations IN ACCESS EXCLUSIVE MODE; SELECT pg_sleep(30);',
|
||||
],
|
||||
{ cwd: REPO_ROOT, env: composeEnv, stdio: ['ignore', 'pipe', 'pipe'] },
|
||||
);
|
||||
let holderErr = '';
|
||||
lockHolder.stderr.on('data', (chunk) => {
|
||||
holderErr += String(chunk);
|
||||
});
|
||||
|
||||
// Wait until the AccessExclusiveLock is actually held (deterministic, so
|
||||
// the app boots into a genuinely blocked run).
|
||||
const lockDeadline = Date.now() + 15_000;
|
||||
let lockHeld = false;
|
||||
while (Date.now() < lockDeadline) {
|
||||
const held = execPsql([
|
||||
'-tA',
|
||||
'-c',
|
||||
"SELECT count(*) FROM pg_locks WHERE locktype = 'relation' AND relation = 'schema_migrations'::regclass AND mode = 'AccessExclusiveLock';",
|
||||
]);
|
||||
if (held.status === 0 && held.stdout.trim() === '1') {
|
||||
lockHeld = true;
|
||||
break;
|
||||
}
|
||||
run(process.execPath, ['-e', 'setTimeout(() => {}, 500)']);
|
||||
}
|
||||
assert.ok(lockHeld, `the probe must hold the ACCESS EXCLUSIVE lock on the ledger (holder stderr: "${holderErr.trim()}")`);
|
||||
|
||||
// Boot the committed server against the probe database.
|
||||
const port = await reservePort();
|
||||
app = bootServer(port, { DATABASE_URL });
|
||||
|
||||
// "app does not report ready before migrations complete": while the run
|
||||
// is blocked, every /health answer must be 503 {"status":"not ready"}.
|
||||
for (let attempt = 0; attempt < 5; attempt += 1) {
|
||||
const response = await waitForAnswer(port, app.child, app.stderr);
|
||||
assert.equal(
|
||||
response.status,
|
||||
503,
|
||||
`GET /health must answer 503 while the migration run is pending (got ${response.status}); server output: ${app.stdout().trim()} ${app.stderr().trim()}`,
|
||||
);
|
||||
assert.deepEqual(
|
||||
await response.json(),
|
||||
{ status: 'not ready' },
|
||||
'the app must report not-ready ({"status":"not ready"}) while migrations are pending',
|
||||
);
|
||||
await delay(300);
|
||||
}
|
||||
assert.equal(
|
||||
app.child.exitCode,
|
||||
null,
|
||||
'the app must stay up (not crash) while the migration run is pending',
|
||||
);
|
||||
|
||||
// Release the lock: the psql session ends, its transaction rolls back and
|
||||
// the ACCESS EXCLUSIVE lock is gone — the app's migration run completes.
|
||||
lockHolder.kill('SIGTERM');
|
||||
await Promise.race([once(lockHolder, 'exit'), delay(2_000)]);
|
||||
if (lockHolder.exitCode === null && lockHolder.signalCode === null) lockHolder.kill('SIGKILL');
|
||||
lockHolder = null;
|
||||
|
||||
// "readiness is reported only after migrations finish": once the run
|
||||
// completes, /health must answer 200 {"status":"ok"}.
|
||||
const readyDeadline = Date.now() + 15_000;
|
||||
let ready = null;
|
||||
while (Date.now() < readyDeadline) {
|
||||
const response = await waitForAnswer(port, app.child, app.stderr);
|
||||
if (response.status === 200) {
|
||||
ready = await response.json();
|
||||
break;
|
||||
}
|
||||
assert.equal(
|
||||
response.status,
|
||||
503,
|
||||
`GET /health must stay 503 only while the run is pending (got ${response.status})`,
|
||||
);
|
||||
await delay(200);
|
||||
}
|
||||
assert.ok(ready, `the app must report ready after the migration run completes (server output: ${app.stdout().trim()} ${app.stderr().trim()})`);
|
||||
assert.deepEqual(ready, { status: 'ok' }, 'the app must report a healthy application ({"status":"ok"}) after migrations finish');
|
||||
|
||||
// The readiness flip must be attributable to a completed migration run:
|
||||
// the app logs the completed run, and the migration ledger still exists.
|
||||
assert.match(
|
||||
app.stdout() + app.stderr(),
|
||||
/startup migration run complete \(applied 0, skipped 0\)/,
|
||||
`the app must log the completed startup migration run (got: ${app.stdout().trim()} ${app.stderr().trim()})`,
|
||||
);
|
||||
const ledgerClass = execPsql(['-tA', '-c', "SELECT to_regclass('public.schema_migrations');"]);
|
||||
assert.equal(
|
||||
ledgerClass.status,
|
||||
0,
|
||||
`ledger existence query must succeed:\n${(ledgerClass.stdout || '')}\n${(ledgerClass.stderr || '')}`.trim(),
|
||||
);
|
||||
assert.match(
|
||||
ledgerClass.stdout,
|
||||
/schema_migrations/,
|
||||
'the migration ledger must exist after the startup migration run',
|
||||
);
|
||||
} finally {
|
||||
if (app) await stopChild(app.child);
|
||||
if (lockHolder) {
|
||||
lockHolder.kill('SIGKILL');
|
||||
await Promise.race([once(lockHolder, 'exit'), delay(1_000)]);
|
||||
}
|
||||
// Rollback note from the issue: revert the readiness gating logic — here,
|
||||
// reset migration state and tear down the isolated project.
|
||||
try {
|
||||
execPsql(['-c', 'DROP TABLE IF EXISTS schema_migrations;']);
|
||||
} catch {
|
||||
// container may already be gone — the compose down below still cleans up
|
||||
}
|
||||
compose(['down', '-v'], { timeout: 120_000 });
|
||||
}
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Non-vacuous probes — the assertions above really do fail on violations
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
test('removing the 503 not-ready branch makes the readiness-gate criterion fail (mutation probe)', () => {
|
||||
const src = read(SERVER_SRC);
|
||||
const withoutNotReady = src.replace(/ } else \{\n sendJson\(res, 503, NOT_READY_PAYLOAD\);\n \}\n/, ' }\n');
|
||||
assert.notEqual(withoutNotReady, src, 'the mutation must actually remove the 503 branch');
|
||||
assert.throws(() => assertReadinessGate(withoutNotReady), /503/);
|
||||
});
|
||||
|
||||
test('answering 200 in the not-ready state makes the gate criterion fail (mutation probe)', () => {
|
||||
const src = read(SERVER_SRC);
|
||||
const notReady200 = src.replace('sendJson(res, 503, NOT_READY_PAYLOAD)', 'sendJson(res, 200, NOT_READY_PAYLOAD)');
|
||||
assert.notEqual(notReady200, src, 'the mutation must actually change the not-ready status code');
|
||||
assert.throws(() => assertReadinessGate(notReady200), /503/);
|
||||
});
|
||||
|
||||
test('replacing the gate with an unconditional healthy answer fails the gate criterion (mutation probe)', () => {
|
||||
const src = read(SERVER_SRC);
|
||||
const noGate = src.replace(/if \(migrationsComplete\) \{/, 'if (true) {');
|
||||
assert.notEqual(noGate, src, 'the mutation must actually bypass the readiness flag');
|
||||
assert.throws(() => assertReadinessGate(noGate), /migrationsComplete/);
|
||||
});
|
||||
|
||||
test('removing the not-ready payload fails the not-ready-report criterion (mutation probe)', () => {
|
||||
const src = read(SERVER_SRC);
|
||||
const healthyNotReady = src.replace("status: 'not ready'", "status: 'ok'");
|
||||
assert.notEqual(healthyNotReady, src, 'the mutation must actually change the not-ready payload');
|
||||
assert.throws(() => assertReadinessGate(healthyNotReady), /not ready/);
|
||||
});
|
||||
|
||||
test('flipping the readiness flag before the migration run fails the readiness-only-after criterion (mutation probe)', () => {
|
||||
const src = read(SERVER_SRC);
|
||||
const earlyFlip = src.replace(
|
||||
/const runner = new MigrationRunner\(pool, MIGRATIONS, new MigrationLedger\(pool\)\);\n runner/,
|
||||
'migrationsComplete = true;\n const runner = new MigrationRunner(pool, MIGRATIONS, new MigrationLedger(pool));\n runner',
|
||||
);
|
||||
assert.notEqual(earlyFlip, src, 'the mutation must actually flip the flag before the run');
|
||||
assert.throws(() => assertReadyAfterRun(earlyFlip), /only after/);
|
||||
});
|
||||
|
||||
test('dropping the migration run fails the readiness-after-run criterion (mutation probe)', () => {
|
||||
const src = read(SERVER_SRC);
|
||||
const noRun = src.replace(/ runner\n \.run\(\)\n/, '');
|
||||
assert.notEqual(noRun, src, 'the mutation must actually remove the runner.run() call');
|
||||
assert.throws(() => assertReadyAfterRun(noRun), /runner\.run\(\)/);
|
||||
});
|
||||
|
||||
test('importing pg directly instead of the driver boundary fails the isolation criterion (mutation probe)', () => {
|
||||
const src = read(SERVER_SRC);
|
||||
const directPg = src.replace(
|
||||
"import { Pool } from '@personal-blog/database-postgres';",
|
||||
"import { Pool } from 'pg';",
|
||||
);
|
||||
assert.notEqual(directPg, src, 'the mutation must actually replace the boundary import');
|
||||
assert.throws(() => assertDriverBoundaryImports(directPg), /must import the pool from the driver boundary/);
|
||||
});
|
||||
|
||||
test('removing the no-DATABASE_URL ready path fails the no-migration criterion (mutation probe)', () => {
|
||||
const src = read(SERVER_SRC);
|
||||
const withoutNoDb = src.replace(/if \(databaseUrl === undefined\) \{\n[\s\S]*?\} else \{/, '} else {');
|
||||
assert.notEqual(withoutNoDb, src, 'the mutation must actually remove the no-DATABASE_URL branch');
|
||||
assert.throws(() => assertNoDatabaseUrlPath(withoutNoDb), /must branch on a missing DATABASE_URL/);
|
||||
});
|
||||
|
||||
test('a placeholder entrypoint (no server at all) fails the readiness-source criterion (mutation probe)', () => {
|
||||
assert.throws(() => assertReadySource('export {};\n'), /driver boundary/);
|
||||
});
|
||||
Reference in New Issue
Block a user