diff --git a/tests/quarto-cmd.ts b/tests/quarto-cmd.ts index 62e7d784344..258d11631b9 100644 --- a/tests/quarto-cmd.ts +++ b/tests/quarto-cmd.ts @@ -183,60 +183,137 @@ export function assertTestBinary(bin: string) { checkedBinary = bin; } +// Result of attempting to stop a process tree. `cancelled` is true only when +// tree enumeration and termination are confirmed. Enumeration, command, or +// kill errors leave it false so callers do not continue while descendants +// may still be running. +interface KillOutcome { + cancelled: boolean; + detail: string; +} + // The launcher waits on Deno, so kill the process tree deepest first. -async function killProcessTree(pid: number) { +async function killProcessTree(pid: number): Promise { if (isWindows) { - let killed = false; try { - // taskkill reports failure through its exit code. + // `Deno.Command.output()` returns non-zero exit codes without throwing. + // Only a spawn failure throws, so check the exit code explicitly. const result = await new Deno.Command("taskkill", { args: ["/PID", String(pid), "/T", "/F"], stdout: "null", - stderr: "null", + stderr: "piped", }).output(); - killed = result.code === 0; - } catch { - // Fall through to a direct kill. - } - if (!killed) { - // Ensure child.output() can resolve even if the tree kill failed. + if (result.code === 0) { + return { + cancelled: true, + detail: `taskkill /T /F pid ${pid} exited 0`, + }; + } + // taskkill did not confirm the tree. Still attempt to unblock + // child.output() by killing the launcher directly, but this reaches + // only the launcher, not its descendants, so the outcome stays + // unconfirmed regardless of whether this fallback succeeds. + try { + Deno.kill(pid, "SIGKILL"); + } catch { + // already exited + } + const stderrText = new TextDecoder().decode(result.stderr).trim(); + return { + cancelled: false, + detail: `taskkill /T /F pid ${pid} exited ${result.code}: ${stderrText}`, + }; + } catch (e) { try { Deno.kill(pid, "SIGKILL"); } catch { // already exited } + return { + cancelled: false, + detail: `taskkill /T /F pid ${pid} failed to spawn: ${String(e)}`, + }; } - return; } + + // On Unix, enumerate descendants with pgrep -P. Exit 0 must have parseable + // output, while exit 1 means no children. Any other result leaves + // cancellation unconfirmed, but all collected pids are still signalled. + // + // This confirms only descendants found during the pgrep walk. A process + // spawned or reparented before termination may not be included. Avoiding + // this race would require process-group support. const pids: number[] = []; const stack = [pid]; + let confirmed = true; + let unconfirmedDetail = ""; while (stack.length > 0) { const current = stack.pop()!; pids.push(current); + + let spawnFailed = false; + let code = -1; + let stdoutText = ""; + let stderrText = ""; try { - // pgrep -P works on Linux and macOS/BSD. const result = new Deno.Command("pgrep", { args: ["-P", String(current)], stdout: "piped", - stderr: "null", + stderr: "piped", }).outputSync(); - const children = new TextDecoder() - .decode(result.stdout) - .split("\n") - .map((line) => parseInt(line.trim(), 10)) - .filter((child) => !isNaN(child)); - stack.push(...children); - } catch { - // pgrep unavailable; fall back to killing what we have + code = result.code; + stdoutText = new TextDecoder().decode(result.stdout); + stderrText = new TextDecoder().decode(result.stderr); + } catch (e) { + spawnFailed = true; + stderrText = String(e); + } + + if (spawnFailed) { + confirmed = false; + unconfirmedDetail = `pgrep -P ${current} failed to spawn: ${stderrText}`; + continue; + } + if (code === 1) { + // No children matched: a genuine leaf, nothing to push. + continue; + } + if (code !== 0) { + // Usage (2) or internal (3) error: enumeration unreliable. + confirmed = false; + unconfirmedDetail = `pgrep -P ${current} exited ${code}: ${stderrText.trim()}`; + continue; + } + const lines = stdoutText.split("\n").map((line) => line.trim()).filter(( + line, + ) => line.length > 0); + const children = lines.map((line) => parseInt(line, 10)); + if (children.some((child) => isNaN(child))) { + confirmed = false; + unconfirmedDetail = + `pgrep -P ${current} produced unparseable output: ${stdoutText.trim()}`; } + stack.push(...children.filter((child) => !isNaN(child))); } + + let signalled = 0; for (const target of pids.reverse()) { try { Deno.kill(target, "SIGKILL"); - } catch { - // already exited + signalled++; + } catch (e) { + if (!(e instanceof Deno.errors.NotFound)) { + confirmed = false; + unconfirmedDetail = `kill pid ${target} failed: ${String(e)}`; + } } } + return { + cancelled: confirmed, + detail: confirmed + ? `signalled ${signalled} pid(s) from root ${pid}` + : (unconfirmedDetail || `could not confirm process tree from root ${pid}`), + }; } export interface RunQuartoOptions { @@ -262,6 +339,30 @@ export interface RunQuartoResult { stderrTail?: string; } +// A render timeout, discriminated from a plain failure so a caller can tell +// whether the render is known to have been stopped. +export class QuartoTimeoutError extends Error { + constructor( + message: string, + readonly timeoutMs: number, + // True only when the render is known to have been stopped. Dev mode + // races a timer against an in-process render it cannot cancel, so it is + // always false there; binary mode sets it from the process-tree kill + // outcome. + readonly renderCancelled: boolean, + // Platform detail of the kill attempt, including the child pid. + // Undefined in dev mode, which attempts no kill. + readonly killDetail?: string, + ) { + super(message); + this.name = "QuartoTimeoutError"; + } +} + +export function isQuartoTimeoutError(e: unknown): e is QuartoTimeoutError { + return e instanceof QuartoTimeoutError; +} + // Dispatch to the in-process dev sources or the configured built binary. export async function runQuarto( args: string[], @@ -281,7 +382,17 @@ async function runDevQuarto( const timeoutMs = options.timeoutMs ?? kDefaultRenderTimeoutMs; let timer: ReturnType | undefined; const timeout = new Promise((_resolve, reject) => { - timer = setTimeout(reject, timeoutMs, `timed out after ${timeoutMs}ms`); + timer = setTimeout( + reject, + timeoutMs, + new QuartoTimeoutError( + `timed out after ${timeoutMs}ms`, + timeoutMs, + // Dev mode cannot stop an in-process render. A timeout rejects the + // caller, but the render continues. + false, + ), + ); }); try { await Promise.race([quarto(args, undefined, options.env), timeout]); @@ -290,9 +401,10 @@ async function runDevQuarto( clearTimeout(timer); } } - // quarto() either resolves or rejects: on CommandError or commandFailed() - // it calls exitWithCleanup(1), which Deno.exits the whole test process - // before this function could return a failure code anyway. + // quarto() calls exitWithCleanup(1), which Deno.exits the whole test + // process, only for a CommandError (Cliffy argument parsing) or + // commandFailed() (set only by the add/remove commands, never render). + // A render failure is neither, so it rejects out of quarto() instead. return { timedOut: false }; } @@ -335,16 +447,34 @@ async function runBinaryQuarto( }).spawn(); let timedOut = false; + let killPromise: Promise | undefined; const timer = setTimeout(() => { timedOut = true; - // child.output() resolves after the kill; avoid an unhandled rejection. - killProcessTree(child.pid).catch(() => {}); + killPromise = killProcessTree(child.pid); }, timeoutMs); // Drain both streams to avoid pipe-buffer deadlocks. const output = await child.output(); clearTimeout(timer); + // The direct child is known dead at this point (child.output() above + // already resolved), but the deno/pandoc descendants it spawned are only + // known dead if killOutcome.cancelled is true. + let killOutcome: KillOutcome = { + cancelled: false, + detail: "no timeout occurred", + }; + if (timedOut && killPromise !== undefined) { + try { + killOutcome = await killPromise; + } catch (e) { + killOutcome = { + cancelled: false, + detail: `killProcessTree rejected: ${String(e)}`, + }; + } + } + const stderrText = new TextDecoder().decode(output.stderr); const stderrTail = stderrText.split("\n").slice(-25).join("\n").trim(); const commandLine = `quarto ${args.join(" ")}`; @@ -359,11 +489,34 @@ async function runBinaryQuarto( }); } + // Warn non-throwing callers when cancellation is unconfirmed. Throwing + // callers receive the same detail in QuartoTimeoutError below. + if (timedOut && !killOutcome.cancelled && !throwOnFailure) { + console.error( + `[binary mode] process-tree kill UNCONFIRMED: ${commandLine} ` + + `(pid ${child.pid}) timed out after ${timeoutMs}ms; ${killOutcome.detail}. ` + + `Descendant quarto/pandoc processes may still be running and writing, ` + + `and the caller is continuing anyway, so later failures in this run ` + + `may be corruption from the orphan rather than genuine.`, + ); + } + if ((output.code !== 0 || timedOut) && throwOnFailure) { + if (timedOut) { + const base = + `${commandLine} (pid ${child.pid}) timed out after ${timeoutMs}ms`; + const message = killOutcome.cancelled + ? base + : `${base}; process-tree kill UNCONFIRMED: ${killOutcome.detail}`; + throw new QuartoTimeoutError( + message, + timeoutMs, + killOutcome.cancelled, + killOutcome.detail, + ); + } throw new Error( - timedOut - ? `${commandLine} timed out after ${timeoutMs}ms` - : `${commandLine} exited with code ${output.code}\nstderr (tail):\n${stderrTail}`, + `${commandLine} exited with code ${output.code}\nstderr (tail):\n${stderrTail}`, ); } diff --git a/tests/smoke/smoke-all.test.ts b/tests/smoke/smoke-all.test.ts index 2082afa844c..127e00fb506 100644 --- a/tests/smoke/smoke-all.test.ts +++ b/tests/smoke/smoke-all.test.ts @@ -5,7 +5,7 @@ */ import { expandGlobSync } from "../../src/core/deno/expand-glob.ts"; -import { testQuartoCmd, Verify } from "../test.ts"; +import { testQuartoCmd, unitTest, Verify } from "../test.ts"; import { initYamlIntelligenceResourcesFromFilesystem } from "../../src/core/schema/utils.ts"; import { initState, @@ -57,7 +57,7 @@ import { findProjectDir, findProjectOutputDir, outputForInput } from "../utils.t import { jupyterNotebookToMarkdown } from "../../src/command/convert/jupyter.ts"; import { basename, dirname, join, relative } from "../../src/deno_ral/path.ts"; import { WalkEntry } from "../../src/deno_ral/fs.ts"; -import { runQuarto } from "../quarto-cmd.ts"; +import { isQuartoTimeoutError, runQuarto } from "../quarto-cmd.ts"; import { safeExistsSync, safeRemoveSync } from "../../src/core/path.ts"; import { runningInCI } from "../../src/core/ci-info.ts"; @@ -162,20 +162,31 @@ interface QuartoInlineTestSpec { // postRenderCleanupFiles is a module-global list swept by EVERY render // teardown, so a registered entry only logs/removes at the teardowns where the // file actually exists (its owning document), not on every subsequent teardown. -const postRenderCleanupFiles: string[] = []; -function registerPostRenderCleanupFile(file: string): void { - postRenderCleanupFiles.push(file); +// Each entry records which input file registered it, so a sweep can be +// scoped to just that file's own entries (see postRenderCleanup below). +const postRenderCleanupFiles: Array<{ file: string; input: string }> = []; +function registerPostRenderCleanupFile(file: string, input: string): void { + postRenderCleanupFiles.push({ file, input }); } -const postRenderCleanup = () => { +// Without `onlyForInputs`, this sweeps every registered path. That is safe +// during normal teardown because the suite runs one file at a time and no +// other test has created a matching artifact. A skipped file never runs +// teardown, so the pass-2 branch scopes the sweep to that file's input. All +// project pre-renders have finished by then; an unscoped sweep could delete +// another project's artifact before verification. +const postRenderCleanup = (onlyForInputs?: Set) => { if (Deno.env.get("QUARTO_TEST_KEEP_OUTPUTS")) { return; } - for (const file of postRenderCleanupFiles) { - if (safeExistsSync(file)) { - console.log(`Cleaning up ${file} in ${Deno.cwd()}`); + for (const entry of postRenderCleanupFiles) { + if (onlyForInputs && !onlyForInputs.has(entry.input)) { + continue; + } + if (safeExistsSync(entry.file)) { + console.log(`Cleaning up ${entry.file} in ${Deno.cwd()}`); // recursive so a registered entry can be a directory (e.g. an embedded // notebook's `*_files` support dir), not just a single file - safeRemoveSync(file, { recursive: true }); + safeRemoveSync(entry.file, { recursive: true }); } } } @@ -243,7 +254,7 @@ function resolveTestSpecs( file = file.replace("${input_stem}", inputStem); } // file is registered for cleanup in testQuartoCmd teardown step - registerPostRenderCleanupFile(join(dirname(input), file)); + registerPostRenderCleanupFile(join(dirname(input), file), input); } } else if (key == "shouldError") { checkWarnings = false; @@ -405,6 +416,26 @@ const projectCleanupEntries: Map< string, Array<{ input: string; format: string; metadata: Record }> > = new Map(); + +function addProjectCleanupEntry( + projectPath: string, + input: string, + format: string, + // deno-lint-ignore no-explicit-any + metadata: Record, +) { + // editor-support-crossref creates no project cleanup entry; adding one + // would resolve its base format to editor and could delete an unrelated + // HTML support directory. + if (format === "editor-support-crossref") { + return; + } + if (!projectCleanupEntries.has(projectPath)) { + projectCleanupEntries.set(projectPath, []); + } + projectCleanupEntries.get(projectPath)!.push({ input, format, metadata }); +} + // The promise for each file's tests, grouped by the project it belongs to, so // we know when a given project's own files are all done (see above). const projectFilePromises: Map[]> = new Map(); @@ -412,9 +443,23 @@ const projectFilePromises: Map[]> = new Map(); // Create an array to hold all the promises for the tests of files let testFilesPromises = []; +const failedProjectPreRenders: Map = new Map(); + +interface DiscoveredFile { + input: string; + // deno-lint-ignore no-explicit-any + metadata: Record; + testSpecs: QuartoInlineTestSpec[]; + projectPath: string | undefined; +} + +// Pass 1: discover all files before registering tests. Because render-project +// is per-file metadata, discovery must finish before all files belonging to a +// failed project can be skipped. +const discovered: DiscoveredFile[] = []; for (const { path: fileName } of files) { const input = relative(Deno.cwd(), fileName); - + const metadata = input.endsWith("md") // qmd or md ? readYamlFromMarkdown(Deno.readTextFileSync(input)) : readYamlFromMarkdown(await jupyterNotebookToMarkdown(input, false)); @@ -444,24 +489,74 @@ for (const { path: fileName } of files) { const projectPath = findRootTestsProjectDir(input); if (projectPath) testedProjects.add(projectPath); - // Render project before testing individual document if required - if ( - (metadata["_quarto"] as any)?.["render-project"] && - projectPath && - !renderedProjects.has(projectPath) - ) { - // fail-loudly pre-render (throwOnFailure defaults to true); - // dispatches to the built binary when QUARTO_TEST_BIN is set - await runQuarto(["render", projectPath]); - renderedProjects.add(projectPath); + discovered.push({ input, metadata, testSpecs, projectPath }); +} + +// Pass 1.5: pre-render each project requested by any discovered file before +// registering tests. +const projectsNeedingPreRender = new Set(); +for (const entry of discovered) { + if ((entry.metadata["_quarto"] as any)?.["render-project"] && entry.projectPath) { + projectsNeedingPreRender.add(entry.projectPath); + } +} +for (const projectPath of projectsNeedingPreRender) { + try { + // Use the built binary when QUARTO_TEST_BIN is set. A failure skips this + // project's tests instead of aborting module evaluation. + await runQuarto(["render", projectPath]); + renderedProjects.add(projectPath); + } catch (err) { + // Abort if the timed-out render may still be running. It could keep + // writing while the remaining tests and cleanup run. + if (isQuartoTimeoutError(err) && !err.renderCancelled) { + console.error( + `[smoke-all] project pre-render for ${projectPath} timed out after ` + + `${err.timeoutMs}ms and the render could not be confirmed stopped` + + (err.killDetail ? ` (${err.killDetail})` : "") + + `; aborting rather than let a still-running render race the rest ` + + `of the suite.`, + ); + throw err; + } + failedProjectPreRenders.set( + projectPath, + err instanceof Error ? err : new Error(String(err)), + ); + } +} + +for (const [projectPath, error] of failedProjectPreRenders) { + unitTest(`smoke-all project pre-render failed: ${projectPath}`, async () => { + throw error; + }); +} + +// Pass 2 (registration). +for (const entry of discovered) { + const { input, metadata, testSpecs, projectPath } = entry; + + if (projectPath && failedProjectPreRenders.has(projectPath)) { + console.log( + `Skipping tests for ${input}: its project's pre-render failed (${projectPath})`, + ); + // Add the cleanup entries that this file's teardown would have added. + for (const testSpec of testSpecs) { + addProjectCleanupEntry(projectPath, input, testSpec.format, metadata); } + // Skipped files do not run teardown. Sweep only the custom cleanup paths + // registered for this input; an unscoped sweep could delete another + // project's artifact before its tests verify it. + postRenderCleanup(new Set([input])); + continue; + } const fileTestsPromise = new Promise(async (resolve, reject) => { try { // Create an array to hold all the promises for the testSpecs let testSpecPromises = []; - + for (const testSpec of testSpecs) { const { format, @@ -492,10 +587,7 @@ for (const { path: fileName } of files) { // files are cleaned once the whole project is done testing // (see projectCleanupEntries above). if (projectPath) { - if (!projectCleanupEntries.has(projectPath)) { - projectCleanupEntries.set(projectPath, []); - } - projectCleanupEntries.get(projectPath)!.push({ input, format, metadata }); + addProjectCleanupEntry(projectPath, input, format, metadata); } else { cleanoutput(input, format, undefined, undefined, metadata); } @@ -509,7 +601,7 @@ for (const { path: fileName } of files) { testSpecReject(error); } })); - + } // Wait for all the promises to resolve @@ -563,4 +655,4 @@ function findRootTestsProjectDir(input: string) { const RootTestsRegex = new RegExp(`${smokeAllRootDir}|${ffMatrixRootDir}`); return findProjectDir(input, RootTestsRegex); -} \ No newline at end of file +} diff --git a/tests/unit/smoke-all-prerender-isolation.test.ts b/tests/unit/smoke-all-prerender-isolation.test.ts new file mode 100644 index 00000000000..a7e163f548b --- /dev/null +++ b/tests/unit/smoke-all-prerender-isolation.test.ts @@ -0,0 +1,269 @@ +/* + * smoke-all-prerender-isolation.test.ts + * + * Copyright (C) 2026 Posit Software, PBC + */ +import { assert, assertEquals } from "testing/asserts"; +import { unitTest } from "../test.ts"; +import { dirname, fromFileUrl, join } from "../../src/deno_ral/path.ts"; +import { safeExistsSync } from "../../src/core/path.ts"; + +// tests/unit/ -> tests/ -> repo root +function resolveQuartoRoot(): string { + const envRoot = Deno.env.get("QUARTO_ROOT"); + if (envRoot && safeExistsSync(join(envRoot, "src", "import_map.json"))) { + return envRoot; + } + const here = dirname(fromFileUrl(import.meta.url)); + const derived = dirname(dirname(here)); + if (safeExistsSync(join(derived, "src", "import_map.json"))) { + return derived; + } + throw new Error( + "Could not resolve the Quarto repo root: QUARTO_ROOT is unset or invalid, " + + `and the derived root ${derived} has no src/import_map.json.`, + ); +} + +interface JUnitTestcase { + name: string; + failed: boolean; +} + +function parseJUnit( + xml: string, +): { total: number; failures: number; cases: JUnitTestcase[] } { + const suiteMatch = xml.match( + /]*\btests="(\d+)"[^>]*\bfailures="(\d+)"/, + ); + const cases: JUnitTestcase[] = []; + for ( + const m of xml.matchAll(/]*)>([\s\S]*?)<\/testcase>/g) + ) { + const nameMatch = m[1].match(/name="([^"]*)"/); + cases.push({ + name: nameMatch ? nameMatch[1] : "", + failed: / c.failed).length, + cases, + }; +} + +unitTest( + "smoke-all isolates a failing project pre-render instead of aborting the whole file", + async () => { + const quartoRoot = resolveQuartoRoot(); + const testsDir = join(quartoRoot, "tests"); + + const tmpRoot = Deno.makeTempDirSync({ + prefix: "quarto-smoke-all-prerender-", + }); + try { + const smokeAllDir = join(tmpRoot, "smoke-all"); + const projectDir = join(smokeAllDir, "_prerender-crash"); + Deno.mkdirSync(projectDir, { recursive: true }); + + Deno.writeTextFileSync( + join(projectDir, "_quarto.yml"), + "project:\n type: default\n render:\n" + + " - broken-b.qmd\n - broken-c.qmd\n", + ); + + // This file does not request a project pre-render. It sorts before files + // that do, but project.render excludes it. Its HTML test spec registers + // cleanup paths for a skipped file. + Deno.writeTextFileSync( + join(projectDir, "broken-a.qmd"), + [ + "---", + "title: a", + "_quarto:", + " tests:", + " html:", + " ensureHtmlElements:", + ' - ["body"]', + "---", + "", + "# a", + "", + ].join("\n"), + ); + // Pre-create outputs for broken-a.qmd. Because project.render excludes + // the file, their removal verifies cleanup for a skipped file. + Deno.writeTextFileSync( + join(projectDir, "broken-a.html"), + "sentinel\n", + ); + const supportDir = join(projectDir, "broken-a_files"); + Deno.mkdirSync(supportDir, { recursive: true }); + Deno.writeTextFileSync( + join(supportDir, "sentinel.txt"), + "sentinel\n", + ); + + // This file requests the project pre-render and makes it fail. + Deno.writeTextFileSync( + join(projectDir, "broken-b.qmd"), + [ + "---", + "title: b", + "_quarto:", + " render-project: true", + "filters:", + " - does-not-exist.lua", + "---", + "", + "# b", + "", + ].join("\n"), + ); + // A second file requests the same project pre-render, verifying that it + // runs once per project. + Deno.writeTextFileSync( + join(projectDir, "broken-c.qmd"), + [ + "---", + "title: c", + "_quarto:", + " render-project: true", + "---", + "", + "# c", + "", + ].join("\n"), + ); + + // This healthy control file sorts after the broken project and must + // still run. + Deno.writeTextFileSync( + join(smokeAllDir, "healthy-canary.qmd"), + ["---", "title: healthy", "---", "", "# healthy", ""].join("\n"), + ); + + assert( + safeExistsSync(join(projectDir, "broken-a.html")), + "sentinel broken-a.html must exist before the child run", + ); + assert( + safeExistsSync(join(supportDir, "sentinel.txt")), + "sentinel broken-a_files/sentinel.txt must exist before the child run", + ); + + const junitPath = join(tmpRoot, "report.xml"); + const importMapArg = `--importmap=${ + join(quartoRoot, "src", "import_map.json") + }`; + + const command = new Deno.Command(Deno.execPath(), { + args: [ + "test", + "--config", + join(testsDir, "test-conf.json"), + "--v8-flags=--enable-experimental-regexp-engine", + "--unstable-kv", + "--unstable-ffi", + "--no-lock", + "--allow-all", + importMapArg, + `--junit-path=${junitPath}`, + "smoke/smoke-all.test.ts", + "--", + join(projectDir, "broken-a.qmd"), + join(projectDir, "broken-b.qmd"), + join(projectDir, "broken-c.qmd"), + join(smokeAllDir, "healthy-canary.qmd"), + ], + cwd: testsDir, + stdout: "piped", + stderr: "piped", + }); + + const output = await command.output(); + const stdout = new TextDecoder().decode(output.stdout); + const stderr = new TextDecoder().decode(output.stderr); + const combined = stdout + stderr; + + const junitXml = Deno.readTextFileSync(junitPath); + const { total, failures, cases } = parseJUnit(junitXml); + + const healthyCase = cases.find((c) => c.name.includes("healthy-canary")); + assert( + healthyCase !== undefined, + `expected a testcase for healthy-canary.qmd; got: ${ + JSON.stringify(cases) + }\n---\n${combined}`, + ); + assertEquals(healthyCase!.failed, false); + + // Exactly one synthetic failure is reported for the broken project, + // even though two files request its pre-render. + const projectFailures = cases.filter((c) => + /_prerender-crash/.test(c.name) && c.failed + ); + assertEquals( + projectFailures.length, + 1, + `expected exactly one synthetic failure for the broken project; got: ${ + JSON.stringify(cases) + }`, + ); + + // No test is registered for any file in the broken project, including + // the file encountered before any file requests a project pre-render. + for (const fileName of ["broken-a.qmd", "broken-b.qmd", "broken-c.qmd"]) { + const registered = cases.some((c) => c.name.includes(fileName)); + assert( + !registered, + `${fileName} must not be registered as its own test; got: ${ + JSON.stringify(cases) + }`, + ); + } + + // Expect only the healthy control and one synthetic project failure. + assertEquals( + total, + 2, + `expected exactly 2 total testcases; got: ${JSON.stringify(cases)}`, + ); + assertEquals(failures, 1); + + // No module-evaluation abort: the failure is isolated to a synthetic + // test, not a crash of the whole smoke-all.test.ts file. + assert( + !combined.includes("Uncaught error from"), + `expected no module-evaluation abort; got:\n${combined}`, + ); + + assertEquals( + output.code, + 1, + "the run still reports failure overall (one synthetic test failed)", + ); + + // Cleanup: the entry synthesized for the skipped broken-a.qmd must + // still remove its sentinel outputs, and the in-place project root + // itself must survive. + assert( + !safeExistsSync(join(projectDir, "broken-a.html")), + "broken-a.html should have been removed by the synthesized cleanup entry", + ); + assert( + !safeExistsSync(supportDir), + "broken-a_files/ should have been removed by the synthesized cleanup entry", + ); + assert( + safeExistsSync(projectDir), + "the in-place project root itself must survive cleanup", + ); + } finally { + Deno.removeSync(tmpRoot, { recursive: true }); + } + }, +);