From 47459d5b4da14cd8ad96dac194c00390323dbb54 Mon Sep 17 00:00:00 2001 From: CarlesUIPath Date: Thu, 19 Nov 2026 14:13:08 +0000 Subject: [PATCH 01/13] feat(evalboard): Task Failure Analysis panel from the skill-doctor report The run page reads the skill-doctor report under /skill-doctor/ (contract v1): status.json (refetched after 60 s), the triage file it names once done, and /fixes.json plus its patches (cached for good). Names are checked against their patterns before any fetch; a schema_version other than 1, or unparsable JSON, is treated as absent, and readRunDoctor never throws into the page. When the report is done, analysis.md is cut at its first `## Findings` line (outside code fences): the AI summary keeps the top, and a native React panel replaces Findings and below (counts by layer and class, new vs recurring, diagnosis cost, Actions, a collapsible task table, Fixes with patch download through /api/file). Otherwise the page is unchanged, plus a one-line status for queued, running, failed, skipped and retracted reports. PR links render only for UiPath/skills pull URLs and build links only for dev.azure.com URLs; anything else is plain text. Internal edition and the skills source only; EVALBOARD_TASK_ANALYSIS=off turns it off. Co-Authored-By: Claude Opus 5.5 --- evalboard/README.md | 7 + .../__tests__/task-failure-analysis.test.tsx | 127 +++++ evalboard/app/runs/[id]/page.tsx | 29 +- .../app/runs/[id]/task-failure-analysis.tsx | 401 ++++++++++++++++ evalboard/lib/__tests__/doctor-blob.test.ts | 170 +++++++ evalboard/lib/__tests__/doctor.test.ts | 250 ++++++++++ .../fixes-1000001/example-grader-fix.patch | 9 + .../skill-doctor/fixes-1000001/fixes.json | 29 ++ .../fixtures/skill-doctor/status.json | 26 ++ .../fixtures/skill-doctor/triage-1000001.json | 54 +++ evalboard/lib/blob.ts | 71 +++ evalboard/lib/doctor.ts | 439 ++++++++++++++++++ evalboard/lib/runs.ts | 48 +- 13 files changed, 1657 insertions(+), 3 deletions(-) create mode 100644 evalboard/app/runs/[id]/__tests__/task-failure-analysis.test.tsx create mode 100644 evalboard/app/runs/[id]/task-failure-analysis.tsx create mode 100644 evalboard/lib/__tests__/doctor-blob.test.ts create mode 100644 evalboard/lib/__tests__/doctor.test.ts create mode 100644 evalboard/lib/__tests__/fixtures/skill-doctor/fixes-1000001/example-grader-fix.patch create mode 100644 evalboard/lib/__tests__/fixtures/skill-doctor/fixes-1000001/fixes.json create mode 100644 evalboard/lib/__tests__/fixtures/skill-doctor/status.json create mode 100644 evalboard/lib/__tests__/fixtures/skill-doctor/triage-1000001.json create mode 100644 evalboard/lib/doctor.ts diff --git a/evalboard/README.md b/evalboard/README.md index 7e1f1b85..12a99357 100644 --- a/evalboard/README.md +++ b/evalboard/README.md @@ -207,5 +207,12 @@ belongs to it. and then every 4 minutes (`lib/warm.ts`, started from `instrumentation.ts`), so the per-run cache is rebuilt by those requests rather than by a visitor. `EVALBOARD_WARM_INTERVAL_SECONDS` sets the interval; `0` turns it off. +- Task Failure Analysis (internal edition, skills source): the run page reads + the skill-doctor report under `/skill-doctor/` (`lib/doctor.ts`, + `ensureRunDoctor`). `status.json` is refetched after 60 s; the triage file it + names, `/fixes.json` and the patches are cached for good. When the + report is done, the panel replaces `analysis.md` from its first `## Findings` + line down; otherwise the AI summary is unchanged, with a status line. + `EVALBOARD_TASK_ANALYSIS=off` turns it off. - Pass rows render green (`bg-green-50 text-green-700`), failures render red (`bg-red-50 text-red-700`), on a white background. diff --git a/evalboard/app/runs/[id]/__tests__/task-failure-analysis.test.tsx b/evalboard/app/runs/[id]/__tests__/task-failure-analysis.test.tsx new file mode 100644 index 00000000..f1127670 --- /dev/null +++ b/evalboard/app/runs/[id]/__tests__/task-failure-analysis.test.tsx @@ -0,0 +1,127 @@ +import { readFileSync } from "node:fs"; +import path from "node:path"; +import { describe, expect, test } from "vitest"; +import { render, screen, within } from "@testing-library/react"; +import { + DoctorStatusLine, + TaskFailureAnalysis, +} from "../task-failure-analysis"; +import { + parseDoctorStatus, + parseFixes, + parseTriage, + type RunDoctor, +} from "@/lib/doctor"; + +const FIX = path.join(__dirname, "..", "..", "..", "..", "lib", "__tests__", "fixtures", "skill-doctor"); +const read = (p: string) => readFileSync(path.join(FIX, p), "utf-8"); +const RUN = "2099-01-03_04-23-30"; + +function status(patch: Record = {}) { + return parseDoctorStatus(JSON.stringify({ ...JSON.parse(read("status.json")), ...patch }))!; +} + +function doctor(): RunDoctor { + return { + status: status(), + triage: parseTriage(read("triage-1000001.json")), + fixes: parseFixes(read("fixes-1000001/fixes.json")), + }; +} + +describe("TaskFailureAnalysis", () => { + test("header: counts by layer and class, new vs recurring, cost, build link", () => { + render(); + const panel = screen.getByRole("region", { name: "Task Failure Analysis" }); + expect(panel).toHaveTextContent("3 failed, 2 diagnosed"); + expect(panel).toHaveTextContent("2 new, 1 recurring"); + expect(panel).toHaveTextContent("diagnosis $1.25"); + expect(panel).toHaveTextContent("grader 1"); + expect(panel).toHaveTextContent("failure 2"); + expect(screen.getByRole("link", { name: "build" })).toHaveAttribute( + "href", + "https://dev.azure.com/example-org/example-project/_build/results?buildId=1000001", + ); + expect(screen.getByRole("link", { name: "nightly" })).toBeInTheDocument(); + }); + + test("task rows link to the task page; only real skills PR URLs are links", () => { + render(); + const links = screen.getAllByRole("link").map((a) => a.getAttribute("href")); + expect(links).toContain(`/runs/${RUN}/example-t1`); + expect(links).toContain("https://github.com/UiPath/skills/pull/123"); + expect(links.some((h) => h?.startsWith("javascript:"))).toBe(false); + expect(links.some((h) => h?.includes("example.invalid"))).toBe(false); + // The rejected URLs are still visible, as text. + expect(screen.getByText("javascript:alert(1)").tagName).toBe("SPAN"); + expect( + screen.getByText("https://github.com.example.invalid/UiPath/skills/pull/124").tagName, + ).toBe("SPAN"); + expect(screen.getByText("Example Owner, Second Owner")).toBeInTheDocument(); + }); + + test("the source travels in the task link", () => { + render(); + const links = screen.getAllByRole("link").map((a) => a.getAttribute("href")); + expect(links).toContain(`/runs/${RUN}/example-t1?src=gha`); + }); + + test("actions and fixes", () => { + render(); + expect(screen.getByText(/1 task diagnosed as a grader problem/)).toBeInTheDocument(); + const patch = screen.getByRole("link", { name: "+3/-1" }); + const href = new URL(patch.getAttribute("href")!, "http://x"); + expect(href.pathname).toBe("/api/file"); + expect(href.searchParams.get("run")).toBe(RUN); + expect(href.searchParams.get("path")).toBe( + "skill-doctor/fixes-1000001/example-grader-fix.patch", + ); + const row = patch.closest("tr")!; + expect(within(row).getByText("replay pass · mutants 9/10 · review SEND")).toBeInTheDocument(); + expect(within(row).getByText("2")).toBeInTheDocument(); + expect(screen.getByText("needs-live-run")).toBeInTheDocument(); + }); + + test("untrusted build URLs are plain text", () => { + const d = doctor(); + d.status = status({ build_url: "javascript:alert(1)", trigger: "https://dev.azure.com.example.invalid/x" }); + render(); + expect(screen.queryByRole("link", { name: "build" })).toBeNull(); + expect(screen.queryByRole("link", { name: "nightly" })).toBeNull(); + expect(screen.getByText("build").tagName).toBe("SPAN"); + }); + + test("no fixes.json: no Fixes table", () => { + const d = { ...doctor(), fixes: null }; + render(); + expect(screen.queryByText("Fixes")).toBeNull(); + }); +}); + +describe("DoctorStatusLine", () => { + test("running: start time and build link", () => { + render(); + const line = screen.getByTestId("doctor-status-line"); + expect(line).toHaveTextContent("running since"); + expect(within(line).getByRole("link", { name: "build" })).toBeInTheDocument(); + }); + + test("queued", () => { + render(); + expect(screen.getByTestId("doctor-status-line")).toHaveTextContent("queued since"); + }); + + test("failed and skipped show the reason", () => { + const { rerender } = render( + , + ); + expect(screen.getByTestId("doctor-status-line")).toHaveTextContent("failed: diagnose timed out"); + rerender(); + expect(screen.getByTestId("doctor-status-line")).toHaveTextContent("skipped: ad-hoc run"); + }); + + test("retracted is a one-line note", () => { + render(); + expect(screen.getByTestId("doctor-status-line")).toHaveTextContent("retracted"); + }); +}); diff --git a/evalboard/app/runs/[id]/page.tsx b/evalboard/app/runs/[id]/page.tsx index 49401abf..34759d79 100644 --- a/evalboard/app/runs/[id]/page.tsx +++ b/evalboard/app/runs/[id]/page.tsx @@ -9,6 +9,7 @@ import { findMatureSourceRuns, readActivationScore, readRunAnalysis, + readRunDoctor, readRunMeta, readRunSummary, readRunTasks, @@ -24,6 +25,11 @@ import { sourceById } from "@/lib/sources"; import { scalarParam } from "@/app/_lib/source-param"; import { fmtRunTime } from "@/lib/format"; import { AnalysisPanel } from "./analysis-panel"; +import { + DoctorStatusLine, + TaskFailureAnalysis, +} from "./task-failure-analysis"; +import { planAnalysis, taskAnalysisEnabled } from "@/lib/doctor"; import { RefreshButton } from "./refresh-button"; import { RunView } from "./run-view"; import { RunIdentity } from "./run-identity"; @@ -43,7 +49,14 @@ export default async function RunPage({ // Which container this run lives in. Unknown/absent coerces to the skills // nightly, so every URL that predates the Scribe tab keeps resolving as-is. const source = sourceById(scalarParam((await searchParams).src)); - const [summary, tasks, activation, analysis, reviewIndex, meta] = + // The skill-doctor report: internal-only, skills nightly only, with an env + // kill switch (EVALBOARD_TASK_ANALYSIS=off). readRunDoctor never throws. + const doctorOn = taskAnalysisEnabled( + isInternal, + source.id, + process.env.EVALBOARD_TASK_ANALYSIS, + ); + const [summary, tasks, activation, analysis, reviewIndex, meta, doctor] = await Promise.all([ readRunSummary(id, source), readRunTasks(id, source), @@ -53,6 +66,7 @@ export default async function RunPage({ readRunAnalysis(id, source), readRunReviewIndex(id, source), readRunMeta(id, source), + doctorOn ? readRunDoctor(id, source) : Promise.resolve(null), ]); if (!summary || !tasks) notFound(); // Mature-skipped tasks weren't executed this run, so they have no detail page @@ -85,6 +99,9 @@ export default async function RunPage({ const tagCounts = reviewIndex ? tagCountsForRun(reviewIndex) : []; const reviewsByTask = reviewIndex ? indexByTask(reviewIndex) : undefined; const human = fmtRunTime(id); + // A done report replaces analysis.md's Findings and everything below it; + // otherwise the AI summary is unchanged, plus a status line. + const plan = planAnalysis(analysis, doctor); const series = isInternal ? seriesForRun(source.id, { series: meta?.series ?? null, @@ -156,7 +173,15 @@ export default async function RunPage({ )} - {analysis && } + {plan.summary && } + {plan.statusLine && } + {plan.panel && ( + + )} {/* The header above streams as soon as run.json is parsed, but the grid waits on the review index and the mature-source scan. With diff --git a/evalboard/app/runs/[id]/task-failure-analysis.tsx b/evalboard/app/runs/[id]/task-failure-analysis.tsx new file mode 100644 index 00000000..35a75c7b --- /dev/null +++ b/evalboard/app/runs/[id]/task-failure-analysis.tsx @@ -0,0 +1,401 @@ +import Link from "next/link"; +import type { ReactNode } from "react"; +import { withSource } from "@/app/_lib/source-param"; +import { fmtTimestamp, fmtUsd, humanizeTaskId } from "@/lib/format"; +import { + DOCTOR_DIR, + isAdoUrl, + isLinkableTaskId, + isSkillsPrUrl, + PATCH_ID_RE, + type DoctorFix, + type DoctorStatus, + type DoctorTask, + type RunDoctor, +} from "@/lib/doctor"; + +// The skill-doctor report for one run, drawn natively from its JSON (never +// the doctor's own page.html). Every string in it is plain text; the only +// links are to this board's task pages and to URLs of an exact known shape. + +function ExternalLink({ + href, + ok, + children, +}: { + href: string | null; + ok: (u: unknown) => boolean; + children?: ReactNode; +}) { + if (!href) return null; + if (!ok(href)) return {children ?? href}; + return ( + + {children ?? href} + + ); +} + +function prNumber(url: string): string { + const m = /\/pull\/(\d+)$/.exec(url); + return m ? `#${m[1]}` : url; +} + +function Counts({ + label, + counts, +}: { + label: string; + counts: Record; +}) { + const entries = Object.entries(counts).sort((a, b) => b[1] - a[1]); + if (entries.length === 0) return null; + return ( +
+ {label} + {entries.map(([k, n]) => ( + + {k} {n} + + ))} +
+ ); +} + +const STATE_TEXT: Record = { + queued: "queued", + running: "running", + done: "done", + failed: "failed", + skipped: "skipped", + retracted: "retracted", +}; + +/** One line under the AI summary when the full panel is not shown. */ +export function DoctorStatusLine({ status }: { status: DoctorStatus }) { + const build = status.buildUrl ? ( + <> + {" · "} + + build + + + ) : null; + let body: ReactNode; + switch (status.state) { + case "queued": + case "running": + body = ( + <> + {STATE_TEXT[status.state]} + {status.startedAt && + ` since ${fmtTimestamp(status.startedAt)}`} + {build} + + ); + break; + case "failed": + case "skipped": + body = ( + <> + {STATE_TEXT[status.state]} + {status.reason && `: ${status.reason}`} + {build} + + ); + break; + case "retracted": + body = <>the report for this run was retracted; + break; + case "done": + body = ( + <> + report done, not shown here + {build} + + ); + break; + } + return ( +

+ skill-doctor: {body} +

+ ); +} + +function TaskRow({ + t, + runId, + sourceId, +}: { + t: DoctorTask; + runId: string; + sourceId: string; +}) { + return ( + + + {isLinkableTaskId(t.taskId) ? ( + + {humanizeTaskId(t.taskId)} + + ) : ( + t.taskId + )} + {t.nightsSeen > 1 && ( + + {t.nightsSeen} nights + + )} + + {t.skill} + + {t.layer} + {t.cls && ( +
{t.cls}
+ )} + + {t.confidence ?? "—"} + {t.why ?? "—"} + {t.fix ?? "—"} + + {t.existingPrs.length === 0 + ? "—" + : t.existingPrs.map((p, i) => ( +
+ + {isSkillsPrUrl(p.url) ? prNumber(p.url) : p.url} + {" "} + + {p.state} + +
+ ))} + + + {t.owners.length ? t.owners.join(", ") : "—"} + + + ); +} + +function gateText(f: DoctorFix): string { + const g = f.gates; + const parts: string[] = []; + if (g.replay) parts.push(`replay ${g.replay}`); + if (g.mutants) parts.push(`mutants ${g.mutants.killed}/${g.mutants.n}`); + if (g.review) parts.push(`review ${g.review}`); + return parts.length ? parts.join(" · ") : "—"; +} + +function FixesTable({ + doctor, + runId, + sourceId, +}: { + doctor: RunDoctor; + runId: string; + sourceId: string; +}) { + const fixes = doctor.fixes; + const dir = doctor.status.fixesDir; + if (!fixes || !dir) return null; + return ( +
+

+ Fixes + + {fixes.fixMode && `fixMode ${fixes.fixMode} · `} + {fmtUsd(fixes.costUsd)} + {fixes.totalCapUsd != null && + ` of ${fmtUsd(fixes.totalCapUsd)} cap`} + +

+ {fixes.candidates.length === 0 ? ( +

No fix candidates.

+ ) : ( + + + + + + + + + + + + + {fixes.candidates.map((f) => ( + + + + + + + + + ))} + +
OutcomeTaskRowsPatchGatesCost
+ {f.outcome} + {f.title && ( +
+ {f.title} +
+ )} +
+ {humanizeTaskId(f.taskId)} + + {f.covers.length} + + {f.patchLines ? ( + PATCH_ID_RE.test(f.id) ? ( + + +{f.patchLines.added}/- + {f.patchLines.removed} + + ) : ( + `+${f.patchLines.added}/-${f.patchLines.removed}` + ) + ) : ( + "—" + )} + {gateText(f)} + {fmtUsd(f.costUsd)} +
+ )} +
+ ); +} + +export function TaskFailureAnalysis({ + doctor, + runId, + sourceId, +}: { + doctor: RunDoctor; + runId: string; + sourceId: string; +}) { + const { status, triage } = doctor; + if (!triage) return null; + const recurring = triage.tasks.filter((t) => t.nightsSeen > 1).length; + const fresh = triage.tasks.length - recurring; + return ( +
+
+
+

+ Task Failure Analysis +

+ + skill-doctor · {triage.summary.rowsFailed} failed,{" "} + {triage.summary.rowsDiagnosed} diagnosed · {fresh} new,{" "} + {recurring} recurring · diagnosis{" "} + {fmtUsd(status.costUsd ?? triage.costUsd)} + {status.buildUrl && ( + <> + {" · "} + + build + + + )} + {status.trigger && ( + <> + {" · "} + + nightly + + + )} + +
+ + + {triage.disclaimer && ( +

+ {triage.disclaimer} +

+ )} +
+ + {triage.actions.length > 0 && ( +
+

+ Actions +

+
    + {triage.actions.map((a, i) => ( +
  • + [{a.kind}]{" "} + {a.text} +
  • + ))} +
+
+ )} + + {/* Collapsed by default: a nightly can list 80+ rows, and the + task grid below should stay in reach. */} +
+ + Tasks ({triage.tasks.length}) + + + + + + + + + + + + + + + + {triage.tasks.map((t, i) => ( + + ))} + +
TaskSkillLayerConfidenceWhyFixExisting PRsOwners
+
+ + +
+ ); +} diff --git a/evalboard/lib/__tests__/doctor-blob.test.ts b/evalboard/lib/__tests__/doctor-blob.test.ts new file mode 100644 index 00000000..280e7e60 --- /dev/null +++ b/evalboard/lib/__tests__/doctor-blob.test.ts @@ -0,0 +1,170 @@ +// @vitest-environment node +import { promises as fs, readFileSync } from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { afterEach, beforeEach, describe, expect, test, vi } from "vitest"; + +const FIX = path.join(__dirname, "fixtures", "skill-doctor"); +const RUN = "2099-01-03_04-23-30"; + +// Blob name (relative to the container) -> body; a missing name is a 404. +const state = vi.hoisted(() => ({ + blobs: {} as Record, + downloads: [] as string[], + failWith: null as null | { statusCode: number }, +})); + +vi.mock("@azure/storage-blob", () => ({ + BlobServiceClient: class { + getContainerClient() { + return { + getBlobClient: (name: string) => ({ + downloadToFile: async (dest: string) => { + state.downloads.push(name); + if (state.failWith) + throw Object.assign(new Error("blob failed"), state.failWith); + if (!(name in state.blobs)) + throw Object.assign(new Error("404"), { statusCode: 404 }); + await fs.writeFile(dest, state.blobs[name]); + }, + }), + }; + } + }, +})); +vi.mock("@azure/identity", () => ({ DefaultAzureCredential: class {} })); + +const fixture = (p: string) => readFileSync(path.join(FIX, p), "utf-8"); +const STATUS = fixture("status.json"); + +function seedBlobs(status: string = STATUS) { + const d = `${RUN}/skill-doctor`; + state.blobs = { + [`${d}/status.json`]: status, + [`${d}/triage-1000001.json`]: fixture("triage-1000001.json"), + [`${d}/fixes-1000001/fixes.json`]: fixture("fixes-1000001/fixes.json"), + [`${d}/fixes-1000001/example-grader-fix.patch`]: fixture( + "fixes-1000001/example-grader-fix.patch", + ), + }; +} + +function statusWith(patch: Record): string { + return JSON.stringify({ ...JSON.parse(STATUS), ...patch }); +} + +let root: string; + +beforeEach(async () => { + vi.resetModules(); + vi.stubEnv("EVALBOARD_LOCAL_RUNS_DIR", ""); + root = await fs.mkdtemp(path.join(os.tmpdir(), "doctor-blob-")); + vi.stubEnv("EVALBOARD_RUNS_DIR", root); + state.downloads = []; + state.failWith = null; + seedBlobs(); +}); + +afterEach(async () => { + vi.unstubAllEnvs(); + await fs.rm(root, { recursive: true, force: true }); +}); + +async function ensure() { + const { ensureRunDoctor } = await import("../blob"); + return ensureRunDoctor("runs", RUN, root); +} + +describe("ensureRunDoctor", () => { + test("a done report: status, triage, fixes.json and the one patch", async () => { + await ensure(); + expect(state.downloads.sort()).toEqual( + [ + `${RUN}/skill-doctor/status.json`, + `${RUN}/skill-doctor/triage-1000001.json`, + `${RUN}/skill-doctor/fixes-1000001/fixes.json`, + `${RUN}/skill-doctor/fixes-1000001/example-grader-fix.patch`, + ].sort(), + ); + }); + + test("no report: one 404, swallowed", async () => { + state.blobs = {}; + await expect(ensure()).resolves.toBeUndefined(); + expect(state.downloads).toEqual([`${RUN}/skill-doctor/status.json`]); + }); + + test("running: the triage is not fetched", async () => { + seedBlobs(statusWith({ state: "running", fixes_dir: null })); + await ensure(); + expect(state.downloads).toEqual([`${RUN}/skill-doctor/status.json`]); + }); + + test("a bad triage_file name is never fetched", async () => { + seedBlobs(statusWith({ triage_file: "../run.json", fixes_dir: null })); + await ensure(); + expect(state.downloads).toEqual([`${RUN}/skill-doctor/status.json`]); + }); + + test("schema_version 2: nothing beyond status.json", async () => { + seedBlobs(statusWith({ schema_version: 2 })); + await ensure(); + expect(state.downloads).toEqual([`${RUN}/skill-doctor/status.json`]); + }); + + test("status.json is refetched after a minute; the triage never", async () => { + await ensure(); + const st = path.join(root, RUN, "skill-doctor", "status.json"); + const old = new Date(Date.now() - 120_000); + await fs.utimes(st, old, old); + state.downloads = []; + await ensure(); + expect(state.downloads).toEqual([`${RUN}/skill-doctor/status.json`]); + }); +}); + +describe("readRunDoctor", () => { + test("parses the cached report", async () => { + const { readRunDoctor } = await import("../runs"); + const d = await readRunDoctor(RUN); + expect(d?.status.state).toBe("done"); + expect(d?.triage?.tasks).toHaveLength(3); + expect(d?.fixes?.candidates).toHaveLength(2); + }); + + test("never throws: a storage error is no report", async () => { + state.failWith = { statusCode: 500 }; + const { readRunDoctor } = await import("../runs"); + const err = vi.spyOn(console, "error").mockImplementation(() => {}); + await expect(readRunDoctor(RUN)).resolves.toBeNull(); + err.mockRestore(); + }); + + test("an unparsable status is no report", async () => { + seedBlobs("{oops"); + const { readRunDoctor } = await import("../runs"); + await expect(readRunDoctor(RUN)).resolves.toBeNull(); + }); + + test("the patch is served by /api/file's path check from the cache", async () => { + const { readRunDoctor, resolveSafePath } = await import("../runs"); + await readRunDoctor(RUN); + state.downloads = []; + const abs = await resolveSafePath( + RUN, + "skill-doctor/fixes-1000001/example-grader-fix.patch", + ); + expect(abs).toBe( + path.join( + await fs.realpath(root), + RUN, + "skill-doctor", + "fixes-1000001", + "example-grader-fix.patch", + ), + ); + // No listing or run.json fetch for a doctor path. + expect(state.downloads).toEqual([]); + expect(await resolveSafePath(RUN, "skill-doctor/../../x")).toBeNull(); + }); +}); diff --git a/evalboard/lib/__tests__/doctor.test.ts b/evalboard/lib/__tests__/doctor.test.ts new file mode 100644 index 00000000..47c19e97 --- /dev/null +++ b/evalboard/lib/__tests__/doctor.test.ts @@ -0,0 +1,250 @@ +import { readFileSync } from "node:fs"; +import path from "node:path"; +import { describe, expect, test } from "vitest"; +import { + isAdoUrl, + isSkillsPrUrl, + parseDoctorStatus, + parseFixes, + parseTriage, + patchIds, + planAnalysis, + splitAnalysis, + taskAnalysisEnabled, + type RunDoctor, +} from "../doctor"; + +const FIX = path.join(__dirname, "fixtures", "skill-doctor"); +const read = (p: string) => readFileSync(path.join(FIX, p), "utf-8"); +const STATUS = read("status.json"); +const TRIAGE = read("triage-1000001.json"); +const FIXES = read("fixes-1000001/fixes.json"); + +function statusWith(patch: Record): string { + return JSON.stringify({ ...JSON.parse(STATUS), ...patch }); +} + +function doctor(patch: Record = {}): RunDoctor { + const status = parseDoctorStatus(statusWith(patch))!; + return { + status, + triage: status.state === "done" ? parseTriage(TRIAGE) : null, + fixes: parseFixes(FIXES), + }; +} + +const MD = [ + "# Analysis", + "", + "## TL;DR", + "Top part.", + "", + "## Findings", + "Bottom part.", + "", + "## Quick Wins", + "More.", + "", +].join("\n"); + +describe("splitAnalysis", () => { + test("no heading: no split", () => { + expect(splitAnalysis("# A\n\n## TL;DR\ntext\n")).toBeNull(); + }); + + test("one heading: cuts at the Findings line", () => { + const s = splitAnalysis(MD)!; + expect(s.top).toBe("# Analysis\n\n## TL;DR\nTop part.\n\n"); + expect(s.findings.startsWith("## Findings\nBottom part.")).toBe(true); + expect(s.top + s.findings).toBe(MD); + }); + + test("cuts at the first of two", () => { + const s = splitAnalysis("a\n## Findings\nb\n## Findings\nc")!; + expect(s.top).toBe("a\n"); + expect(s.findings).toBe("## Findings\nb\n## Findings\nc"); + }); + + test("a Findings line inside a fenced code block is not a split point", () => { + const md = "x\n```md\n## Findings\n```\ny\n~~~\n## Findings\n~~~\nz\n"; + expect(splitAnalysis(md)).toBeNull(); + const s = splitAnalysis(`${md}## Findings\nreal\n`)!; + expect(s.findings).toBe("## Findings\nreal\n"); + }); + + test("### Findings, a suffix or leading spaces are not split points", () => { + expect(splitAnalysis("### Findings\n")).toBeNull(); + expect(splitAnalysis("## Findings (3)\n")).toBeNull(); + expect(splitAnalysis(" ## Findings\n")).toBeNull(); + expect(splitAnalysis("## Findings \n")).toBeNull(); + }); + + test("CRLF line endings still split", () => { + const s = splitAnalysis("a\r\n## Findings\r\nb")!; + expect(s.top).toBe("a\r\n"); + }); +}); + +describe("parseDoctorStatus", () => { + test("reads the fixture", () => { + const s = parseDoctorStatus(STATUS)!; + expect(s.state).toBe("done"); + expect(s.triageFile).toBe("triage-1000001.json"); + expect(s.fixesDir).toBe("fixes-1000001"); + expect(s.fixes?.counts).toEqual({ "pr-ready": 1, "needs-live-run": 1 }); + expect(s.costUsd).toBe(1.25); + }); + + test("schema_version 2 is absent", () => { + expect(parseDoctorStatus(statusWith({ schema_version: 2 }))).toBeNull(); + }); + + test("unparsable JSON, a non-object or an unknown state is absent", () => { + expect(parseDoctorStatus("{not json")).toBeNull(); + expect(parseDoctorStatus("[1]")).toBeNull(); + expect(parseDoctorStatus(null)).toBeNull(); + expect(parseDoctorStatus(statusWith({ state: "exploded" }))).toBeNull(); + }); + + test("a bad triage_file name is treated as absent", () => { + for (const bad of [ + "../run.json", + "triage-1/../../x.json", + "summary.json", + "triage-.json", + "triage-1.json.bak", + ]) { + expect(parseDoctorStatus(statusWith({ triage_file: bad }))!.triageFile).toBeNull(); + } + }); + + test("a bad fixes_dir is treated as absent; the old shape has none", () => { + expect(parseDoctorStatus(statusWith({ fixes_dir: "../x" }))!.fixesDir).toBeNull(); + expect(parseDoctorStatus(statusWith({ fixes_dir: "fixes" }))!.fixesDir).toBe("fixes"); + const old = JSON.parse(STATUS); + delete old.fixes_dir; + delete old.fixes; + delete old.trigger; + const s = parseDoctorStatus(JSON.stringify(old))!; + expect(s.fixesDir).toBeNull(); + expect(s.fixes).toBeNull(); + expect(s.trigger).toBeNull(); + }); +}); + +describe("parseTriage / parseFixes", () => { + test("reads the fixture", () => { + const t = parseTriage(TRIAGE)!; + expect(t.tasks.map((x) => x.taskId)).toEqual(["example-e1", "example-t1", "example-t2"]); + expect(t.summary.byLayer).toEqual({ env: 1, grader: 1, skill: 1 }); + expect(t.tasks[1].nightsSeen).toBe(3); + expect(t.tasks[1].existingPrs).toHaveLength(3); + expect(t.actions).toHaveLength(2); + }); + + test("schema_version 2 or a broken triage is absent", () => { + expect(parseTriage(JSON.stringify({ ...JSON.parse(TRIAGE), schema_version: 2 }))).toBeNull(); + expect(parseTriage("{")).toBeNull(); + expect(parseTriage(JSON.stringify({ schema_version: 1 }))).toBeNull(); + }); + + test("fixes: candidates, gates and patch ids", () => { + const f = parseFixes(FIXES)!; + expect(f.candidates.map((c) => c.outcome)).toEqual(["pr-ready", "needs-live-run"]); + expect(f.candidates[0].gates).toEqual({ replay: "pass", mutants: { n: 10, killed: 9 }, review: "SEND" }); + expect(patchIds(f)).toEqual(["example-grader-fix"]); + }); + + test("a candidate id that is not a safe file name gets no patch", () => { + const doc = JSON.parse(FIXES); + doc.candidates[0].id = "../../run"; + expect(patchIds(parseFixes(JSON.stringify(doc)))).toEqual([]); + }); +}); + +describe("link validation", () => { + test("skills PR links", () => { + expect(isSkillsPrUrl("https://github.com/UiPath/skills/pull/123")).toBe(true); + for (const bad of [ + "javascript:alert(1)", + "http://github.com/UiPath/skills/pull/123", + "https://github.com/UiPath/skills/pull/123/files", + "https://github.com/UiPath/skills/pull/123?x=1", + "https://github.com.example.invalid/UiPath/skills/pull/123", + "https://github.com/UiPath/skills-fork/pull/123", + "https://github.com/Other/skills/pull/123", + "https://evil.example/https://github.com/UiPath/skills/pull/1", + " https://github.com/UiPath/skills/pull/123", + ]) { + expect(isSkillsPrUrl(bad)).toBe(false); + } + }); + + test("build links", () => { + expect(isAdoUrl("https://dev.azure.com/example-org/example-project/_build/results?buildId=1")).toBe(true); + for (const bad of [ + "javascript:alert(1)", + "http://dev.azure.com/example-org/x", + "https://dev.azure.com.example.invalid/x", + "https://dev.azure.com@example.invalid/x", + "https://example.invalid/dev.azure.com/x", + "https://dev.azure.com/x\"onmouseover=\"y", + null, + ]) { + expect(isAdoUrl(bad)).toBe(false); + } + }); +}); + +describe("taskAnalysisEnabled (gating + kill switch)", () => { + test("internal + skills + switch not off", () => { + expect(taskAnalysisEnabled(true, "skills", undefined)).toBe(true); + expect(taskAnalysisEnabled(true, "skills", "")).toBe(true); + expect(taskAnalysisEnabled(true, "skills", "on")).toBe(true); + }); + test("off, OSS or another source", () => { + expect(taskAnalysisEnabled(true, "skills", "off")).toBe(false); + expect(taskAnalysisEnabled(false, "skills", undefined)).toBe(false); + expect(taskAnalysisEnabled(true, "scribe", undefined)).toBe(false); + expect(taskAnalysisEnabled(true, "gha", undefined)).toBe(false); + }); +}); + +describe("planAnalysis (states)", () => { + test("absent: today's view, no status line", () => { + expect(planAnalysis(MD, null)).toEqual({ summary: MD, panel: null, statusLine: null }); + expect(planAnalysis(null, null)).toEqual({ summary: null, panel: null, statusLine: null }); + }); + + test("done with a split: summary keeps the top, the panel replaces the rest", () => { + const d = doctor(); + const p = planAnalysis(MD, d); + expect(p.summary).toBe("# Analysis\n\n## TL;DR\nTop part.\n\n"); + expect(p.summary).not.toContain("Findings"); + expect(p.panel).toBe(d); + expect(p.statusLine).toBeNull(); + }); + + test("done without a split: today's view plus a status line", () => { + const md = "# A\n\n## TL;DR\nonly\n"; + const p = planAnalysis(md, doctor()); + expect(p).toMatchObject({ summary: md, panel: null }); + expect(p.statusLine?.state).toBe("done"); + expect(planAnalysis(null, doctor()).panel).toBeNull(); + }); + + test("done but the triage did not load: no panel", () => { + const d = { ...doctor(), triage: null }; + expect(planAnalysis(MD, d)).toMatchObject({ summary: MD, panel: null }); + }); + + test.each(["queued", "running", "failed", "skipped", "retracted"])( + "%s: the full analysis (Findings included) plus a status line", + (state) => { + const p = planAnalysis(MD, doctor({ state, reason: "why" })); + expect(p.summary).toBe(MD); + expect(p.panel).toBeNull(); + expect(p.statusLine?.state).toBe(state); + }, + ); +}); diff --git a/evalboard/lib/__tests__/fixtures/skill-doctor/fixes-1000001/example-grader-fix.patch b/evalboard/lib/__tests__/fixtures/skill-doctor/fixes-1000001/example-grader-fix.patch new file mode 100644 index 00000000..730e99e9 --- /dev/null +++ b/evalboard/lib/__tests__/fixtures/skill-doctor/fixes-1000001/example-grader-fix.patch @@ -0,0 +1,9 @@ +diff --git a/tests/tasks/example/example.yaml b/tests/tasks/example/example.yaml +--- a/tests/tasks/example/example.yaml ++++ b/tests/tasks/example/example.yaml +@@ -1,2 +1,4 @@ +-expected: literal ++expected: documented ++# example ++# example + other: value diff --git a/evalboard/lib/__tests__/fixtures/skill-doctor/fixes-1000001/fixes.json b/evalboard/lib/__tests__/fixtures/skill-doctor/fixes-1000001/fixes.json new file mode 100644 index 00000000..b81f04c1 --- /dev/null +++ b/evalboard/lib/__tests__/fixtures/skill-doctor/fixes-1000001/fixes.json @@ -0,0 +1,29 @@ +{ + "schema_version": 1, + "run_id": "2099-01-03_04-23-30", + "fix_mode": "dryrun", + "base_sha": "0a1b2c3d4e", + "run_commit": "0a1b2c3d", + "pr_data": "gh", + "considered": 2, + "cost_usd": 2.5, + "total_cap_usd": 60, + "generated_at": "2099-01-03T09:20:00Z", + "skipped": [], + "candidates": [ + { + "id": "example-grader-fix", "task_id": "example-t1", "covers": ["example-t1", "example-t3"], "layer": "grader", + "outcome": "pr-ready", "cost_usd": 1.5, "patch_lines": {"added": 3, "removed": 1}, "title": "fix(example): match the documented output", + "problem": "", "files": ["tests/tasks/example/example.yaml"], "overlap": [], "open_prs_on_files": [], + "gates": {"replay": {"status": "pass", "table": "", "evidence": ""}, "mutants": {"n": 10, "killed": 9, "survivors": 1, "pre_existing": 0}, + "flip_judge": null, "genuine": null, "review": "SEND", "holes": null, "history": []} + }, + { + "id": "example-skill-fix", "task_id": "example-t2", "covers": ["example-t2"], "layer": "skill", + "outcome": "needs-live-run", "cost_usd": 1.0, "patch_lines": null, "title": "", + "problem": "blocked-env", "files": [], "overlap": [], "open_prs_on_files": [], + "gates": {"replay": {"status": "blocked-env", "table": "", "evidence": ""}, "mutants": null, + "flip_judge": null, "genuine": null, "review": null, "holes": null, "history": []} + } + ] +} diff --git a/evalboard/lib/__tests__/fixtures/skill-doctor/status.json b/evalboard/lib/__tests__/fixtures/skill-doctor/status.json new file mode 100644 index 00000000..316a3341 --- /dev/null +++ b/evalboard/lib/__tests__/fixtures/skill-doctor/status.json @@ -0,0 +1,26 @@ +{ + "schema_version": 1, + "state": "done", + "reason": "", + "run_id": "2099-01-03_04-23-30", + "harness": "claude-code", + "model": "example-model", + "adhoc": false, + "triage_only": false, + "history_base_run": "2099-01-02_04-25-52", + "run_json_etag": "\"0x8D000000000000A\"", + "started_at": "2099-01-03T09:00:00Z", + "finished_at": "2099-01-03T09:12:00Z", + "updated_at": "2099-01-03T09:20:00Z", + "doctor_version": "0.3.0a0", + "build_id": "1000001", + "build_url": "https://dev.azure.com/example-org/example-project/_build/results?buildId=1000001", + "cost_usd": 1.25, + "cost_complete": true, + "rows_failed": 3, + "rows_diagnosed": 2, + "triage_file": "triage-1000001.json", + "trigger": "https://dev.azure.com/example-org/example-project/_build/results?buildId=1000000", + "fixes_dir": "fixes-1000001", + "fixes": {"fix_mode": "dryrun", "cost_usd": 2.5, "total_cap_usd": 60, "counts": {"pr-ready": 1, "needs-live-run": 1}} +} diff --git a/evalboard/lib/__tests__/fixtures/skill-doctor/triage-1000001.json b/evalboard/lib/__tests__/fixtures/skill-doctor/triage-1000001.json new file mode 100644 index 00000000..bc5d96e3 --- /dev/null +++ b/evalboard/lib/__tests__/fixtures/skill-doctor/triage-1000001.json @@ -0,0 +1,54 @@ +{ + "schema_version": 1, + "disclaimer": "Probable causes, not verified fixes. Existing PRs are context; only a doctor PR is a proposed fix.", + "run": { + "run_id": "2099-01-03_04-23-30", "harness": "claude-code", "model": "example-model", "skills_git_commit": "0a1b2c3d", + "cli_version": "1.0.0", "tasks_run": 10, "rows_executed": 10, "adhoc": false, "history_base_run": "2099-01-02_04-25-52", + "run_json_etag": "\"0x8D000000000000A\"", "build_id": "1000001", + "build_url": "https://dev.azure.com/example-org/example-project/_build/results?buildId=1000001", + "doctor_version": "0.3.0a0", "generated_at": "2099-01-03T09:12:00Z", "mode": "diagnosed", + "mode_reason": "example", "cost_usd": 1.25 + }, + "summary": { + "rows_failed": 3, "rows_diagnosed": 2, + "by_class": {"env": 1, "failure": 2}, + "by_layer": {"env": 1, "grader": 1, "skill": 1}, + "vs_history": {"new": 2, "persisting": 1, "fixed": 0, "fixed_tasks": [], "base_run": "2099-01-02_04-25-52"}, + "env_groups": [{"cause": "connection example-conn", "count": 1, "task_ids": ["example-e1"]}] + }, + "tasks": [ + { + "task_id": "example-e1", "skill": "uipath-example", "class": "env", "family": "preflight", "state": "new", + "nights_seen": 1, "owners": ["Example Owner"], "run_status": "ERROR", "criterion": null, "error": "pre-run check failed", + "contested": null, "env_group": "connection example-conn", "diagnosed": false, "layer": "env", "confidence": null, + "why": "example-conn: no enabled connection", "evidence": null, "solvable": "env-fix", + "fix": "Reauthorize the connection, then re-run.", "related": null, "passed_before": null, + "existing_prs": null, "doctor_pr": null + }, + { + "task_id": "example-t1", "skill": "uipath-example", "class": "failure", "family": "grader", "state": "known", + "nights_seen": 3, "owners": ["Example Owner", "Second Owner"], "run_status": "FAILURE", "criterion": "file_contains", + "error": null, "contested": null, "env_group": null, "diagnosed": true, "layer": "grader", "confidence": "high", + "why": "The grader expects a literal the task prompt never asks for.", "evidence": "example evidence", + "solvable": "grader-fix", "fix": "Match the documented output instead of the literal.", "related": null, + "passed_before": null, + "existing_prs": [ + {"url": "https://github.com/UiPath/skills/pull/123", "title": "fix: example grader", "state": "open"}, + {"url": "https://github.com.example.invalid/UiPath/skills/pull/124", "title": "look-alike host", "state": "open"}, + {"url": "javascript:alert(1)", "title": "script url", "state": "open"} + ], + "doctor_pr": null + }, + { + "task_id": "example-t2", "skill": "uipath-example-two", "class": "failure", "family": "skill", "state": "new", + "nights_seen": 1, "owners": [], "run_status": "FAILURE", "criterion": "run_command", "error": null, + "contested": null, "env_group": null, "diagnosed": true, "layer": "skill", "confidence": "medium", + "why": "The skill documents a removed flag.", "evidence": null, "solvable": "skill-fix", + "fix": "Document the current flag.", "related": null, "passed_before": null, "existing_prs": [], "doctor_pr": null + } + ], + "actions": [ + {"kind": "env", "count": 1, "text": "Environment: 1 task stopped at the pre-run check. Reauthorize the connection, then re-run.", "task_ids": ["example-e1"]}, + {"kind": "layer", "count": 1, "text": "uipath-example: 1 task diagnosed as a grader problem.", "task_ids": ["example-t1"]} + ] +} diff --git a/evalboard/lib/blob.ts b/evalboard/lib/blob.ts index ba2a17d8..70c9d7ef 100644 --- a/evalboard/lib/blob.ts +++ b/evalboard/lib/blob.ts @@ -3,6 +3,7 @@ import path from "node:path"; import { randomBytes } from "node:crypto"; import type { BlobServiceClient, ContainerClient } from "@azure/storage-blob"; import { DEFAULT_VARIANT_ID, isValidVariantId } from "./variants"; +import { DOCTOR_DIR, parseDoctorStatus, parseFixes, patchIds } from "./doctor"; const ACCOUNT = "coderevaltests"; @@ -342,6 +343,76 @@ export async function ensureRunReviewIndex( }); } +// skill-doctor report sidecars under `/skill-doctor/` (see lib/doctor.ts). +// status.json changes while the doctor runs, so a copy older than a minute is +// fetched again. The triage file it names (once `done`), fixes.json and the +// patches are immutable and cached forever. No listing: every name comes from +// status.json or fixes.json and is checked against its pattern first. 404s are +// swallowed as for analysis.md. +export const DOCTOR_STATUS_REFRESH_MS = 60_000; + +async function downloadOptional( + container: string, + blobName: string, + destRoot: string, + refreshAfterMs?: number, +): Promise { + try { + await downloadBlob(container, blobName, destRoot, refreshAfterMs); + } catch (err) { + if (!isNotFound(err)) throw err; + } +} + +export async function ensureRunDoctor( + container: string, + runId: string, + destRoot: string, +): Promise { + assertValidId(runId, "runId"); + if (LOCAL_RUNS_DIR) return; + return dedupe(`doctor:${container}:${runId}`, async () => { + const base = `${runId}/${DOCTOR_DIR}`; + await downloadOptional( + container, + `${base}/status.json`, + destRoot, + DOCTOR_STATUS_REFRESH_MS, + ); + const status = parseDoctorStatus( + await fs + .readFile(path.join(destRoot, base, "status.json"), "utf-8") + .catch(() => null), + ); + if (!status) return; + const ops: Promise[] = []; + if (status.state === "done" && status.triageFile) { + ops.push( + downloadOptional(container, `${base}/${status.triageFile}`, destRoot), + ); + } + if (status.fixesDir) { + const dir = `${base}/${status.fixesDir}`; + ops.push( + (async () => { + await downloadOptional(container, `${dir}/fixes.json`, destRoot); + const fixes = parseFixes( + await fs + .readFile(path.join(destRoot, dir, "fixes.json"), "utf-8") + .catch(() => null), + ); + await Promise.all( + patchIds(fixes).map((id) => + downloadOptional(container, `${dir}/${id}.patch`, destRoot), + ), + ); + })(), + ); + } + await Promise.all(ops); + }); +} + // There is deliberately no whole-run fetch. `ensureRunDir` used to pull EVERY // blob under the run prefix with no concurrency cap (~10k blobs / ~400 MB for a // nightly). Per-task download, below, is the supported shape. diff --git a/evalboard/lib/doctor.ts b/evalboard/lib/doctor.ts new file mode 100644 index 00000000..8426da02 --- /dev/null +++ b/evalboard/lib/doctor.ts @@ -0,0 +1,439 @@ +// The skill-doctor report of one nightly run (contract v1), read from +// `/skill-doctor/`: +// - status.json: pointer + lock, changes while the doctor runs; +// - triage-.json: the per-task failure analysis, immutable; +// - /fixes.json: proposed fixes, optional, create-only. +// +// Pure data + string helpers, no Node builtins: the parsed shapes are passed to +// the run page's components, and the tests import this module directly. The +// types are written by hand against the doctor's status_v1 / triage_v1 schemas +// and keep only the fields the board shows. Anything that does not parse, or a +// schema_version other than 1, is treated as absent: the page must never fail +// because of a doctor file. + +export const DOCTOR_DIR = "skill-doctor"; +export const TRIAGE_FILE_RE = /^triage-[A-Za-z0-9._-]+\.json$/; +export const FIXES_DIR_RE = /^fixes(-[0-9]+)?$/; +// Candidate ids name `.patch` inside the fixes dir. +export const PATCH_ID_RE = /^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$/; + +export const DOCTOR_STATES = [ + "queued", + "running", + "done", + "failed", + "skipped", + "retracted", +] as const; +export type DoctorState = (typeof DOCTOR_STATES)[number]; + +export interface DoctorFixesDigest { + fixMode: string | null; + costUsd: number | null; + totalCapUsd: number | null; + counts: Record; +} + +export interface DoctorStatus { + state: DoctorState; + reason: string; + startedAt: string | null; + finishedAt: string | null; + buildUrl: string | null; + costUsd: number | null; + rowsFailed: number | null; + rowsDiagnosed: number | null; + triageFile: string | null; + fixesDir: string | null; + fixes: DoctorFixesDigest | null; + trigger: string | null; +} + +export interface DoctorPr { + url: string; + title: string; + state: string; +} + +export interface DoctorTask { + taskId: string; + skill: string; + layer: string; + cls: string; + confidence: string | null; + contested: string[] | null; + criterion: string | null; + diagnosed: boolean; + why: string | null; + fix: string | null; + solvable: string | null; + state: string | null; + nightsSeen: number; + owners: string[]; + existingPrs: DoctorPr[]; + doctorPr: string | null; + evidence: string | null; + envGroup: string | null; +} + +export interface DoctorAction { + kind: string; + count: number; + text: string; + taskIds: string[]; +} + +export interface DoctorTriage { + disclaimer: string; + costUsd: number | null; + buildUrl: string | null; + summary: { + rowsFailed: number; + rowsDiagnosed: number; + byClass: Record; + byLayer: Record; + vsHistory: { + new: number; + persisting: number; + fixed: number; + baseRun: string | null; + } | null; + }; + tasks: DoctorTask[]; + actions: DoctorAction[]; +} + +export interface DoctorGates { + replay: string | null; + mutants: { n: number; killed: number } | null; + review: string | null; +} + +export interface DoctorFix { + id: string; + taskId: string; + covers: string[]; + outcome: string; + costUsd: number | null; + patchLines: { added: number; removed: number } | null; + title: string; + problem: string; + gates: DoctorGates; +} + +export interface DoctorFixes { + fixMode: string | null; + costUsd: number | null; + totalCapUsd: number | null; + candidates: DoctorFix[]; +} + +export interface RunDoctor { + status: DoctorStatus; + // Set only when the status is `done` and its triage file parsed. + triage: DoctorTriage | null; + fixes: DoctorFixes | null; +} + +// ---------- coercion helpers ---------- + +type Obj = Record; + +function isObj(v: unknown): v is Obj { + return typeof v === "object" && v !== null && !Array.isArray(v); +} +function str(v: unknown): string | null { + return typeof v === "string" ? v : null; +} +function num(v: unknown): number | null { + return typeof v === "number" && Number.isFinite(v) ? v : null; +} +function int(v: unknown, dflt = 0): number { + return typeof v === "number" && Number.isInteger(v) && v >= 0 ? v : dflt; +} +function strList(v: unknown): string[] { + return Array.isArray(v) ? v.filter((x): x is string => typeof x === "string") : []; +} +function countMap(v: unknown): Record { + const out: Record = {}; + if (!isObj(v)) return out; + for (const [k, n] of Object.entries(v)) { + if (typeof n === "number" && Number.isInteger(n) && n >= 0) out[k] = n; + } + return out; +} + +function parseJson(raw: string | null | undefined): unknown { + if (raw == null) return null; + try { + return JSON.parse(raw); + } catch { + return null; + } +} + +// ---------- parsers ---------- + +export function parseDoctorStatus(raw: string | null | undefined): DoctorStatus | null { + const j = parseJson(raw); + if (!isObj(j) || j.schema_version !== 1) return null; + const state = j.state; + if (typeof state !== "string" || !(DOCTOR_STATES as readonly string[]).includes(state)) + return null; + const triageFile = str(j.triage_file); + const fixesDir = str(j.fixes_dir); + let fixes: DoctorFixesDigest | null = null; + if (isObj(j.fixes)) { + fixes = { + fixMode: str(j.fixes.fix_mode), + costUsd: num(j.fixes.cost_usd), + totalCapUsd: num(j.fixes.total_cap_usd), + counts: countMap(j.fixes.counts), + }; + } + return { + state: state as DoctorState, + reason: str(j.reason) ?? "", + startedAt: str(j.started_at), + finishedAt: str(j.finished_at), + buildUrl: str(j.build_url), + costUsd: num(j.cost_usd), + rowsFailed: typeof j.rows_failed === "number" ? int(j.rows_failed) : null, + rowsDiagnosed: typeof j.rows_diagnosed === "number" ? int(j.rows_diagnosed) : null, + // A name outside the pattern is treated as absent: it lands in a path. + triageFile: triageFile && TRIAGE_FILE_RE.test(triageFile) ? triageFile : null, + fixesDir: fixesDir && FIXES_DIR_RE.test(fixesDir) ? fixesDir : null, + fixes, + trigger: str(j.trigger), + }; +} + +function parsePr(v: unknown): DoctorPr | null { + if (!isObj(v)) return null; + const url = str(v.url); + if (!url) return null; + return { url, title: str(v.title) ?? "", state: str(v.state) ?? "unknown" }; +} + +function parseTask(v: unknown): DoctorTask | null { + if (!isObj(v)) return null; + const taskId = str(v.task_id); + if (!taskId) return null; + return { + taskId, + skill: str(v.skill) ?? "", + layer: str(v.layer) ?? "undiagnosed", + cls: str(v.class) ?? "", + confidence: str(v.confidence), + contested: Array.isArray(v.contested) ? strList(v.contested) : null, + criterion: str(v.criterion), + diagnosed: v.diagnosed === true, + why: str(v.why), + fix: str(v.fix), + solvable: str(v.solvable), + state: str(v.state), + nightsSeen: int(v.nights_seen, 1), + owners: strList(v.owners), + existingPrs: Array.isArray(v.existing_prs) + ? v.existing_prs.map(parsePr).filter((p): p is DoctorPr => p !== null) + : [], + doctorPr: str(v.doctor_pr), + evidence: str(v.evidence), + envGroup: str(v.env_group), + }; +} + +export function parseTriage(raw: string | null | undefined): DoctorTriage | null { + const j = parseJson(raw); + if (!isObj(j) || j.schema_version !== 1) return null; + if (!isObj(j.summary) || !Array.isArray(j.tasks)) return null; + const s = j.summary; + const run = isObj(j.run) ? j.run : {}; + const vh = isObj(s.vs_history) ? s.vs_history : null; + return { + disclaimer: str(j.disclaimer) ?? "", + costUsd: num(run.cost_usd), + buildUrl: str(run.build_url), + summary: { + rowsFailed: int(s.rows_failed), + rowsDiagnosed: int(s.rows_diagnosed), + byClass: countMap(s.by_class), + byLayer: countMap(s.by_layer), + vsHistory: vh + ? { + new: int(vh.new), + persisting: int(vh.persisting), + fixed: int(vh.fixed), + baseRun: str(vh.base_run), + } + : null, + }, + tasks: j.tasks.map(parseTask).filter((t): t is DoctorTask => t !== null), + actions: Array.isArray(j.actions) + ? j.actions.flatMap((a): DoctorAction[] => { + if (!isObj(a) || typeof a.text !== "string") return []; + return [ + { + kind: str(a.kind) ?? "", + count: int(a.count), + text: a.text, + taskIds: strList(a.task_ids), + }, + ]; + }) + : [], + }; +} + +function parseGates(v: unknown): DoctorGates { + if (!isObj(v)) return { replay: null, mutants: null, review: null }; + const replay = isObj(v.replay) ? str(v.replay.status) : null; + const m = isObj(v.mutants) ? v.mutants : null; + const mutants = + m && typeof m.n === "number" && typeof m.killed === "number" + ? { n: int(m.n), killed: int(m.killed) } + : null; + return { replay, mutants, review: str(v.review) }; +} + +export function parseFixes(raw: string | null | undefined): DoctorFixes | null { + const j = parseJson(raw); + if (!isObj(j) || !Array.isArray(j.candidates)) return null; + // fixes.json carries its own schema_version 1; an unknown one is absent. + if (j.schema_version !== undefined && j.schema_version !== 1) return null; + return { + fixMode: str(j.fix_mode), + costUsd: num(j.cost_usd), + totalCapUsd: num(j.total_cap_usd), + candidates: j.candidates.flatMap((c): DoctorFix[] => { + if (!isObj(c)) return []; + const id = str(c.id); + const taskId = str(c.task_id); + if (!id || !taskId) return []; + const pl = isObj(c.patch_lines) ? c.patch_lines : null; + return [ + { + id, + taskId, + covers: strList(c.covers), + outcome: str(c.outcome) ?? "unknown", + costUsd: num(c.cost_usd), + patchLines: pl + ? { added: int(pl.added), removed: int(pl.removed) } + : null, + title: str(c.title) ?? "", + problem: str(c.problem) ?? "", + gates: parseGates(c.gates), + }, + ]; + }), + }; +} + +/** Patch ids whose `.patch` the board fetches and links. */ +export function patchIds(fixes: DoctorFixes | null): string[] { + if (!fixes) return []; + return fixes.candidates + .filter((c) => c.patchLines !== null && PATCH_ID_RE.test(c.id)) + .map((c) => c.id); +} + +// ---------- links ---------- + +// The report's text comes from a model and from the run's own logs, so a URL +// in it is shown as a link only when it has exactly the expected shape. +const SKILLS_PR_RE = /^https:\/\/github\.com\/UiPath\/skills\/pull\/\d+$/; +const ADO_RE = /^https:\/\/dev\.azure\.com\/[^\s"'<>]*$/; + +export function isSkillsPrUrl(u: unknown): u is string { + return typeof u === "string" && SKILLS_PR_RE.test(u); +} + +export function isAdoUrl(u: unknown): u is string { + return typeof u === "string" && ADO_RE.test(u); +} + +// Same shape as blob.ts's task-id check (kept here so this module stays free +// of Node builtins): a task link is only built from an id the task page accepts. +const TASK_ID_RE = /^[\w.-]+(\/[\w.-]+)*$/; + +export function isLinkableTaskId(id: string): boolean { + return ( + id.length > 0 && + id.length < 256 && + TASK_ID_RE.test(id) && + !id.split("/").some((s) => s === "." || s === "..") + ); +} + +// ---------- analysis.md split ---------- + +/** + * Cut analysis.md at its first line that is exactly `## Findings`, outside a + * fenced code block. Null when there is no such line. + */ +export function splitAnalysis( + md: string, +): { top: string; findings: string } | null { + const lines = md.split("\n"); + let fence: string | null = null; + let offset = 0; + for (const raw of lines) { + const line = raw.endsWith("\r") ? raw.slice(0, -1) : raw; + const f = /^ {0,3}(`{3,}|~{3,})/.exec(line); + if (f) { + const mark = f[1]; + if (fence === null) fence = mark; + else if (mark[0] === fence[0] && mark.length >= fence.length && line.trim() === mark) + fence = null; + } else if (fence === null && line === "## Findings") { + return { top: md.slice(0, offset), findings: md.slice(offset) }; + } + offset += raw.length + 1; + } + return null; +} + +// ---------- gating + page plan ---------- + +/** The panel is internal-only, for the skills nightly, with an env kill switch. */ +export function taskAnalysisEnabled( + internal: boolean, + sourceId: string, + env: string | undefined, +): boolean { + return internal && sourceId === "skills" && env !== "off"; +} + +export interface AnalysisPlan { + // Markdown for the AI summary panel (null: no panel). + summary: string | null; + // Render the Task Failure Analysis panel in place of the cut-off part. + panel: RunDoctor | null; + // One-line doctor status under the summary (null: none). + statusLine: DoctorStatus | null; +} + +/** + * What the run page shows. The panel replaces Findings and below only when + * the report is done and analysis.md has a split point; every other case keeps + * today's view, plus a status line when a status exists. + */ +export function planAnalysis( + analysis: string | null, + doctor: RunDoctor | null, +): AnalysisPlan { + if (!doctor) return { summary: analysis, panel: null, statusLine: null }; + const { status } = doctor; + if (status.state === "done" && doctor.triage && analysis) { + const split = splitAnalysis(analysis); + if (split) { + return { + summary: split.top.trim() ? split.top : null, + panel: doctor, + statusLine: null, + }; + } + } + return { summary: analysis, panel: null, statusLine: status }; +} diff --git a/evalboard/lib/runs.ts b/evalboard/lib/runs.ts index feb7d7c8..eebd195d 100644 --- a/evalboard/lib/runs.ts +++ b/evalboard/lib/runs.ts @@ -6,6 +6,7 @@ import { LOCAL_RUNS_DIR, ensureActivationSummary, ensureRunAnalysis, + ensureRunDoctor, ensureRunExperiment, ensureRunMeta, ensureRunSummary, @@ -19,6 +20,13 @@ import { DEFAULT_VARIANT_ID, isValidVariantId } from "./variants"; import { DEFAULT_SOURCE, runsDirFor, type Source } from "./sources"; import { DELIVERABLE_KINDS, DELIVERABLE_NAMES } from "./artifact-kinds"; import { messageCostUsd, normalizeModel } from "./pricing"; +import { + DOCTOR_DIR, + parseDoctorStatus, + parseFixes, + parseTriage, + type RunDoctor, +} from "./doctor"; // Resolution order: // 1. EVALBOARD_LOCAL_RUNS_DIR — local mode, points at a coder_eval runs dir @@ -2680,6 +2688,41 @@ export async function readRunAnalysis( .catch(() => null); } +// ---------- skill-doctor report (skill-doctor/ sidecars) ---------- + +// The run's skill-doctor report, or null when there is none, it does not parse, +// or its schema_version is not 1. Never throws: a doctor file problem must not +// take the run page down, so a fetch error degrades to "no report". +export async function readRunDoctor( + runId: string, + source: Source = DEFAULT_SOURCE, +): Promise { + const dir = runsDirFor(RUNS_DIR, source); + try { + await ensureRunDoctor(source.container, runId, dir); + } catch (err) { + console.error(`ensureRunDoctor failed for run ${runId}:`, err); + } + const base = path.join(dir, runId, DOCTOR_DIR); + const read = (name: string) => + fs.readFile(path.join(base, name), "utf-8").catch(() => null); + try { + const status = parseDoctorStatus(await read("status.json")); + if (!status) return null; + const triage = + status.state === "done" && status.triageFile + ? parseTriage(await read(status.triageFile)) + : null; + const fixes = status.fixesDir + ? parseFixes(await read(path.join(status.fixesDir, "fixes.json"))) + : null; + return { status, triage, fixes }; + } catch (err) { + console.error(`readRunDoctor failed for run ${runId}:`, err); + return null; + } +} + // ---------- run metadata (meta.json sidecar) ---------- // Optional, user-supplied at upload time. Null on pipeline runs and any run @@ -2952,7 +2995,10 @@ export async function resolveSafePath( // (`activation/default//…`), so parts[0..1] are not (variant, task) // there; that case falls through to the run-summary fetch as it always did. const parts = relPath.split("/"); - if (parts[0] !== "activation" && isValidVariantId(parts[0]) && parts[1]) { + if (parts[0] === DOCTOR_DIR) { + // skill-doctor sidecars (the fix patches) are fetched by name by + // readRunDoctor when the run page renders, never by a listing here. + } else if (parts[0] !== "activation" && isValidVariantId(parts[0]) && parts[1]) { if (!isValidId(parts[1])) return null; await ensureTaskDir(source.container, runId, parts[1], dir, parts[0]); } else { From 8cfa175bbc29def47befcd1f03af77bf2e28691c Mon Sep 17 00:00:00 2001 From: CarlesUIPath Date: Thu, 19 Nov 2026 14:26:19 +0000 Subject: [PATCH 02/13] fix(evalboard): review round for the Task Failure Analysis panel - readRunDoctor is the one place that decides which doctor files a status implies and parses each once; ensureRunDoctorFile only downloads. - Parsed types carry only what the page renders; the id validators move to a Node-free lib/ids.ts shared by blob.ts and doctor.ts. - One predicate (patchIds, capped at 10, ids checked) picks the patches to fetch and to link; /api/file fetches a patch on demand and serves only /.patch from skill-doctor/. - The header counts the per-task triage state (new / changed / known) and shows vs_history only when there is a base run. - A done report gets the panel even without a `## Findings` split or an analysis.md. - parseFixes requires schema_version 1; a file that does not parse is logged by name; the kill switch accepts "off" in any case. - Tests for the task-link check, the text fallbacks, the fixes.json version gate, the patch cap and the cold-cache patch fetch. Co-Authored-By: Claude Opus 5.5 --- evalboard/README.md | 11 +- .../__tests__/task-failure-analysis.test.tsx | 43 +++- .../app/runs/[id]/task-failure-analysis.tsx | 85 ++++---- evalboard/lib/__tests__/doctor-blob.test.ts | 102 ++++++--- evalboard/lib/__tests__/doctor.test.ts | 71 +++++- evalboard/lib/blob.ts | 112 +++------- evalboard/lib/doctor.ts | 206 ++++++++---------- evalboard/lib/ids.ts | 30 +++ evalboard/lib/runs.ts | 83 +++++-- 9 files changed, 443 insertions(+), 300 deletions(-) create mode 100644 evalboard/lib/ids.ts diff --git a/evalboard/README.md b/evalboard/README.md index 12a99357..bb9e5f27 100644 --- a/evalboard/README.md +++ b/evalboard/README.md @@ -209,10 +209,11 @@ belongs to it. `EVALBOARD_WARM_INTERVAL_SECONDS` sets the interval; `0` turns it off. - Task Failure Analysis (internal edition, skills source): the run page reads the skill-doctor report under `/skill-doctor/` (`lib/doctor.ts`, - `ensureRunDoctor`). `status.json` is refetched after 60 s; the triage file it - names, `/fixes.json` and the patches are cached for good. When the - report is done, the panel replaces `analysis.md` from its first `## Findings` - line down; otherwise the AI summary is unchanged, with a status line. - `EVALBOARD_TASK_ANALYSIS=off` turns it off. + `readRunDoctor`). `status.json` is refetched after 60 s; the triage file it + names, `/fixes.json` and up to 10 patches are cached for good, and + `/api/file` fetches a patch on demand. A done report gets the panel; when + `analysis.md` has a `## Findings` line, the AI summary stops there and the + panel replaces the rest. Other states keep the AI summary, with a status + line. `EVALBOARD_TASK_ANALYSIS=off` turns it off. - Pass rows render green (`bg-green-50 text-green-700`), failures render red (`bg-red-50 text-red-700`), on a white background. diff --git a/evalboard/app/runs/[id]/__tests__/task-failure-analysis.test.tsx b/evalboard/app/runs/[id]/__tests__/task-failure-analysis.test.tsx index f1127670..157e3c5d 100644 --- a/evalboard/app/runs/[id]/__tests__/task-failure-analysis.test.tsx +++ b/evalboard/app/runs/[id]/__tests__/task-failure-analysis.test.tsx @@ -30,11 +30,13 @@ function doctor(): RunDoctor { } describe("TaskFailureAnalysis", () => { - test("header: counts by layer and class, new vs recurring, cost, build link", () => { + test("header: counts by layer, class and task state, history, cost, build link", () => { render(); const panel = screen.getByRole("region", { name: "Task Failure Analysis" }); expect(panel).toHaveTextContent("3 failed, 2 diagnosed"); - expect(panel).toHaveTextContent("2 new, 1 recurring"); + expect(panel).toHaveTextContent("new 2"); + expect(panel).toHaveTextContent("known 1"); + expect(panel).toHaveTextContent("vs 2099-01-02_04-25-52: 2 new, 1 persisting, 0 fixed"); expect(panel).toHaveTextContent("diagnosis $1.25"); expect(panel).toHaveTextContent("grader 1"); expect(panel).toHaveTextContent("failure 2"); @@ -91,6 +93,36 @@ describe("TaskFailureAnalysis", () => { expect(screen.getByText("build").tagName).toBe("SPAN"); }); + test("no base run: no history line", () => { + const d = doctor(); + d.triage!.summary.vsHistory = null; + render(); + expect(screen.queryByText(/^vs /)).toBeNull(); + }); + + test("a task id the task page would reject is plain text, not a link", () => { + const d = doctor(); + d.triage!.tasks[0].taskId = "../escape"; + render(); + expect(screen.getByText("../escape").closest("a")).toBeNull(); + const links = screen.getAllByRole("link").map((a) => a.getAttribute("href")); + expect(links.some((h) => h?.includes("escape"))).toBe(false); + }); + + test("a patch whose id is not a safe file name shows its size without a link", () => { + const d = doctor(); + d.fixes!.candidates[0].id = "../bad"; + render(); + expect(screen.getByText("+3/-1").closest("a")).toBeNull(); + }); + + test("no candidates", () => { + const d = doctor(); + d.fixes!.candidates = []; + render(); + expect(screen.getByText("No fix candidates.")).toBeInTheDocument(); + }); + test("no fixes.json: no Fixes table", () => { const d = { ...doctor(), fixes: null }; render(); @@ -120,6 +152,13 @@ describe("DoctorStatusLine", () => { expect(screen.getByTestId("doctor-status-line")).toHaveTextContent("skipped: ad-hoc run"); }); + test("done without a triage says why there is no panel", () => { + render(); + expect(screen.getByTestId("doctor-status-line")).toHaveTextContent( + "report done, but its triage file is missing or did not parse", + ); + }); + test("retracted is a one-line note", () => { render(); expect(screen.getByTestId("doctor-status-line")).toHaveTextContent("retracted"); diff --git a/evalboard/app/runs/[id]/task-failure-analysis.tsx b/evalboard/app/runs/[id]/task-failure-analysis.tsx index 35a75c7b..ddf6f378 100644 --- a/evalboard/app/runs/[id]/task-failure-analysis.tsx +++ b/evalboard/app/runs/[id]/task-failure-analysis.tsx @@ -7,7 +7,7 @@ import { isAdoUrl, isLinkableTaskId, isSkillsPrUrl, - PATCH_ID_RE, + patchIds, type DoctorFix, type DoctorStatus, type DoctorTask, @@ -70,15 +70,6 @@ function Counts({ ); } -const STATE_TEXT: Record = { - queued: "queued", - running: "running", - done: "done", - failed: "failed", - skipped: "skipped", - retracted: "retracted", -}; - /** One line under the AI summary when the full panel is not shown. */ export function DoctorStatusLine({ status }: { status: DoctorStatus }) { const build = status.buildUrl ? ( @@ -95,7 +86,7 @@ export function DoctorStatusLine({ status }: { status: DoctorStatus }) { case "running": body = ( <> - {STATE_TEXT[status.state]} + {status.state} {status.startedAt && ` since ${fmtTimestamp(status.startedAt)}`} {build} @@ -106,7 +97,7 @@ export function DoctorStatusLine({ status }: { status: DoctorStatus }) { case "skipped": body = ( <> - {STATE_TEXT[status.state]} + {status.state} {status.reason && `: ${status.reason}`} {build} @@ -118,7 +109,8 @@ export function DoctorStatusLine({ status }: { status: DoctorStatus }) { case "done": body = ( <> - report done, not shown here + report done, but its triage file is missing or did not + parse {build} ); @@ -156,9 +148,10 @@ function TaskRow({ ) : ( t.taskId )} - {t.nightsSeen > 1 && ( + {t.state && t.state !== "new" && ( - {t.nightsSeen} nights + {t.state} + {t.nightsSeen > 1 && `, ${t.nightsSeen} nights`} )} @@ -202,6 +195,17 @@ function gateText(f: DoctorFix): string { return parts.length ? parts.join(" · ") : "—"; } +function PatchCell({ fix, href }: { fix: DoctorFix; href: string | null }) { + if (!fix.patchLines) return <>—; + const text = `+${fix.patchLines.added}/-${fix.patchLines.removed}`; + if (!href) return <>{text}; + return ( + + {text} + + ); +} + function FixesTable({ doctor, runId, @@ -214,12 +218,14 @@ function FixesTable({ const fixes = doctor.fixes; const dir = doctor.status.fixesDir; if (!fixes || !dir) return null; + // The same predicate readRunDoctor fetches with. + const linked = patchIds(fixes); return (

Fixes - {fixes.fixMode && `fixMode ${fixes.fixMode} · `} + {fixes.fixMode && `mode ${fixes.fixMode} · `} {fmtUsd(fixes.costUsd)} {fixes.totalCapUsd != null && ` of ${fmtUsd(fixes.totalCapUsd)} cap`} @@ -260,24 +266,17 @@ function FixesTable({ {f.covers.length} - {f.patchLines ? ( - PATCH_ID_RE.test(f.id) ? ( - - +{f.patchLines.added}/- - {f.patchLines.removed} - - ) : ( - `+${f.patchLines.added}/-${f.patchLines.removed}` - ) - ) : ( - "—" - )} + {gateText(f)} @@ -303,8 +302,12 @@ export function TaskFailureAnalysis({ }) { const { status, triage } = doctor; if (!triage) return null; - const recurring = triage.tasks.filter((t) => t.nightsSeen > 1).length; - const fresh = triage.tasks.length - recurring; + // Per-task triage state (new / changed / known), counted for the header. + const byState: Record = {}; + for (const t of triage.tasks) { + if (t.state) byState[t.state] = (byState[t.state] ?? 0) + 1; + } + const vh = triage.summary.vsHistory; return (
skill-doctor · {triage.summary.rowsFailed} failed,{" "} - {triage.summary.rowsDiagnosed} diagnosed · {fresh} new,{" "} - {recurring} recurring · diagnosis{" "} + {triage.summary.rowsDiagnosed} diagnosed · diagnosis{" "} {fmtUsd(status.costUsd ?? triage.costUsd)} {status.buildUrl && ( <> @@ -340,6 +342,13 @@ export function TaskFailureAnalysis({

+ + {vh && ( +

+ vs {vh.baseRun}: {vh.new} new, {vh.persisting}{" "} + persisting, {vh.fixed} fixed +

+ )} {triage.disclaimer && (

{triage.disclaimer} diff --git a/evalboard/lib/__tests__/doctor-blob.test.ts b/evalboard/lib/__tests__/doctor-blob.test.ts index 280e7e60..a801f38c 100644 --- a/evalboard/lib/__tests__/doctor-blob.test.ts +++ b/evalboard/lib/__tests__/doctor-blob.test.ts @@ -70,56 +70,80 @@ afterEach(async () => { await fs.rm(root, { recursive: true, force: true }); }); -async function ensure() { - const { ensureRunDoctor } = await import("../blob"); - return ensureRunDoctor("runs", RUN, root); +async function read() { + const { readRunDoctor } = await import("../runs"); + return readRunDoctor(RUN); } -describe("ensureRunDoctor", () => { +const D = `${RUN}/skill-doctor`; + +describe("readRunDoctor: what it fetches", () => { test("a done report: status, triage, fixes.json and the one patch", async () => { - await ensure(); + await read(); expect(state.downloads.sort()).toEqual( [ - `${RUN}/skill-doctor/status.json`, - `${RUN}/skill-doctor/triage-1000001.json`, - `${RUN}/skill-doctor/fixes-1000001/fixes.json`, - `${RUN}/skill-doctor/fixes-1000001/example-grader-fix.patch`, + `${D}/status.json`, + `${D}/triage-1000001.json`, + `${D}/fixes-1000001/fixes.json`, + `${D}/fixes-1000001/example-grader-fix.patch`, ].sort(), ); }); test("no report: one 404, swallowed", async () => { state.blobs = {}; - await expect(ensure()).resolves.toBeUndefined(); - expect(state.downloads).toEqual([`${RUN}/skill-doctor/status.json`]); + await expect(read()).resolves.toBeNull(); + expect(state.downloads).toEqual([`${D}/status.json`]); }); test("running: the triage is not fetched", async () => { seedBlobs(statusWith({ state: "running", fixes_dir: null })); - await ensure(); - expect(state.downloads).toEqual([`${RUN}/skill-doctor/status.json`]); + expect((await read())?.status.state).toBe("running"); + expect(state.downloads).toEqual([`${D}/status.json`]); }); test("a bad triage_file name is never fetched", async () => { seedBlobs(statusWith({ triage_file: "../run.json", fixes_dir: null })); - await ensure(); - expect(state.downloads).toEqual([`${RUN}/skill-doctor/status.json`]); + expect((await read())?.triage).toBeNull(); + expect(state.downloads).toEqual([`${D}/status.json`]); }); - test("schema_version 2: nothing beyond status.json", async () => { + test("schema_version 2: nothing beyond status.json, and a warning", async () => { seedBlobs(statusWith({ schema_version: 2 })); - await ensure(); - expect(state.downloads).toEqual([`${RUN}/skill-doctor/status.json`]); + const warn = vi.spyOn(console, "warn").mockImplementation(() => {}); + await expect(read()).resolves.toBeNull(); + expect(state.downloads).toEqual([`${D}/status.json`]); + expect(warn).toHaveBeenCalledTimes(1); + // Names the file, never its contents. + expect(String(warn.mock.calls[0][0])).toContain("status.json"); + expect(String(warn.mock.calls[0][0])).not.toContain("schema_version"); + warn.mockRestore(); }); - test("status.json is refetched after a minute; the triage never", async () => { - await ensure(); + test("status.json is refetched after a minute, not within it; the rest never", async () => { + await read(); + state.downloads = []; + await read(); + expect(state.downloads).toEqual([]); const st = path.join(root, RUN, "skill-doctor", "status.json"); const old = new Date(Date.now() - 120_000); await fs.utimes(st, old, old); - state.downloads = []; - await ensure(); - expect(state.downloads).toEqual([`${RUN}/skill-doctor/status.json`]); + await read(); + expect(state.downloads).toEqual([`${D}/status.json`]); + }); + + test("patch fan-out is capped and ids are checked", async () => { + const doc = JSON.parse(fixture("fixes-1000001/fixes.json")); + const base = doc.candidates[0]; + doc.candidates = [ + { ...base, id: "../escape" }, + ...Array.from({ length: 15 }, (_, i) => ({ ...base, id: `fix-${i}` })), + ]; + state.blobs[`${D}/fixes-1000001/fixes.json`] = JSON.stringify(doc); + await read(); + const patches = state.downloads.filter((n) => n.endsWith(".patch")); + expect(patches).toHaveLength(10); + expect(patches.some((n) => n.includes(".."))).toBe(false); }); }); @@ -146,7 +170,7 @@ describe("readRunDoctor", () => { await expect(readRunDoctor(RUN)).resolves.toBeNull(); }); - test("the patch is served by /api/file's path check from the cache", async () => { + test("/api/file's path check serves a cached patch without a listing", async () => { const { readRunDoctor, resolveSafePath } = await import("../runs"); await readRunDoctor(RUN); state.downloads = []; @@ -163,8 +187,34 @@ describe("readRunDoctor", () => { "example-grader-fix.patch", ), ); - // No listing or run.json fetch for a doctor path. expect(state.downloads).toEqual([]); - expect(await resolveSafePath(RUN, "skill-doctor/../../x")).toBeNull(); + }); + + test("a cold cache fetches the patch on demand, by name", async () => { + const { resolveSafePath } = await import("../runs"); + const abs = await resolveSafePath( + RUN, + "skill-doctor/fixes-1000001/example-grader-fix.patch", + ); + expect(state.downloads).toEqual([ + `${D}/fixes-1000001/example-grader-fix.patch`, + ]); + expect(await fs.readFile(abs!, "utf-8")).toContain("diff --git"); + }); + + test("only patch paths are served from skill-doctor/", async () => { + const { resolveSafePath } = await import("../runs"); + for (const bad of [ + "skill-doctor/status.json", + "skill-doctor/triage-1000001.json", + "skill-doctor/fixes-1000001/fixes.json", + "skill-doctor/fixes-1000001/../status.json", + "skill-doctor/other/x.patch", + "skill-doctor/fixes-1000001/a/b.patch", + "skill-doctor/../../x", + ]) { + expect(await resolveSafePath(RUN, bad)).toBeNull(); + } + expect(state.downloads).toEqual([]); }); }); diff --git a/evalboard/lib/__tests__/doctor.test.ts b/evalboard/lib/__tests__/doctor.test.ts index 47c19e97..0143645e 100644 --- a/evalboard/lib/__tests__/doctor.test.ts +++ b/evalboard/lib/__tests__/doctor.test.ts @@ -3,6 +3,8 @@ import path from "node:path"; import { describe, expect, test } from "vitest"; import { isAdoUrl, + isLinkableTaskId, + patchIdFromPath, isSkillsPrUrl, parseDoctorStatus, parseFixes, @@ -91,7 +93,6 @@ describe("parseDoctorStatus", () => { expect(s.state).toBe("done"); expect(s.triageFile).toBe("triage-1000001.json"); expect(s.fixesDir).toBe("fixes-1000001"); - expect(s.fixes?.counts).toEqual({ "pr-ready": 1, "needs-live-run": 1 }); expect(s.costUsd).toBe(1.25); }); @@ -127,7 +128,6 @@ describe("parseDoctorStatus", () => { delete old.trigger; const s = parseDoctorStatus(JSON.stringify(old))!; expect(s.fixesDir).toBeNull(); - expect(s.fixes).toBeNull(); expect(s.trigger).toBeNull(); }); }); @@ -138,6 +138,8 @@ describe("parseTriage / parseFixes", () => { expect(t.tasks.map((x) => x.taskId)).toEqual(["example-e1", "example-t1", "example-t2"]); expect(t.summary.byLayer).toEqual({ env: 1, grader: 1, skill: 1 }); expect(t.tasks[1].nightsSeen).toBe(3); + expect(t.tasks.map((x) => x.state)).toEqual(["new", "known", "new"]); + expect(t.summary.vsHistory?.baseRun).toBe("2099-01-02_04-25-52"); expect(t.tasks[1].existingPrs).toHaveLength(3); expect(t.actions).toHaveLength(2); }); @@ -148,17 +150,44 @@ describe("parseTriage / parseFixes", () => { expect(parseTriage(JSON.stringify({ schema_version: 1 }))).toBeNull(); }); + test("vs_history without a base run is not shown", () => { + const doc = JSON.parse(TRIAGE); + doc.summary.vs_history.base_run = null; + expect(parseTriage(JSON.stringify(doc))!.summary.vsHistory).toBeNull(); + }); + + test("an unknown task state is null", () => { + const doc = JSON.parse(TRIAGE); + doc.tasks[0].state = "persisting"; + expect(parseTriage(JSON.stringify(doc))!.tasks[0].state).toBeNull(); + }); + + test("fixes.json with schema_version 2, or none, is absent", () => { + const doc = JSON.parse(FIXES); + expect(parseFixes(JSON.stringify({ ...doc, schema_version: 2 }))).toBeNull(); + delete doc.schema_version; + expect(parseFixes(JSON.stringify(doc))).toBeNull(); + }); + test("fixes: candidates, gates and patch ids", () => { const f = parseFixes(FIXES)!; expect(f.candidates.map((c) => c.outcome)).toEqual(["pr-ready", "needs-live-run"]); expect(f.candidates[0].gates).toEqual({ replay: "pass", mutants: { n: 10, killed: 9 }, review: "SEND" }); - expect(patchIds(f)).toEqual(["example-grader-fix"]); + expect([...patchIds(f)]).toEqual(["example-grader-fix"]); }); test("a candidate id that is not a safe file name gets no patch", () => { const doc = JSON.parse(FIXES); doc.candidates[0].id = "../../run"; - expect(patchIds(parseFixes(JSON.stringify(doc)))).toEqual([]); + expect(patchIds(parseFixes(JSON.stringify(doc))).size).toBe(0); + }); + + test("patchIdFromPath", () => { + expect(patchIdFromPath("fixes-1/a.patch")).toEqual({ fixesDir: "fixes-1", id: "a" }); + expect(patchIdFromPath("fixes/a.b.patch")).toEqual({ fixesDir: "fixes", id: "a.b" }); + for (const bad of ["fixes-1/fixes.json", "fixes-1/.patch", "x/a.patch", "fixes-1/../a.patch", "a.patch", "fixes-1/a/b.patch"]) { + expect(patchIdFromPath(bad)).toBeNull(); + } }); }); @@ -196,6 +225,18 @@ describe("link validation", () => { }); }); +describe("isLinkableTaskId", () => { + test("accepts the ids the task page accepts", () => { + expect(isLinkableTaskId("example-t1")).toBe(true); + expect(isLinkableTaskId("suite/row-3")).toBe(true); + }); + test("rejects path tricks, URLs and odd characters", () => { + for (const bad of ["", "../x", "a/../b", "a/./b", "/abs", "a//b", "https://x", "a b", "a?b", "a#b", "x".repeat(256)]) { + expect(isLinkableTaskId(bad)).toBe(false); + } + }); +}); + describe("taskAnalysisEnabled (gating + kill switch)", () => { test("internal + skills + switch not off", () => { expect(taskAnalysisEnabled(true, "skills", undefined)).toBe(true); @@ -204,6 +245,8 @@ describe("taskAnalysisEnabled (gating + kill switch)", () => { }); test("off, OSS or another source", () => { expect(taskAnalysisEnabled(true, "skills", "off")).toBe(false); + expect(taskAnalysisEnabled(true, "skills", "OFF")).toBe(false); + expect(taskAnalysisEnabled(true, "skills", " Off ")).toBe(false); expect(taskAnalysisEnabled(false, "skills", undefined)).toBe(false); expect(taskAnalysisEnabled(true, "scribe", undefined)).toBe(false); expect(taskAnalysisEnabled(true, "gha", undefined)).toBe(false); @@ -225,17 +268,25 @@ describe("planAnalysis (states)", () => { expect(p.statusLine).toBeNull(); }); - test("done without a split: today's view plus a status line", () => { + test("done without a split: the full analysis, then the panel", () => { const md = "# A\n\n## TL;DR\nonly\n"; - const p = planAnalysis(md, doctor()); - expect(p).toMatchObject({ summary: md, panel: null }); - expect(p.statusLine?.state).toBe("done"); - expect(planAnalysis(null, doctor()).panel).toBeNull(); + const d = doctor(); + expect(planAnalysis(md, d)).toEqual({ summary: md, panel: d, statusLine: null }); + }); + + test("done without analysis.md: the panel alone", () => { + const d = doctor(); + expect(planAnalysis(null, d)).toEqual({ summary: null, panel: d, statusLine: null }); + }); + + test("an analysis.md that starts with Findings: no summary, the panel", () => { + const d = doctor(); + expect(planAnalysis("## Findings\nx\n", d)).toEqual({ summary: null, panel: d, statusLine: null }); }); test("done but the triage did not load: no panel", () => { const d = { ...doctor(), triage: null }; - expect(planAnalysis(MD, d)).toMatchObject({ summary: MD, panel: null }); + expect(planAnalysis(MD, d)).toEqual({ summary: MD, panel: null, statusLine: d.status }); }); test.each(["queued", "running", "failed", "skipped", "retracted"])( diff --git a/evalboard/lib/blob.ts b/evalboard/lib/blob.ts index 70c9d7ef..67ed24b9 100644 --- a/evalboard/lib/blob.ts +++ b/evalboard/lib/blob.ts @@ -3,7 +3,8 @@ import path from "node:path"; import { randomBytes } from "node:crypto"; import type { BlobServiceClient, ContainerClient } from "@azure/storage-blob"; import { DEFAULT_VARIANT_ID, isValidVariantId } from "./variants"; -import { DOCTOR_DIR, parseDoctorStatus, parseFixes, patchIds } from "./doctor"; +import { isValidId, isValidTaskId } from "./ids"; +import { DOCTOR_DIR } from "./doctor"; const ACCOUNT = "coderevaltests"; @@ -13,32 +14,7 @@ const ACCOUNT = "coderevaltests"; // directly. export const LOCAL_RUNS_DIR = process.env.EVALBOARD_LOCAL_RUNS_DIR || null; -// Run / task IDs get reflected into filesystem paths and blob prefixes, so -// reject anything outside a narrow whitelist before any side effect. -const ID_RE = /^[\w.-]+$/; -// Task IDs from dataset-expanded tasks look like "sentiment-classification/r3" -// (suite/row). Each segment must still pass the narrow ID_RE — only the slash -// separator between segments is additionally allowed. -const TASK_ID_RE = /^[\w.-]+(\/[\w.-]+)*$/; - -export function isValidId(id: unknown): id is string { - return ( - typeof id === "string" && - id.length > 0 && - id.length < 128 && - ID_RE.test(id) - ); -} - -export function isValidTaskId(id: unknown): id is string { - return ( - typeof id === "string" && - id.length > 0 && - id.length < 256 && - TASK_ID_RE.test(id) && - !id.split("/").some((s) => s === "." || s === "..") - ); -} +export { isValidId, isValidTaskId }; function assertValidId(id: string, label: string): void { if (!isValidId(id)) { @@ -343,73 +319,41 @@ export async function ensureRunReviewIndex( }); } -// skill-doctor report sidecars under `/skill-doctor/` (see lib/doctor.ts). -// status.json changes while the doctor runs, so a copy older than a minute is -// fetched again. The triage file it names (once `done`), fixes.json and the -// patches are immutable and cached forever. No listing: every name comes from -// status.json or fixes.json and is checked against its pattern first. 404s are +// skill-doctor report sidecars under `/skill-doctor/` (see lib/doctor.ts +// and readRunDoctor, which decides which files a status implies). Download +// only, like the siblings above: status.json is mutable and passes a refresh +// age; the triage file, fixes.json and the patches are immutable and are never +// fetched again once cached. `relPath` comes from a parsed, pattern-checked +// field, and is checked here again before it becomes a blob name. 404s are // swallowed as for analysis.md. -export const DOCTOR_STATUS_REFRESH_MS = 60_000; - -async function downloadOptional( - container: string, - blobName: string, - destRoot: string, - refreshAfterMs?: number, -): Promise { - try { - await downloadBlob(container, blobName, destRoot, refreshAfterMs); - } catch (err) { - if (!isNotFound(err)) throw err; - } -} +const DOCTOR_REL_RE = /^[A-Za-z0-9._-]+(\/[A-Za-z0-9._-]+)?$/; -export async function ensureRunDoctor( +export async function ensureRunDoctorFile( container: string, runId: string, + relPath: string, destRoot: string, + refreshAfterMs?: number, ): Promise { assertValidId(runId, "runId"); + if ( + !DOCTOR_REL_RE.test(relPath) || + relPath.split("/").some((s) => s === "." || s === "..") + ) { + throw new Error(`Invalid skill-doctor path: ${JSON.stringify(relPath)}`); + } if (LOCAL_RUNS_DIR) return; - return dedupe(`doctor:${container}:${runId}`, async () => { - const base = `${runId}/${DOCTOR_DIR}`; - await downloadOptional( - container, - `${base}/status.json`, - destRoot, - DOCTOR_STATUS_REFRESH_MS, - ); - const status = parseDoctorStatus( - await fs - .readFile(path.join(destRoot, base, "status.json"), "utf-8") - .catch(() => null), - ); - if (!status) return; - const ops: Promise[] = []; - if (status.state === "done" && status.triageFile) { - ops.push( - downloadOptional(container, `${base}/${status.triageFile}`, destRoot), - ); - } - if (status.fixesDir) { - const dir = `${base}/${status.fixesDir}`; - ops.push( - (async () => { - await downloadOptional(container, `${dir}/fixes.json`, destRoot); - const fixes = parseFixes( - await fs - .readFile(path.join(destRoot, dir, "fixes.json"), "utf-8") - .catch(() => null), - ); - await Promise.all( - patchIds(fixes).map((id) => - downloadOptional(container, `${dir}/${id}.patch`, destRoot), - ), - ); - })(), + return dedupe(`doctor:${container}:${runId}/${relPath}`, async () => { + try { + await downloadBlob( + container, + `${runId}/${DOCTOR_DIR}/${relPath}`, + destRoot, + refreshAfterMs, ); + } catch (err) { + if (!isNotFound(err)) throw err; } - await Promise.all(ops); }); } diff --git a/evalboard/lib/doctor.ts b/evalboard/lib/doctor.ts index 8426da02..180fcf7e 100644 --- a/evalboard/lib/doctor.ts +++ b/evalboard/lib/doctor.ts @@ -7,17 +7,22 @@ // Pure data + string helpers, no Node builtins: the parsed shapes are passed to // the run page's components, and the tests import this module directly. The // types are written by hand against the doctor's status_v1 / triage_v1 schemas -// and keep only the fields the board shows. Anything that does not parse, or a -// schema_version other than 1, is treated as absent: the page must never fail -// because of a doctor file. +// and carry only the fields the run page renders. A file that does not parse, +// or whose schema_version is not 1, is treated as absent: the page must never +// fail because of a doctor file. + +import { isValidTaskId } from "./ids"; +import { SKILLS_SOURCE } from "./sources"; export const DOCTOR_DIR = "skill-doctor"; -export const TRIAGE_FILE_RE = /^triage-[A-Za-z0-9._-]+\.json$/; +const TRIAGE_FILE_RE = /^triage-[A-Za-z0-9._-]+\.json$/; export const FIXES_DIR_RE = /^fixes(-[0-9]+)?$/; // Candidate ids name `.patch` inside the fixes dir. export const PATCH_ID_RE = /^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$/; +// At most this many patches are fetched (and linked) per run. +export const MAX_PATCHES = 10; -export const DOCTOR_STATES = [ +const DOCTOR_STATES = [ "queued", "running", "done", @@ -27,25 +32,18 @@ export const DOCTOR_STATES = [ ] as const; export type DoctorState = (typeof DOCTOR_STATES)[number]; -export interface DoctorFixesDigest { - fixMode: string | null; - costUsd: number | null; - totalCapUsd: number | null; - counts: Record; +function isDoctorState(v: unknown): v is DoctorState { + return typeof v === "string" && (DOCTOR_STATES as readonly string[]).includes(v); } export interface DoctorStatus { state: DoctorState; reason: string; startedAt: string | null; - finishedAt: string | null; buildUrl: string | null; costUsd: number | null; - rowsFailed: number | null; - rowsDiagnosed: number | null; triageFile: string | null; fixesDir: string | null; - fixes: DoctorFixesDigest | null; trigger: string | null; } @@ -55,48 +53,42 @@ export interface DoctorPr { state: string; } +// triage_v1 task.state: new tonight, changed since the last night, or known. +export type DoctorTaskState = "new" | "changed" | "known"; + export interface DoctorTask { taskId: string; skill: string; layer: string; cls: string; + state: DoctorTaskState | null; + nightsSeen: number; confidence: string | null; - contested: string[] | null; - criterion: string | null; - diagnosed: boolean; why: string | null; fix: string | null; - solvable: string | null; - state: string | null; - nightsSeen: number; owners: string[]; existingPrs: DoctorPr[]; - doctorPr: string | null; - evidence: string | null; - envGroup: string | null; } export interface DoctorAction { kind: string; - count: number; text: string; - taskIds: string[]; } export interface DoctorTriage { disclaimer: string; costUsd: number | null; - buildUrl: string | null; summary: { rowsFailed: number; rowsDiagnosed: number; byClass: Record; byLayer: Record; + // Only set when the doctor compared against a base run. vsHistory: { new: number; persisting: number; fixed: number; - baseRun: string | null; + baseRun: string; } | null; }; tasks: DoctorTask[]; @@ -117,7 +109,6 @@ export interface DoctorFix { costUsd: number | null; patchLines: { added: number; removed: number } | null; title: string; - problem: string; gates: DoctorGates; } @@ -176,34 +167,18 @@ function parseJson(raw: string | null | undefined): unknown { export function parseDoctorStatus(raw: string | null | undefined): DoctorStatus | null { const j = parseJson(raw); - if (!isObj(j) || j.schema_version !== 1) return null; - const state = j.state; - if (typeof state !== "string" || !(DOCTOR_STATES as readonly string[]).includes(state)) - return null; + if (!isObj(j) || j.schema_version !== 1 || !isDoctorState(j.state)) return null; const triageFile = str(j.triage_file); const fixesDir = str(j.fixes_dir); - let fixes: DoctorFixesDigest | null = null; - if (isObj(j.fixes)) { - fixes = { - fixMode: str(j.fixes.fix_mode), - costUsd: num(j.fixes.cost_usd), - totalCapUsd: num(j.fixes.total_cap_usd), - counts: countMap(j.fixes.counts), - }; - } return { - state: state as DoctorState, + state: j.state, reason: str(j.reason) ?? "", startedAt: str(j.started_at), - finishedAt: str(j.finished_at), buildUrl: str(j.build_url), costUsd: num(j.cost_usd), - rowsFailed: typeof j.rows_failed === "number" ? int(j.rows_failed) : null, - rowsDiagnosed: typeof j.rows_diagnosed === "number" ? int(j.rows_diagnosed) : null, // A name outside the pattern is treated as absent: it lands in a path. triageFile: triageFile && TRIAGE_FILE_RE.test(triageFile) ? triageFile : null, fixesDir: fixesDir && FIXES_DIR_RE.test(fixesDir) ? fixesDir : null, - fixes, trigger: str(j.trigger), }; } @@ -215,6 +190,10 @@ function parsePr(v: unknown): DoctorPr | null { return { url, title: str(v.title) ?? "", state: str(v.state) ?? "unknown" }; } +function taskState(v: unknown): DoctorTaskState | null { + return v === "new" || v === "changed" || v === "known" ? v : null; +} + function parseTask(v: unknown): DoctorTask | null { if (!isObj(v)) return null; const taskId = str(v.task_id); @@ -224,22 +203,15 @@ function parseTask(v: unknown): DoctorTask | null { skill: str(v.skill) ?? "", layer: str(v.layer) ?? "undiagnosed", cls: str(v.class) ?? "", + state: taskState(v.state), + nightsSeen: int(v.nights_seen, 1), confidence: str(v.confidence), - contested: Array.isArray(v.contested) ? strList(v.contested) : null, - criterion: str(v.criterion), - diagnosed: v.diagnosed === true, why: str(v.why), fix: str(v.fix), - solvable: str(v.solvable), - state: str(v.state), - nightsSeen: int(v.nights_seen, 1), owners: strList(v.owners), existingPrs: Array.isArray(v.existing_prs) ? v.existing_prs.map(parsePr).filter((p): p is DoctorPr => p !== null) : [], - doctorPr: str(v.doctor_pr), - evidence: str(v.evidence), - envGroup: str(v.env_group), }; } @@ -250,37 +222,32 @@ export function parseTriage(raw: string | null | undefined): DoctorTriage | null const s = j.summary; const run = isObj(j.run) ? j.run : {}; const vh = isObj(s.vs_history) ? s.vs_history : null; + const baseRun = vh ? str(vh.base_run) : null; return { disclaimer: str(j.disclaimer) ?? "", costUsd: num(run.cost_usd), - buildUrl: str(run.build_url), summary: { rowsFailed: int(s.rows_failed), rowsDiagnosed: int(s.rows_diagnosed), byClass: countMap(s.by_class), byLayer: countMap(s.by_layer), - vsHistory: vh - ? { - new: int(vh.new), - persisting: int(vh.persisting), - fixed: int(vh.fixed), - baseRun: str(vh.base_run), - } - : null, + vsHistory: + vh && baseRun + ? { + new: int(vh.new), + persisting: int(vh.persisting), + fixed: int(vh.fixed), + baseRun, + } + : null, }, tasks: j.tasks.map(parseTask).filter((t): t is DoctorTask => t !== null), actions: Array.isArray(j.actions) - ? j.actions.flatMap((a): DoctorAction[] => { - if (!isObj(a) || typeof a.text !== "string") return []; - return [ - { - kind: str(a.kind) ?? "", - count: int(a.count), - text: a.text, - taskIds: strList(a.task_ids), - }, - ]; - }) + ? j.actions.flatMap((a): DoctorAction[] => + isObj(a) && typeof a.text === "string" + ? [{ kind: str(a.kind) ?? "", text: a.text }] + : [], + ) : [], }; } @@ -298,9 +265,7 @@ function parseGates(v: unknown): DoctorGates { export function parseFixes(raw: string | null | undefined): DoctorFixes | null { const j = parseJson(raw); - if (!isObj(j) || !Array.isArray(j.candidates)) return null; - // fixes.json carries its own schema_version 1; an unknown one is absent. - if (j.schema_version !== undefined && j.schema_version !== 1) return null; + if (!isObj(j) || j.schema_version !== 1 || !Array.isArray(j.candidates)) return null; return { fixMode: str(j.fix_mode), costUsd: num(j.cost_usd), @@ -322,7 +287,6 @@ export function parseFixes(raw: string | null | undefined): DoctorFixes | null { ? { added: int(pl.added), removed: int(pl.removed) } : null, title: str(c.title) ?? "", - problem: str(c.problem) ?? "", gates: parseGates(c.gates), }, ]; @@ -330,12 +294,28 @@ export function parseFixes(raw: string | null | undefined): DoctorFixes | null { }; } -/** Patch ids whose `.patch` the board fetches and links. */ -export function patchIds(fixes: DoctorFixes | null): string[] { - if (!fixes) return []; - return fixes.candidates - .filter((c) => c.patchLines !== null && PATCH_ID_RE.test(c.id)) - .map((c) => c.id); +/** + * The candidates whose `.patch` the board fetches and links: one predicate + * for both, so a link never points at a patch that is not fetched. A patch + * exists only when fixes.json reports patch lines; the id must be a safe file + * name; at most MAX_PATCHES per run. + */ +export function patchIds(fixes: DoctorFixes | null): Set { + const ids = new Set(); + for (const c of fixes?.candidates ?? []) { + if (ids.size >= MAX_PATCHES) break; + if (c.patchLines !== null && PATCH_ID_RE.test(c.id)) ids.add(c.id); + } + return ids; +} + +/** `/.patch` → the id, or null for any other doctor path. */ +export function patchIdFromPath(rel: string): { fixesDir: string; id: string } | null { + const parts = rel.split("/"); + if (parts.length !== 2 || !FIXES_DIR_RE.test(parts[0])) return null; + const m = /^(.+)\.patch$/.exec(parts[1]); + if (!m || !PATCH_ID_RE.test(m[1])) return null; + return { fixesDir: parts[0], id: m[1] }; } // ---------- links ---------- @@ -353,18 +333,8 @@ export function isAdoUrl(u: unknown): u is string { return typeof u === "string" && ADO_RE.test(u); } -// Same shape as blob.ts's task-id check (kept here so this module stays free -// of Node builtins): a task link is only built from an id the task page accepts. -const TASK_ID_RE = /^[\w.-]+(\/[\w.-]+)*$/; - -export function isLinkableTaskId(id: string): boolean { - return ( - id.length > 0 && - id.length < 256 && - TASK_ID_RE.test(id) && - !id.split("/").some((s) => s === "." || s === "..") - ); -} +/** A task link is built only from an id the task page itself accepts. */ +export const isLinkableTaskId = isValidTaskId; // ---------- analysis.md split ---------- @@ -396,44 +366,50 @@ export function splitAnalysis( // ---------- gating + page plan ---------- -/** The panel is internal-only, for the skills nightly, with an env kill switch. */ +/** + * The panel is internal-only, for the skills nightly, with an env kill switch + * (EVALBOARD_TASK_ANALYSIS=off, in any case). + */ export function taskAnalysisEnabled( internal: boolean, sourceId: string, env: string | undefined, ): boolean { - return internal && sourceId === "skills" && env !== "off"; + return ( + internal && + sourceId === SKILLS_SOURCE.id && + (env ?? "").trim().toLowerCase() !== "off" + ); } export interface AnalysisPlan { // Markdown for the AI summary panel (null: no panel). summary: string | null; - // Render the Task Failure Analysis panel in place of the cut-off part. + // The Task Failure Analysis panel, after the summary (null: none). panel: RunDoctor | null; - // One-line doctor status under the summary (null: none). + // One-line doctor status when there is no panel (null: none). statusLine: DoctorStatus | null; } /** - * What the run page shows. The panel replaces Findings and below only when - * the report is done and analysis.md has a split point; every other case keeps - * today's view, plus a status line when a status exists. + * What the run page shows. A done report with a parsed triage gets the panel; + * when analysis.md has a split point, the summary keeps only the part above + * Findings (the panel replaces the rest), else the whole file. Every other + * state keeps today's view, plus a status line. */ export function planAnalysis( analysis: string | null, doctor: RunDoctor | null, ): AnalysisPlan { if (!doctor) return { summary: analysis, panel: null, statusLine: null }; - const { status } = doctor; - if (status.state === "done" && doctor.triage && analysis) { - const split = splitAnalysis(analysis); - if (split) { - return { - summary: split.top.trim() ? split.top : null, - panel: doctor, - statusLine: null, - }; - } + if (doctor.status.state === "done" && doctor.triage) { + const split = analysis ? splitAnalysis(analysis) : null; + const summary = split ? split.top : analysis; + return { + summary: summary && summary.trim() ? summary : null, + panel: doctor, + statusLine: null, + }; } - return { summary: analysis, panel: null, statusLine: status }; + return { summary: analysis, panel: null, statusLine: doctor.status }; } diff --git a/evalboard/lib/ids.ts b/evalboard/lib/ids.ts new file mode 100644 index 00000000..c8380c9e --- /dev/null +++ b/evalboard/lib/ids.ts @@ -0,0 +1,30 @@ +// Run / task id validators. Pure (no Node builtins), so client components and +// lib/doctor.ts can share the exact check the data layer applies; blob.ts +// re-exports them for its existing importers. + +// Run / task IDs get reflected into filesystem paths and blob prefixes, so +// reject anything outside a narrow whitelist before any side effect. +const ID_RE = /^[\w.-]+$/; +// Task IDs from dataset-expanded tasks look like "sentiment-classification/r3" +// (suite/row). Each segment must still pass the narrow ID_RE — only the slash +// separator between segments is additionally allowed. +const TASK_ID_RE = /^[\w.-]+(\/[\w.-]+)*$/; + +export function isValidId(id: unknown): id is string { + return ( + typeof id === "string" && + id.length > 0 && + id.length < 128 && + ID_RE.test(id) + ); +} + +export function isValidTaskId(id: unknown): id is string { + return ( + typeof id === "string" && + id.length > 0 && + id.length < 256 && + TASK_ID_RE.test(id) && + !id.split("/").some((s) => s === "." || s === "..") + ); +} diff --git a/evalboard/lib/runs.ts b/evalboard/lib/runs.ts index eebd195d..b3c8ef9b 100644 --- a/evalboard/lib/runs.ts +++ b/evalboard/lib/runs.ts @@ -6,7 +6,7 @@ import { LOCAL_RUNS_DIR, ensureActivationSummary, ensureRunAnalysis, - ensureRunDoctor, + ensureRunDoctorFile, ensureRunExperiment, ensureRunMeta, ensureRunSummary, @@ -25,6 +25,8 @@ import { parseDoctorStatus, parseFixes, parseTriage, + patchIdFromPath, + patchIds, type RunDoctor, } from "./doctor"; @@ -2690,32 +2692,63 @@ export async function readRunAnalysis( // ---------- skill-doctor report (skill-doctor/ sidecars) ---------- +// status.json changes while the doctor runs: a cached copy older than this is +// fetched again. Everything it points at is immutable. +const DOCTOR_STATUS_REFRESH_MS = 60_000; + // The run's skill-doctor report, or null when there is none, it does not parse, -// or its schema_version is not 1. Never throws: a doctor file problem must not -// take the run page down, so a fetch error degrades to "no report". +// or its schema_version is not 1. This is the one place that decides which +// files a status implies: status.json → (when done) the triage file it names, +// and /fixes.json → the patches fixes.json lists. Never throws: a +// doctor file problem must not take the run page down, so a fetch error +// degrades to "no report" (or to the parts that did load). export async function readRunDoctor( runId: string, source: Source = DEFAULT_SOURCE, ): Promise { const dir = runsDirFor(RUNS_DIR, source); - try { - await ensureRunDoctor(source.container, runId, dir); - } catch (err) { - console.error(`ensureRunDoctor failed for run ${runId}:`, err); - } const base = path.join(dir, runId, DOCTOR_DIR); - const read = (name: string) => - fs.readFile(path.join(base, name), "utf-8").catch(() => null); + const get = async (rel: string, refreshAfterMs?: number) => { + try { + await ensureRunDoctorFile(source.container, runId, rel, dir, refreshAfterMs); + } catch (err) { + console.error(`skill-doctor: fetching ${rel} failed for run ${runId}:`, err); + } + }; + // Read + parse once. A file that exists but does not parse is logged by + // name only (never its contents), so contract drift is visible. + const load = async (rel: string, parse: (raw: string) => T | null) => { + const raw = await fs.readFile(path.join(base, rel), "utf-8").catch(() => null); + if (raw === null) return null; + const parsed = parse(raw); + if (parsed === null) { + console.warn( + `skill-doctor: ${rel} for run ${runId} did not parse as contract v1; ignored`, + ); + } + return parsed; + }; try { - const status = parseDoctorStatus(await read("status.json")); + await get("status.json", DOCTOR_STATUS_REFRESH_MS); + const status = await load("status.json", parseDoctorStatus); if (!status) return null; - const triage = - status.state === "done" && status.triageFile - ? parseTriage(await read(status.triageFile)) - : null; - const fixes = status.fixesDir - ? parseFixes(await read(path.join(status.fixesDir, "fixes.json"))) - : null; + const triageRel = + status.state === "done" && status.triageFile ? status.triageFile : null; + const fixesRel = status.fixesDir ? `${status.fixesDir}/fixes.json` : null; + await Promise.all([ + triageRel && get(triageRel), + fixesRel && get(fixesRel), + ]); + const [triage, fixes] = await Promise.all([ + triageRel ? load(triageRel, parseTriage) : null, + fixesRel ? load(fixesRel, parseFixes) : null, + ]); + // Cached patches are not fetched again (downloadBlob skips them). + if (status.fixesDir) { + await Promise.all( + [...patchIds(fixes)].map((id) => get(`${status.fixesDir}/${id}.patch`)), + ); + } return { status, triage, fixes }; } catch (err) { console.error(`readRunDoctor failed for run ${runId}:`, err); @@ -2996,8 +3029,18 @@ export async function resolveSafePath( // there; that case falls through to the run-summary fetch as it always did. const parts = relPath.split("/"); if (parts[0] === DOCTOR_DIR) { - // skill-doctor sidecars (the fix patches) are fetched by name by - // readRunDoctor when the run page renders, never by a listing here. + // skill-doctor/ belongs to the doctor contract, so no experiment arm can + // live there: a variant of that name would share the blob prefix with + // the report. Only a fix patch is served from it, fetched by name on + // demand (a cold cache or a direct link works), never by a listing. + const patch = patchIdFromPath(parts.slice(1).join("/")); + if (!patch) return null; + await ensureRunDoctorFile( + source.container, + runId, + `${patch.fixesDir}/${patch.id}.patch`, + dir, + ); } else if (parts[0] !== "activation" && isValidVariantId(parts[0]) && parts[1]) { if (!isValidId(parts[1])) return null; await ensureTaskDir(source.container, runId, parts[1], dir, parts[0]); From 1cfb6abec37d6fe48475237567c636305f7aa940 Mon Sep 17 00:00:00 2001 From: CarlesUIPath Date: Thu, 19 Nov 2026 14:33:29 +0000 Subject: [PATCH 03/13] fix(evalboard): one fetch path for doctor patches; kill switch covers /api/file - /api/file serves a skill-doctor patch only when this run's status names its fixes dir and fixes.json lists it (patchIds), loading both through readRunDoctor on a cold cache: one fetch path, and a made-up name costs no blob call. - The skill-doctor branch returns nothing when the panel is off (OSS edition, another source, or EVALBOARD_TASK_ANALYSIS=off). - ensureRunDoctorFile reuses the ids.ts validator; doctor.ts constants nothing imports are module-private; stale comments fixed. - Tests: the download guard, row badges, unlisted patches, the kill switch on /api/file, and a storage error on the patch fetch. Co-Authored-By: Claude Opus 5.5 --- evalboard/README.md | 5 +- .../__tests__/task-failure-analysis.test.tsx | 13 ++++ .../app/runs/[id]/task-failure-analysis.tsx | 2 +- evalboard/lib/__tests__/doctor-blob.test.ts | 78 +++++++++++++++++-- evalboard/lib/blob.ts | 8 +- evalboard/lib/doctor.ts | 17 ++-- evalboard/lib/runs.ts | 30 +++++-- 7 files changed, 123 insertions(+), 30 deletions(-) diff --git a/evalboard/README.md b/evalboard/README.md index bb9e5f27..8191d22c 100644 --- a/evalboard/README.md +++ b/evalboard/README.md @@ -210,8 +210,9 @@ belongs to it. - Task Failure Analysis (internal edition, skills source): the run page reads the skill-doctor report under `/skill-doctor/` (`lib/doctor.ts`, `readRunDoctor`). `status.json` is refetched after 60 s; the triage file it - names, `/fixes.json` and up to 10 patches are cached for good, and - `/api/file` fetches a patch on demand. A done report gets the panel; when + names, `/fixes.json` and up to 10 patches are cached for good. + `/api/file` serves only the patches fixes.json lists, loading them through + the same reader on a cold cache. A done report gets the panel; when `analysis.md` has a `## Findings` line, the AI summary stops there and the panel replaces the rest. Other states keep the AI summary, with a status line. `EVALBOARD_TASK_ANALYSIS=off` turns it off. diff --git a/evalboard/app/runs/[id]/__tests__/task-failure-analysis.test.tsx b/evalboard/app/runs/[id]/__tests__/task-failure-analysis.test.tsx index 157e3c5d..269403ad 100644 --- a/evalboard/app/runs/[id]/__tests__/task-failure-analysis.test.tsx +++ b/evalboard/app/runs/[id]/__tests__/task-failure-analysis.test.tsx @@ -93,6 +93,19 @@ describe("TaskFailureAnalysis", () => { expect(screen.getByText("build").tagName).toBe("SPAN"); }); + test("row badges: nothing for new, the state for changed, nights for known", () => { + const d = doctor(); + d.triage!.tasks[0].state = "changed"; + d.triage!.tasks[0].nightsSeen = 1; + render(); + const e1 = screen.getByRole("link", { name: "Example e1" }).closest("td")!; + expect(within(e1).getByText("changed")).toBeInTheDocument(); + const t1 = screen.getByRole("link", { name: "Example t1" }).closest("td")!; + expect(within(t1).getByText("known, 3 nights")).toBeInTheDocument(); + const t2 = screen.getByRole("link", { name: "Example t2" }).closest("td")!; + expect(within(t2).queryByText(/new|nights/)).toBeNull(); + }); + test("no base run: no history line", () => { const d = doctor(); d.triage!.summary.vsHistory = null; diff --git a/evalboard/app/runs/[id]/task-failure-analysis.tsx b/evalboard/app/runs/[id]/task-failure-analysis.tsx index ddf6f378..0e74243e 100644 --- a/evalboard/app/runs/[id]/task-failure-analysis.tsx +++ b/evalboard/app/runs/[id]/task-failure-analysis.tsx @@ -218,7 +218,7 @@ function FixesTable({ const fixes = doctor.fixes; const dir = doctor.status.fixesDir; if (!fixes || !dir) return null; - // The same predicate readRunDoctor fetches with. + // The same set readRunDoctor fetches and /api/file serves. const linked = patchIds(fixes); return (

diff --git a/evalboard/lib/__tests__/doctor-blob.test.ts b/evalboard/lib/__tests__/doctor-blob.test.ts index a801f38c..bd6f10d2 100644 --- a/evalboard/lib/__tests__/doctor-blob.test.ts +++ b/evalboard/lib/__tests__/doctor-blob.test.ts @@ -12,6 +12,8 @@ const state = vi.hoisted(() => ({ blobs: {} as Record, downloads: [] as string[], failWith: null as null | { statusCode: number }, + // Fail only this blob name (with failWith's status). + failOn: null as null | string, })); vi.mock("@azure/storage-blob", () => ({ @@ -21,7 +23,7 @@ vi.mock("@azure/storage-blob", () => ({ getBlobClient: (name: string) => ({ downloadToFile: async (dest: string) => { state.downloads.push(name); - if (state.failWith) + if (state.failWith && (state.failOn === null || state.failOn === name)) throw Object.assign(new Error("blob failed"), state.failWith); if (!(name in state.blobs)) throw Object.assign(new Error("404"), { statusCode: 404 }); @@ -60,8 +62,10 @@ beforeEach(async () => { vi.stubEnv("EVALBOARD_LOCAL_RUNS_DIR", ""); root = await fs.mkdtemp(path.join(os.tmpdir(), "doctor-blob-")); vi.stubEnv("EVALBOARD_RUNS_DIR", root); + vi.stubEnv("EVALBOARD_EDITION", "internal"); state.downloads = []; state.failWith = null; + state.failOn = null; seedBlobs(); }); @@ -77,6 +81,31 @@ async function read() { const D = `${RUN}/skill-doctor`; +describe("ensureRunDoctorFile guard", () => { + test.each([ + ["../x"], + ["fixes-1/../status.json"], + ["a/b/c"], + [""], + ["/abs.json"], + ["a b.json"], + ])("rejects %j before any download", async (rel) => { + const { ensureRunDoctorFile } = await import("../blob"); + await expect(ensureRunDoctorFile("runs", RUN, rel, root)).rejects.toThrow( + /Invalid skill-doctor path/, + ); + expect(state.downloads).toEqual([]); + }); + + test("rejects a bad run id before any download", async () => { + const { ensureRunDoctorFile } = await import("../blob"); + await expect(ensureRunDoctorFile("runs", "../x", "status.json", root)).rejects.toThrow( + /Invalid runId/, + ); + expect(state.downloads).toEqual([]); + }); +}); + describe("readRunDoctor: what it fetches", () => { test("a done report: status, triage, fixes.json and the one patch", async () => { await read(); @@ -190,18 +219,57 @@ describe("readRunDoctor", () => { expect(state.downloads).toEqual([]); }); - test("a cold cache fetches the patch on demand, by name", async () => { + test("a cold cache: /api/file loads status + fixes.json, then the listed patch", async () => { const { resolveSafePath } = await import("../runs"); const abs = await resolveSafePath( RUN, "skill-doctor/fixes-1000001/example-grader-fix.patch", ); - expect(state.downloads).toEqual([ - `${D}/fixes-1000001/example-grader-fix.patch`, - ]); + expect(state.downloads).toContain(`${D}/status.json`); + expect(state.downloads).toContain(`${D}/fixes-1000001/fixes.json`); + expect(state.downloads).toContain(`${D}/fixes-1000001/example-grader-fix.patch`); expect(await fs.readFile(abs!, "utf-8")).toContain("diff --git"); }); + test("a patch fixes.json does not list, or another fixes dir, is refused", async () => { + const { resolveSafePath } = await import("../runs"); + expect(await resolveSafePath(RUN, "skill-doctor/fixes-1000001/made-up.patch")).toBeNull(); + expect( + await resolveSafePath(RUN, "skill-doctor/fixes-9/example-grader-fix.patch"), + ).toBeNull(); + expect(state.downloads.some((n) => n.endsWith("made-up.patch"))).toBe(false); + expect(state.downloads.some((n) => n.includes("fixes-9"))).toBe(false); + }); + + test("kill switch off or the OSS edition: nothing served, nothing fetched", async () => { + vi.stubEnv("EVALBOARD_TASK_ANALYSIS", "Off"); + let { resolveSafePath } = await import("../runs"); + const rel = "skill-doctor/fixes-1000001/example-grader-fix.patch"; + expect(await resolveSafePath(RUN, rel)).toBeNull(); + vi.resetModules(); + vi.stubEnv("EVALBOARD_TASK_ANALYSIS", ""); + vi.stubEnv("EVALBOARD_EDITION", ""); + ({ resolveSafePath } = await import("../runs")); + expect(await resolveSafePath(RUN, rel)).toBeNull(); + expect(state.downloads).toEqual([]); + }); + + test("a storage error on the patch: no throw, nothing on disk", async () => { + state.failWith = { statusCode: 500 }; + state.failOn = `${D}/fixes-1000001/example-grader-fix.patch`; + const err = vi.spyOn(console, "error").mockImplementation(() => {}); + const { resolveSafePath } = await import("../runs"); + const abs = await resolveSafePath( + RUN, + "skill-doctor/fixes-1000001/example-grader-fix.patch", + ); + // /api/file then answers 404 for the missing file. + expect(abs).not.toBeNull(); + await expect(fs.access(abs!)).rejects.toThrow(); + expect(err).toHaveBeenCalled(); + err.mockRestore(); + }); + test("only patch paths are served from skill-doctor/", async () => { const { resolveSafePath } = await import("../runs"); for (const bad of [ diff --git a/evalboard/lib/blob.ts b/evalboard/lib/blob.ts index 67ed24b9..16b1897a 100644 --- a/evalboard/lib/blob.ts +++ b/evalboard/lib/blob.ts @@ -326,8 +326,6 @@ export async function ensureRunReviewIndex( // fetched again once cached. `relPath` comes from a parsed, pattern-checked // field, and is checked here again before it becomes a blob name. 404s are // swallowed as for analysis.md. -const DOCTOR_REL_RE = /^[A-Za-z0-9._-]+(\/[A-Za-z0-9._-]+)?$/; - export async function ensureRunDoctorFile( container: string, runId: string, @@ -336,10 +334,8 @@ export async function ensureRunDoctorFile( refreshAfterMs?: number, ): Promise { assertValidId(runId, "runId"); - if ( - !DOCTOR_REL_RE.test(relPath) || - relPath.split("/").some((s) => s === "." || s === "..") - ) { + // One file directly under skill-doctor/, or one level down (the fixes dir). + if (!isValidTaskId(relPath) || relPath.split("/").length > 2) { throw new Error(`Invalid skill-doctor path: ${JSON.stringify(relPath)}`); } if (LOCAL_RUNS_DIR) return; diff --git a/evalboard/lib/doctor.ts b/evalboard/lib/doctor.ts index 180fcf7e..13991264 100644 --- a/evalboard/lib/doctor.ts +++ b/evalboard/lib/doctor.ts @@ -16,11 +16,11 @@ import { SKILLS_SOURCE } from "./sources"; export const DOCTOR_DIR = "skill-doctor"; const TRIAGE_FILE_RE = /^triage-[A-Za-z0-9._-]+\.json$/; -export const FIXES_DIR_RE = /^fixes(-[0-9]+)?$/; +const FIXES_DIR_RE = /^fixes(-[0-9]+)?$/; // Candidate ids name `.patch` inside the fixes dir. -export const PATCH_ID_RE = /^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$/; -// At most this many patches are fetched (and linked) per run. -export const MAX_PATCHES = 10; +const PATCH_ID_RE = /^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$/; +// At most this many patches per run are fetched, linked and served. +const MAX_PATCHES = 10; const DOCTOR_STATES = [ "queued", @@ -295,10 +295,11 @@ export function parseFixes(raw: string | null | undefined): DoctorFixes | null { } /** - * The candidates whose `.patch` the board fetches and links: one predicate - * for both, so a link never points at a patch that is not fetched. A patch - * exists only when fixes.json reports patch lines; the id must be a safe file - * name; at most MAX_PATCHES per run. + * The only patches the board touches. readRunDoctor fetches exactly these, + * the Fixes table links exactly these, and /api/file serves only these (via + * readRunDoctor, so there is one fetch path). A patch exists only when + * fixes.json reports patch lines; the id must be a safe file name; at most + * MAX_PATCHES per run. */ export function patchIds(fixes: DoctorFixes | null): Set { const ids = new Set(); diff --git a/evalboard/lib/runs.ts b/evalboard/lib/runs.ts index b3c8ef9b..a459d48a 100644 --- a/evalboard/lib/runs.ts +++ b/evalboard/lib/runs.ts @@ -27,8 +27,10 @@ import { parseTriage, patchIdFromPath, patchIds, + taskAnalysisEnabled, type RunDoctor, } from "./doctor"; +import { isInternal } from "./edition"; // Resolution order: // 1. EVALBOARD_LOCAL_RUNS_DIR — local mode, points at a coder_eval runs dir @@ -3031,16 +3033,28 @@ export async function resolveSafePath( if (parts[0] === DOCTOR_DIR) { // skill-doctor/ belongs to the doctor contract, so no experiment arm can // live there: a variant of that name would share the blob prefix with - // the report. Only a fix patch is served from it, fetched by name on - // demand (a cold cache or a direct link works), never by a listing. + // the report. Only a fix patch is served from it, and only one that + // this run's status + fixes.json list (patchIds). readRunDoctor is the + // one fetch path: it loads status and fixes.json when they are not + // cached and fetches exactly those patches, so a cold cache or a direct + // link works and a made-up name costs no blob call. Off with the panel. + if ( + !taskAnalysisEnabled( + isInternal, + source.id, + process.env.EVALBOARD_TASK_ANALYSIS, + ) + ) + return null; const patch = patchIdFromPath(parts.slice(1).join("/")); if (!patch) return null; - await ensureRunDoctorFile( - source.container, - runId, - `${patch.fixesDir}/${patch.id}.patch`, - dir, - ); + const doctor = await readRunDoctor(runId, source); + if ( + !doctor || + doctor.status.fixesDir !== patch.fixesDir || + !patchIds(doctor.fixes).has(patch.id) + ) + return null; } else if (parts[0] !== "activation" && isValidVariantId(parts[0]) && parts[1]) { if (!isValidId(parts[1])) return null; await ensureTaskDir(source.container, runId, parts[1], dir, parts[0]); From 41376a4b9d725f3655e69cd35e48441b3802b951 Mon Sep 17 00:00:00 2001 From: CarlesUIPath Date: Thu, 19 Nov 2026 16:20:08 +0000 Subject: [PATCH 04/13] feat(evalboard): Task failure analysis as a collapsed block shaped like the doctor's page MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The block now sits under the AI summary and opens like it: a bordered section with a header button, collapsed by default. The header carries " errors · proposed PRs ready", or the doctor's state when the report is not done. Inside, in the order of the skill-doctor page: the stats row and the failures-by-layer bar, Fix first (the ranked actions, with an "N PR ready" badge when an action's tasks have a pr-ready fix), then Skill failures: layer chips, search, Expand all / Collapse all, and one collapsed accordion per skill with a card per task (pills, diagnosis, the proposed PR row with outcome, patch, gates and cost, and the Fix / Failing check / Evidence / Passed before / PRs / Related / Owner list). Tasks stopped at a pre-run check collapse into one block per cause. The separate task and fix tables are gone. States, gating, link allowlists, patch rules and the kill switch are unchanged. Co-Authored-By: Claude Opus 5.5 --- evalboard/README.md | 9 +- .../__tests__/task-failure-analysis.test.tsx | 356 ++++--- evalboard/app/runs/[id]/page.tsx | 12 +- .../app/runs/[id]/task-failure-analysis.tsx | 986 ++++++++++++------ evalboard/lib/__tests__/doctor-blob.test.ts | 2 +- evalboard/lib/__tests__/doctor.test.ts | 87 +- .../fixtures/skill-doctor/status.json | 2 +- .../fixtures/skill-doctor/triage-1000001.json | 251 ++++- evalboard/lib/doctor.ts | 116 +-- 9 files changed, 1236 insertions(+), 585 deletions(-) diff --git a/evalboard/README.md b/evalboard/README.md index 8191d22c..5f027d4c 100644 --- a/evalboard/README.md +++ b/evalboard/README.md @@ -212,9 +212,10 @@ belongs to it. `readRunDoctor`). `status.json` is refetched after 60 s; the triage file it names, `/fixes.json` and up to 10 patches are cached for good. `/api/file` serves only the patches fixes.json lists, loading them through - the same reader on a cold cache. A done report gets the panel; when - `analysis.md` has a `## Findings` line, the AI summary stops there and the - panel replaces the rest. Other states keep the AI summary, with a status - line. `EVALBOARD_TASK_ANALYSIS=off` turns it off. + the same reader on a cold cache. The report is a collapsed block + under the AI summary, laid out like the doctor's own page (stats, failures by + layer, Fix first, per-skill failure cards). When the report is done and + `analysis.md` has a `## Findings` line, the AI summary stops there; other + states keep the AI summary and show the doctor's state in the block header. `EVALBOARD_TASK_ANALYSIS=off` turns it off. - Pass rows render green (`bg-green-50 text-green-700`), failures render red (`bg-red-50 text-red-700`), on a white background. diff --git a/evalboard/app/runs/[id]/__tests__/task-failure-analysis.test.tsx b/evalboard/app/runs/[id]/__tests__/task-failure-analysis.test.tsx index 269403ad..3f776ca4 100644 --- a/evalboard/app/runs/[id]/__tests__/task-failure-analysis.test.tsx +++ b/evalboard/app/runs/[id]/__tests__/task-failure-analysis.test.tsx @@ -1,9 +1,10 @@ import { readFileSync } from "node:fs"; import path from "node:path"; import { describe, expect, test } from "vitest"; -import { render, screen, within } from "@testing-library/react"; +import { fireEvent, render, screen, within } from "@testing-library/react"; import { - DoctorStatusLine, + digest, + groupBySkill, TaskFailureAnalysis, } from "../task-failure-analysis"; import { @@ -21,159 +22,268 @@ function status(patch: Record = {}) { return parseDoctorStatus(JSON.stringify({ ...JSON.parse(read("status.json")), ...patch }))!; } -function doctor(): RunDoctor { +function doctor(patch: Record = {}): RunDoctor { + const s = status(patch); return { - status: status(), - triage: parseTriage(read("triage-1000001.json")), + status: s, + triage: s.state === "done" ? parseTriage(read("triage-1000001.json")) : null, fixes: parseFixes(read("fixes-1000001/fixes.json")), }; } -describe("TaskFailureAnalysis", () => { - test("header: counts by layer, class and task state, history, cost, build link", () => { - render(); - const panel = screen.getByRole("region", { name: "Task Failure Analysis" }); - expect(panel).toHaveTextContent("3 failed, 2 diagnosed"); - expect(panel).toHaveTextContent("new 2"); - expect(panel).toHaveTextContent("known 1"); - expect(panel).toHaveTextContent("vs 2099-01-02_04-25-52: 2 new, 1 persisting, 0 fixed"); - expect(panel).toHaveTextContent("diagnosis $1.25"); - expect(panel).toHaveTextContent("grader 1"); - expect(panel).toHaveTextContent("failure 2"); - expect(screen.getByRole("link", { name: "build" })).toHaveAttribute( - "href", - "https://dev.azure.com/example-org/example-project/_build/results?buildId=1000001", +function renderBlock(d: RunDoctor = doctor(), sourceId = "skills") { + return render(); +} + +const block = () => screen.getByRole("region", { name: "Task failure analysis" }); +const header = () => within(block()).getAllByRole("button")[0]; +const openBlock = () => fireEvent.click(header()); +const skillButton = (skill: string) => + screen.getByRole("button", { name: new RegExp(`^▸?▾? ?${skill} `) }); +const openSkill = (skill: string) => fireEvent.click(skillButton(skill)); + +describe("digest", () => { + test("done: errors and PR-ready fixes", () => { + expect(digest(doctor())).toBe("5 errors · 1 proposed PR ready"); + }); + test("no PR-ready fix: the errors only", () => { + const d = doctor(); + d.fixes!.candidates = d.fixes!.candidates.filter((c) => c.outcome !== "pr-ready"); + expect(digest(d)).toBe("5 errors"); + expect(digest({ ...d, fixes: null })).toBe("5 errors"); + }); + test("not done: the state", () => { + expect(digest(doctor({ state: "running" }))).toMatch(/^running since /); + expect(digest(doctor({ state: "queued", started_at: null }))).toBe("queued"); + expect(digest(doctor({ state: "failed", reason: "diagnose timed out" }))).toBe( + "failed: diagnose timed out", ); - expect(screen.getByRole("link", { name: "nightly" })).toBeInTheDocument(); - }); - - test("task rows link to the task page; only real skills PR URLs are links", () => { - render(); - const links = screen.getAllByRole("link").map((a) => a.getAttribute("href")); - expect(links).toContain(`/runs/${RUN}/example-t1`); - expect(links).toContain("https://github.com/UiPath/skills/pull/123"); - expect(links.some((h) => h?.startsWith("javascript:"))).toBe(false); - expect(links.some((h) => h?.includes("example.invalid"))).toBe(false); - // The rejected URLs are still visible, as text. - expect(screen.getByText("javascript:alert(1)").tagName).toBe("SPAN"); - expect( - screen.getByText("https://github.com.example.invalid/UiPath/skills/pull/124").tagName, - ).toBe("SPAN"); - expect(screen.getByText("Example Owner, Second Owner")).toBeInTheDocument(); - }); - - test("the source travels in the task link", () => { - render(); - const links = screen.getAllByRole("link").map((a) => a.getAttribute("href")); - expect(links).toContain(`/runs/${RUN}/example-t1?src=gha`); - }); - - test("actions and fixes", () => { - render(); - expect(screen.getByText(/1 task diagnosed as a grader problem/)).toBeInTheDocument(); - const patch = screen.getByRole("link", { name: "+3/-1" }); - const href = new URL(patch.getAttribute("href")!, "http://x"); - expect(href.pathname).toBe("/api/file"); - expect(href.searchParams.get("run")).toBe(RUN); - expect(href.searchParams.get("path")).toBe( - "skill-doctor/fixes-1000001/example-grader-fix.patch", + expect(digest(doctor({ state: "skipped", reason: "ad-hoc run" }))).toBe("skipped: ad-hoc run"); + expect(digest(doctor({ state: "retracted" }))).toBe("the report for this run was retracted"); + expect(digest({ ...doctor(), triage: null })).toMatch(/triage file is missing/); + }); +}); + +describe("TaskFailureAnalysis block", () => { + test("collapsed by default, like the AI summary: header and digest only", () => { + renderBlock(); + expect(header()).toHaveAttribute("aria-expanded", "false"); + expect(header()).toHaveTextContent("Task failure analysis"); + expect(header()).toHaveTextContent("5 errors · 1 proposed PR ready"); + expect(screen.queryByText("Fix first")).toBeNull(); + expect(screen.queryByText("Skill failures")).toBeNull(); + }); + + test("not done: no body, the state and the build link", () => { + renderBlock(doctor({ state: "failed", reason: "boom" })); + expect(within(block()).queryByRole("button")).toBeNull(); + expect(block()).toHaveTextContent("failed: boom"); + expect(screen.getByRole("link", { name: "build" })).toBeInTheDocument(); + }); + + test("body order: stats, layer bar, Fix first, Skill failures", () => { + renderBlock(); + openBlock(); + const text = block().textContent!; + const at = (s: string) => text.indexOf(s); + expect(at("rows run")).toBeGreaterThan(-1); + expect(at("rows run")).toBeLessThan(at("Failures by layer")); + expect(at("Failures by layer")).toBeLessThan(at("Fix first")); + expect(at("Fix first")).toBeLessThan(at("Skill failures")); + // No separate fix table or task table any more. + expect(screen.queryByRole("table")).toBeNull(); + }); + + test("stats as on the doctor page", () => { + renderBlock(); + openBlock(); + const stats = screen.getByTestId("doctor-stats"); + expect(stats).toHaveTextContent("10rows run"); + expect(stats).toHaveTextContent("5failed rows"); + expect(stats).toHaveTextContent("3environment"); + expect(stats).toHaveTextContent("2fixable in the repo"); + expect(stats).toHaveTextContent("0agent mistakes"); + expect(screen.getByRole("img", { name: "Failures by layer" }).children).toHaveLength(3); + }); + + test("Fix first: ranked actions, allowlisted links, a PR-ready badge", () => { + renderBlock(); + openBlock(); + const fixFirst = screen.getByRole("region", { name: "Fix first" }); + const items = within(fixFirst).getAllByRole("listitem"); + expect(items).toHaveLength(3); + expect(items[0]).toHaveTextContent(/^3 · Environment/); + expect(within(items[1]).getByRole("link")).toHaveAttribute( + "href", + "https://github.com/UiPath/skills/pull/123", ); - const row = patch.closest("tr")!; - expect(within(row).getByText("replay pass · mutants 9/10 · review SEND")).toBeInTheDocument(); - expect(within(row).getByText("2")).toBeInTheDocument(); - expect(screen.getByText("needs-live-run")).toBeInTheDocument(); + // javascript: and look-alike hosts stay text. + expect(within(items[1]).getAllByRole("link")).toHaveLength(1); + expect(items[1]).toHaveTextContent("javascript:alert(1)"); + expect(within(items[1]).getByTestId("action-ready")).toHaveTextContent("1 PR ready"); + expect(within(items[0]).queryByTestId("action-ready")).toBeNull(); }); - test("untrusted build URLs are plain text", () => { - const d = doctor(); - d.status = status({ build_url: "javascript:alert(1)", trigger: "https://dev.azure.com.example.invalid/x" }); - render(); - expect(screen.queryByRole("link", { name: "build" })).toBeNull(); - expect(screen.queryByRole("link", { name: "nightly" })).toBeNull(); - expect(screen.getByText("build").tagName).toBe("SPAN"); + test("skills: grouped and sorted, all collapsed, with counts and layer dots", () => { + renderBlock(); + openBlock(); + const a = skillButton("uipath-example"); + const b = skillButton("uipath-example-two"); + expect(a).toHaveAttribute("aria-expanded", "false"); + expect(b).toHaveAttribute("aria-expanded", "false"); + expect(a).toHaveTextContent("2 failed · 1 proposed PR ready"); + expect(a).toHaveTextContent("1 environment"); + expect(a).toHaveTextContent("1 grader"); + expect(b).toHaveTextContent("3 failed"); + expect(b).not.toHaveTextContent("ready"); + expect(screen.queryAllByTestId("task-card")).toHaveLength(0); }); - test("row badges: nothing for new, the state for changed, nights for known", () => { - const d = doctor(); - d.triage!.tasks[0].state = "changed"; - d.triage!.tasks[0].nightsSeen = 1; - render(); - const e1 = screen.getByRole("link", { name: "Example e1" }).closest("td")!; - expect(within(e1).getByText("changed")).toBeInTheDocument(); - const t1 = screen.getByRole("link", { name: "Example t1" }).closest("td")!; - expect(within(t1).getByText("known, 3 nights")).toBeInTheDocument(); - const t2 = screen.getByRole("link", { name: "Example t2" }).closest("td")!; - expect(within(t2).queryByText(/new|nights/)).toBeNull(); - }); - - test("no base run: no history line", () => { - const d = doctor(); - d.triage!.summary.vsHistory = null; - render(); - expect(screen.queryByText(/^vs /)).toBeNull(); + test("env tasks collapse into one block per pre-run check", () => { + renderBlock(); + openBlock(); + openSkill("uipath-example-two"); + const groups = screen.getAllByTestId("env-group"); + expect(groups).toHaveLength(1); + const btn = within(groups[0]).getByRole("button"); + expect(btn).toHaveTextContent("2 tasks stopped at the pre-run check: connection example-conn"); + fireEvent.click(btn); + expect(within(groups[0]).getByText(/Reauthorize the connection/)).toBeInTheDocument(); + expect(within(groups[0]).getAllByRole("link").map((l) => l.textContent)).toEqual([ + "example-e2", + "example-e3", + ]); + // The one non-env task is a card. + expect(screen.getAllByTestId("task-card")).toHaveLength(1); }); - test("a task id the task page would reject is plain text, not a link", () => { - const d = doctor(); - d.triage!.tasks[0].taskId = "../escape"; - render(); - expect(screen.getByText("../escape").closest("a")).toBeNull(); - const links = screen.getAllByRole("link").map((a) => a.getAttribute("href")); - expect(links.some((h) => h?.includes("escape"))).toBe(false); + test("a card: task link, pills, diagnosis, proposed PR row, dl fields", () => { + renderBlock(); + openBlock(); + openSkill("uipath-example"); + const card = screen + .getAllByTestId("task-card") + .find((c) => c.textContent!.includes("example-t1"))!; + expect(within(card).getByRole("link", { name: "example-t1" })).toHaveAttribute( + "href", + `/runs/${RUN}/example-t1`, + ); + for (const pill of ["Grader", "Fixable: grader", "failure", "high confidence"]) { + expect(within(card).getByText(pill)).toBeInTheDocument(); + } + expect(card).toHaveTextContent("The grader expects a literal"); + const fix = within(card).getByTestId("fix-candidate"); + expect(fix).toHaveTextContent("Proposed PR: pr-ready"); + expect(fix).toHaveTextContent("replay pass · mutants 9/10 · review SEND"); + expect(fix).toHaveTextContent("$1.50"); + const patch = within(fix).getByRole("link", { name: "+3/-1" }); + const href = new URL(patch.getAttribute("href")!, "http://x"); + expect(href.pathname).toBe("/api/file"); + expect(href.searchParams.get("path")).toBe( + "skill-doctor/fixes-1000001/example-grader-fix.patch", + ); + const dts = within(card).getAllByRole("term").map((d) => d.textContent); + expect(dts).toEqual([ + "Fix", + "Failing check", + "Evidence", + "Passed before", + "Open PRs on these files", + "Related skill PRs", + "Related", + "Owner", + ]); + expect(within(card).getByText("literal").tagName).toBe("CODE"); + expect(within(card).getByRole("link", { name: "#123" })).toBeInTheDocument(); + expect(within(card).getByRole("link", { name: "#125" })).toBeInTheDocument(); + expect(within(card).getByText("javascript:alert(1)").tagName).toBe("SPAN"); + expect(within(card).getByText("Example Owner, Second Owner")).toBeInTheDocument(); }); - test("a patch whose id is not a safe file name shows its size without a link", () => { + test("a fix candidate that is not ready, and an unsafe patch id", () => { const d = doctor(); d.fixes!.candidates[0].id = "../bad"; - render(); - expect(screen.getByText("+3/-1").closest("a")).toBeNull(); + renderBlock(d); + openBlock(); + fireEvent.click(screen.getByRole("button", { name: "Expand all" })); + const t1 = screen.getAllByTestId("task-card").find((c) => c.textContent!.includes("example-t1"))!; + expect(within(t1).getByText("+3/-1").closest("a")).toBeNull(); + const t2 = screen.getAllByTestId("task-card").find((c) => c.textContent!.includes("example-t2"))!; + expect(within(t2).getByTestId("fix-candidate")).toHaveTextContent("Proposed PR: needs-live-run"); }); - test("no candidates", () => { + test("a task id the task page would reject is plain text", () => { const d = doctor(); - d.fixes!.candidates = []; - render(); - expect(screen.getByText("No fix candidates.")).toBeInTheDocument(); + d.triage!.tasks[1].taskId = "../escape"; + renderBlock(d); + openBlock(); + openSkill("uipath-example"); + expect(screen.getByText("../escape").closest("a")).toBeNull(); }); - test("no fixes.json: no Fixes table", () => { - const d = { ...doctor(), fixes: null }; - render(); - expect(screen.queryByText("Fixes")).toBeNull(); + test("Expand all / Collapse all", () => { + renderBlock(); + openBlock(); + fireEvent.click(screen.getByRole("button", { name: "Expand all" })); + expect(skillButton("uipath-example")).toHaveAttribute("aria-expanded", "true"); + expect(skillButton("uipath-example-two")).toHaveAttribute("aria-expanded", "true"); + fireEvent.click(screen.getByRole("button", { name: "Collapse all" })); + expect(skillButton("uipath-example")).toHaveAttribute("aria-expanded", "false"); }); -}); -describe("DoctorStatusLine", () => { - test("running: start time and build link", () => { - render(); - const line = screen.getByTestId("doctor-status-line"); - expect(line).toHaveTextContent("running since"); - expect(within(line).getByRole("link", { name: "build" })).toBeInTheDocument(); + test("layer chips filter, and the last one off shows all again", () => { + renderBlock(); + openBlock(); + const chips = within(screen.getByRole("toolbar", { name: "Filter failures" })); + fireEvent.click(chips.getByRole("button", { name: "Skill" })); + expect(chips.getByRole("button", { name: "Skill" })).toHaveAttribute("aria-pressed", "true"); + expect(chips.getByRole("button", { name: "Grader" })).toHaveAttribute("aria-pressed", "false"); + expect(screen.queryByRole("button", { name: /^▸ uipath-example 2/ })).toBeNull(); + expect(skillButton("uipath-example-two")).toHaveTextContent("1 failed"); + fireEvent.click(chips.getByRole("button", { name: "Skill" })); + expect(chips.getByRole("button", { name: "Grader" })).toHaveAttribute("aria-pressed", "true"); + expect(skillButton("uipath-example")).toHaveTextContent("2 failed"); }); - test("queued", () => { - render(); - expect(screen.getByTestId("doctor-status-line")).toHaveTextContent("queued since"); + test("search filters and opens the matches", () => { + renderBlock(); + openBlock(); + fireEvent.change(screen.getByRole("searchbox", { name: "Search failures" }), { + target: { value: "removed flag" }, + }); + expect(screen.queryByRole("button", { name: /uipath-example 2 failed/ })).toBeNull(); + expect(skillButton("uipath-example-two")).toHaveAttribute("aria-expanded", "true"); + expect(screen.getAllByTestId("task-card")).toHaveLength(1); + fireEvent.change(screen.getByRole("searchbox"), { target: { value: "zzz-nothing" } }); + expect(screen.getByText("No failures match this filter.")).toBeInTheDocument(); }); - test("failed and skipped show the reason", () => { - const { rerender } = render( - , - ); - expect(screen.getByTestId("doctor-status-line")).toHaveTextContent("failed: diagnose timed out"); - rerender(); - expect(screen.getByTestId("doctor-status-line")).toHaveTextContent("skipped: ad-hoc run"); + test("untrusted build URLs are plain text", () => { + renderBlock(doctor({ state: "running", build_url: "javascript:alert(1)" })); + expect(screen.queryByRole("link", { name: "build" })).toBeNull(); + expect(screen.getByText("build").tagName).toBe("SPAN"); }); - test("done without a triage says why there is no panel", () => { - render(); - expect(screen.getByTestId("doctor-status-line")).toHaveTextContent( - "report done, but its triage file is missing or did not parse", + test("the source travels in task links", () => { + renderBlock(doctor(), "gha"); + openBlock(); + openSkill("uipath-example"); + expect(screen.getByRole("link", { name: "example-t1" })).toHaveAttribute( + "href", + `/runs/${RUN}/example-t1?src=gha`, ); }); +}); - test("retracted is a one-line note", () => { - render(); - expect(screen.getByTestId("doctor-status-line")).toHaveTextContent("retracted"); +describe("groupBySkill", () => { + test("cards sorted by layer rank, env tasks grouped", () => { + const g = groupBySkill(doctor(), null, ""); + expect(g.map((x) => x.skill)).toEqual(["uipath-example", "uipath-example-two"]); + expect(g[0].cards.map((t) => t.taskId)).toEqual(["example-t1"]); + expect(g[0].envGroups).toEqual([ + { cause: "connection example-conn", tasks: [expect.objectContaining({ taskId: "example-e1" })] }, + ]); + expect(g[0].ready).toBe(1); + expect(g[1].ready).toBe(0); }); }); diff --git a/evalboard/app/runs/[id]/page.tsx b/evalboard/app/runs/[id]/page.tsx index 34759d79..9abb9b76 100644 --- a/evalboard/app/runs/[id]/page.tsx +++ b/evalboard/app/runs/[id]/page.tsx @@ -25,10 +25,7 @@ import { sourceById } from "@/lib/sources"; import { scalarParam } from "@/app/_lib/source-param"; import { fmtRunTime } from "@/lib/format"; import { AnalysisPanel } from "./analysis-panel"; -import { - DoctorStatusLine, - TaskFailureAnalysis, -} from "./task-failure-analysis"; +import { TaskFailureAnalysis } from "./task-failure-analysis"; import { planAnalysis, taskAnalysisEnabled } from "@/lib/doctor"; import { RefreshButton } from "./refresh-button"; import { RunView } from "./run-view"; @@ -100,7 +97,7 @@ export default async function RunPage({ const reviewsByTask = reviewIndex ? indexByTask(reviewIndex) : undefined; const human = fmtRunTime(id); // A done report replaces analysis.md's Findings and everything below it; - // otherwise the AI summary is unchanged, plus a status line. + // otherwise the AI summary is unchanged. The doctor block sits under it. const plan = planAnalysis(analysis, doctor); const series = isInternal ? seriesForRun(source.id, { @@ -174,10 +171,9 @@ export default async function RunPage({
{plan.summary && } - {plan.statusLine && } - {plan.panel && ( + {plan.doctor && ( diff --git a/evalboard/app/runs/[id]/task-failure-analysis.tsx b/evalboard/app/runs/[id]/task-failure-analysis.tsx index 0e74243e..4d2354e4 100644 --- a/evalboard/app/runs/[id]/task-failure-analysis.tsx +++ b/evalboard/app/runs/[id]/task-failure-analysis.tsx @@ -1,7 +1,9 @@ +"use client"; + import Link from "next/link"; -import type { ReactNode } from "react"; +import { useMemo, useState, type ReactNode } from "react"; import { withSource } from "@/app/_lib/source-param"; -import { fmtTimestamp, fmtUsd, humanizeTaskId } from "@/lib/format"; +import { fmtTimestamp, fmtUsd } from "@/lib/format"; import { DOCTOR_DIR, isAdoUrl, @@ -9,14 +11,85 @@ import { isSkillsPrUrl, patchIds, type DoctorFix, + type DoctorPr, type DoctorStatus, type DoctorTask, type RunDoctor, } from "@/lib/doctor"; -// The skill-doctor report for one run, drawn natively from its JSON (never -// the doctor's own page.html). Every string in it is plain text; the only -// links are to this board's task pages and to URLs of an exact known shape. +// The skill-doctor report for one run, drawn natively from its JSON in the +// order of the doctor's own page (stats → failures by layer → Fix first → +// skill failures), never by embedding that page. Every string is plain text; +// the only links are this board's task pages and URLs of an exact known shape. + +// Layer order, labels and colors, as on the doctor's page. +export const LAYERS: Record = { + env: { label: "Environment", color: "#56708f" }, + grader: { label: "Grader", color: "#b07410" }, + task: { label: "Task setup", color: "#7a4fb0" }, + agent: { label: "Agent mistake", color: "#c2413a" }, + skill: { label: "Skill", color: "#d0661c" }, + upstream: { label: "Upstream (CLI)", color: "#2f6fd0" }, + flaky: { label: "Flaky", color: "#7d8794" }, + undiagnosed: { label: "Not diagnosed", color: "#9aa5b3" }, +}; +const ORDER = Object.keys(LAYERS); +const UNKNOWN_LAYER = { label: "Other", color: "#9aa5b3" }; +const layerOf = (k: string) => LAYERS[k] ?? { ...UNKNOWN_LAYER, label: k }; +const rank = (k: string) => { + const i = ORDER.indexOf(k); + return i === -1 ? ORDER.length : i; +}; + +const SOLVABLE: Record = { + "env-fix": "Fix the tenant", + "yes-grader": "Fixable: grader", + "yes-task": "Fixable: task", + "yes-skill": "Fixable: skill", + "upstream-report": "Upstream report", + no: "Not a repo fix", +}; +const isFixable = (t: DoctorTask) => /^yes/.test(t.solvable ?? ""); +const isReady = (c: DoctorFix) => c.outcome === "pr-ready"; + +// ---------- text ---------- + +/** + * Plain text with `code` spans, and URLs turned into links only when they + * pass an allowlist (a UiPath/skills PR or a dev.azure.com build). + */ +export function RichText({ text }: { text: string }) { + const out: ReactNode[] = []; + text.split(/(`[^`]+`)/).forEach((part, i) => { + if (part.length > 2 && part.startsWith("`") && part.endsWith("`")) { + out.push( + + {part.slice(1, -1)} + , + ); + return; + } + part.split(/(https:\/\/[^\s)]+)/).forEach((seg, j) => { + if (!seg) return; + const url = seg.replace(/[.,;:]+$/, ""); + if (isSkillsPrUrl(url) || isAdoUrl(url)) { + out.push( + + {url} + , + seg.slice(url.length), + ); + } else out.push(seg); + }); + }); + return <>{out}; +} function ExternalLink({ href, @@ -41,148 +114,60 @@ function ExternalLink({ ); } -function prNumber(url: string): string { +function prLabel(url: string): string { const m = /\/pull\/(\d+)$/.exec(url); return m ? `#${m[1]}` : url; } -function Counts({ - label, - counts, -}: { - label: string; - counts: Record; -}) { - const entries = Object.entries(counts).sort((a, b) => b[1] - a[1]); - if (entries.length === 0) return null; +function PrList({ prs }: { prs: DoctorPr[] }) { return ( -
- {label} - {entries.map(([k, n]) => ( - - {k} {n} + <> + {prs.map((p, i) => ( + + {i > 0 && "; "} + + {isSkillsPrUrl(p.url) ? prLabel(p.url) : p.url} + + {p.title && ` ${p.title}`} + ({p.state}) ))} -
+ ); } -/** One line under the AI summary when the full panel is not shown. */ -export function DoctorStatusLine({ status }: { status: DoctorStatus }) { - const build = status.buildUrl ? ( - <> - {" · "} - - build - - - ) : null; - let body: ReactNode; +// ---------- digest + state ---------- + +export function digest(doctor: RunDoctor): string { + const { status, triage, fixes } = doctor; switch (status.state) { case "queued": case "running": - body = ( - <> - {status.state} - {status.startedAt && - ` since ${fmtTimestamp(status.startedAt)}`} - {build} - - ); - break; + return status.startedAt + ? `${status.state} since ${fmtTimestamp(status.startedAt)}` + : status.state; case "failed": case "skipped": - body = ( - <> - {status.state} - {status.reason && `: ${status.reason}`} - {build} - - ); - break; + return status.reason ? `${status.state}: ${status.reason}` : status.state; case "retracted": - body = <>the report for this run was retracted; - break; - case "done": - body = ( - <> - report done, but its triage file is missing or did not - parse - {build} - - ); - break; + return "the report for this run was retracted"; + case "done": { + if (!triage) return "report done, but its triage file is missing or did not parse"; + const ready = (fixes?.candidates ?? []).filter(isReady).length; + const errors = `${triage.rowsFailed} error${triage.rowsFailed === 1 ? "" : "s"}`; + return ready + ? `${errors} · ${ready} proposed PR${ready === 1 ? "" : "s"} ready` + : errors; + } } - return ( -

- skill-doctor: {body} -

- ); } -function TaskRow({ - t, - runId, - sourceId, -}: { - t: DoctorTask; - runId: string; - sourceId: string; -}) { - return ( - - - {isLinkableTaskId(t.taskId) ? ( - - {humanizeTaskId(t.taskId)} - - ) : ( - t.taskId - )} - {t.state && t.state !== "new" && ( - - {t.state} - {t.nightsSeen > 1 && `, ${t.nightsSeen} nights`} - - )} - - {t.skill} - - {t.layer} - {t.cls && ( -
{t.cls}
- )} - - {t.confidence ?? "—"} - {t.why ?? "—"} - {t.fix ?? "—"} - - {t.existingPrs.length === 0 - ? "—" - : t.existingPrs.map((p, i) => ( -
- - {isSkillsPrUrl(p.url) ? prNumber(p.url) : p.url} - {" "} - - {p.state} - -
- ))} - - - {t.owners.length ? t.owners.join(", ") : "—"} - - +// ---------- fixes ---------- + +function patchHref(runId: string, sourceId: string, dir: string, id: string) { + return withSource( + `/api/file?run=${encodeURIComponent(runId)}&path=${encodeURIComponent(`${DOCTOR_DIR}/${dir}/${id}.patch`)}`, + sourceId, ); } @@ -195,102 +180,541 @@ function gateText(f: DoctorFix): string { return parts.length ? parts.join(" · ") : "—"; } -function PatchCell({ fix, href }: { fix: DoctorFix; href: string | null }) { +interface FixCtx { + runId: string; + sourceId: string; + dir: string | null; + // The same set readRunDoctor fetches and /api/file serves. + linked: Set; +} + +function PatchCell({ fix, ctx }: { fix: DoctorFix; ctx: FixCtx }) { if (!fix.patchLines) return <>—; const text = `+${fix.patchLines.added}/-${fix.patchLines.removed}`; - if (!href) return <>{text}; + if (!ctx.dir || !ctx.linked.has(fix.id)) return {text}; return ( - + {text} ); } -function FixesTable({ +// ---------- grouping ---------- + +export interface SkillGroup { + skill: string; + cards: DoctorTask[]; + envGroups: { cause: string; tasks: DoctorTask[] }[]; + layerCounts: [string, number][]; + // Tasks with a proposed PR ready: a pr-ready fix candidate or a doctor PR. + ready: number; + total: number; +} + +export function fixesForTask(doctor: RunDoctor, taskId: string): DoctorFix[] { + return (doctor.fixes?.candidates ?? []).filter( + (c) => c.taskId === taskId || c.covers.includes(taskId), + ); +} + +/** How many of these tasks have a pr-ready fix candidate. */ +export function readyTasks(doctor: RunDoctor, taskIds: string[]): number { + return taskIds.filter((id) => fixesForTask(doctor, id).some(isReady)).length; +} + +/** Visible failures by skill, as the doctor's page groups them. */ +export function groupBySkill( + doctor: RunDoctor, + active: Set | null, + query: string, +): SkillGroup[] { + const q = query.trim().toLowerCase(); + const vis = (doctor.triage?.tasks ?? []).filter( + (t) => + (!active || active.has(t.layer)) && + (!q || JSON.stringify(t).toLowerCase().includes(q)), + ); + const by = new Map(); + for (const t of vis) { + const k = t.skill || "(no skill)"; + by.set(k, [...(by.get(k) ?? []), t]); + } + return [...by.keys()] + .sort((a, b) => a.localeCompare(b)) + .map((skill) => { + const items = by.get(skill)!; + const groups = new Map(); + const cards: DoctorTask[] = []; + for (const t of items) { + if (t.envGroup) groups.set(t.envGroup, [...(groups.get(t.envGroup) ?? []), t]); + else cards.push(t); + } + cards.sort((a, b) => rank(b.layer) - rank(a.layer) || a.taskId.localeCompare(b.taskId)); + const lc = new Map(); + for (const t of items) lc.set(t.layer, (lc.get(t.layer) ?? 0) + 1); + return { + skill, + cards, + envGroups: [...groups.entries()].map(([cause, tasks]) => ({ cause, tasks })), + layerCounts: [...lc.entries()].sort((a, b) => rank(a[0]) - rank(b[0])), + ready: items.filter( + (t) => isSkillsPrUrl(t.doctorPr) || fixesForTask(doctor, t.taskId).some(isReady), + ).length, + total: items.length, + }; + }); +} + +// ---------- cards ---------- + +function Pill({ children, color }: { children: ReactNode; color?: string }) { + return ( + + {children} + + ); +} + +function TaskId({ id, ctx }: { id: string; ctx: FixCtx }) { + if (!isLinkableTaskId(id)) return {id}; + return ( + + {id} + + ); +} + +function TaskCard({ t, doctor, ctx }: { t: DoctorTask; doctor: RunDoctor; ctx: FixCtx }) { + const layer = layerOf(t.layer); + const cands = fixesForTask(doctor, t.taskId); + const rows: [string, ReactNode, boolean?][] = []; + if (t.fix) rows.push(["Fix", ]); + if (t.criterion) rows.push(["Failing check", ]); + if (t.evidence) rows.push(["Evidence", , true]); + if (t.passedBefore) rows.push(["Passed before", ]); + if (t.existingPrs.length) rows.push(["Open PRs on these files", ]); + if (t.relatedSkillPrs.length) + rows.push(["Related skill PRs", ]); + if (t.related) rows.push(["Related", ]); + if (t.owners.length) rows.push(["Owner", t.owners.join(", ")]); + return ( +
+
+ + {layer.label} + {t.solvable && ( + {SOLVABLE[t.solvable.split(" ")[0]] ?? t.solvable} + )} + {t.runStatus && ( + {t.runStatus.replace(/_/g, " ").toLowerCase()} + )} + {t.confidence && {t.confidence} confidence} +
+ {t.why && ( +

+ +

+ )} + + {rows.length > 0 && ( +
+ {rows.map(([k, v, mono]) => ( +
+
{k}
+
+ {v} +
+
+ ))} +
+ )} +
+ ); +} + +// page.html's "Proposed PR" row: the doctor PR when there is one, else each +// fix candidate for this task (outcome, patch, gates, cost), else "none yet" +// for a repo-fixable failure. +function ProposedFix({ t, cands, ctx }: { t: DoctorTask; cands: DoctorFix[]; ctx: FixCtx }) { + const box = "inline-flex flex-wrap items-baseline gap-x-2 rounded-md border px-3 py-1 text-xs"; + if (isSkillsPrUrl(t.doctorPr) || cands.length) { + return ( +
+ {isSkillsPrUrl(t.doctorPr) && ( + + Proposed PR: {prLabel(t.doctorPr)} ↗ + + )} + {cands.map((c) => ( + + + Proposed PR: {c.outcome} + + {c.patchLines && ( + + patch + + )} + {gateText(c)} + {fmtUsd(c.costUsd)} + {c.title && {c.title}} + + ))} +
+ ); + } + if (!isFixable(t)) return null; + return ( +
+ + Proposed PR: none yet + +
+ ); +} + +function EnvGroup({ cause, tasks, ctx }: { cause: string; tasks: DoctorTask[]; ctx: FixCtx }) { + const [open, setOpen] = useState(false); + const first = tasks[0]; + return ( +
+ + {open && ( +
+ {first.why && ( +

+ +

+ )} + {first.fix && ( +

+ Fix: +

+ )} +
    + {tasks.map((t, i) => ( +
  • + +
  • + ))} +
+
+ )} +
+ ); +} + +function SkillAccordion({ + g, + open, + onToggle, doctor, - runId, - sourceId, + ctx, }: { + g: SkillGroup; + open: boolean; + onToggle: () => void; doctor: RunDoctor; - runId: string; - sourceId: string; + ctx: FixCtx; }) { - const fixes = doctor.fixes; - const dir = doctor.status.fixesDir; - if (!fixes || !dir) return null; - // The same set readRunDoctor fetches and /api/file serves. - const linked = patchIds(fixes); return ( -
-

- Fixes - - {fixes.fixMode && `mode ${fixes.fixMode} · `} - {fmtUsd(fixes.costUsd)} - {fixes.totalCapUsd != null && - ` of ${fmtUsd(fixes.totalCapUsd)} cap`} +
+

- {fixes.candidates.length === 0 ? ( -

No fix candidates.

- ) : ( - - - - - - - - - - - - - {fixes.candidates.map((f) => ( - - - - - - - - + {g.skill} + + {g.total} failed + {g.ready > 0 && + ` · ${g.ready} proposed PR${g.ready === 1 ? "" : "s"} ready`} + + + {g.layerCounts.map(([k, n]) => ( + + + {n} {layerOf(k).label.toLowerCase()} + + ))} + + + {open && ( +
+ {g.cards.map((t, i) => ( + + ))} + {g.envGroups.map((e) => ( + + ))} +
+ )} + + ); +} + +// ---------- body ---------- + +const chip = (pressed: boolean) => + `rounded-full border px-3 py-0.5 text-xs ${pressed ? "border-gray-900 bg-gray-900 text-white" : "border-gray-200 bg-white text-gray-800"}`; + +function Body({ doctor, ctx }: { doctor: RunDoctor; ctx: FixCtx }) { + const triage = doctor.triage!; + const tasks = triage.tasks; + const counts = useMemo(() => { + const m = new Map(); + for (const t of tasks) m.set(t.layer, (m.get(t.layer) ?? 0) + 1); + return [...m.entries()].sort((a, b) => rank(a[0]) - rank(b[0])); + }, [tasks]); + const present = counts.map(([k]) => k); + // null: every layer shown (all chips pressed), as on the doctor's page. + const [active, setActive] = useState | null>(null); + const [query, setQuery] = useState(""); + const [openSkills, setOpenSkills] = useState>(new Set()); + const groups = groupBySkill(doctor, active, query); + const n = tasks.length; + const count = (k: string) => counts.find(([l]) => l === k)?.[1] ?? 0; + const stats: [number | null, string][] = [ + [triage.rowsRun, "rows run"], + [n, "failed rows"], + [count("env"), "environment"], + [tasks.filter(isFixable).length, "fixable in the repo"], + [count("agent"), "agent mistakes"], + ]; + const clickChip = (k: string) => { + setActive((cur) => { + if (!cur) return new Set([k]); + const next = new Set(cur); + if (next.has(k)) next.delete(k); + else next.add(k); + return next.size === 0 || present.every((p) => next.has(p)) ? null : next; + }); + }; + return ( +
+
+ {stats + .filter(([v]) => v !== null) + .map(([v, l]) => ( +
+ {v} + {l} +
+ ))} +
+ + {n > 0 && ( +
+
+ Failures by layer +
+
+ {counts.map(([k, c]) => ( + + ))} +
+
+ {counts.map(([k, c]) => ( + + + {layerOf(k).label} {c} + ))} -
-
OutcomeTaskRowsPatchGatesCost
- {f.outcome} - {f.title && ( -
- {f.title} -
- )} -
- {humanizeTaskId(f.taskId)} - - {f.covers.length} - - - {gateText(f)} - {fmtUsd(f.costUsd)} -
+
+ + )} + + {triage.actions.length > 0 && ( +
+

Fix first

+
    + {triage.actions.map((a, i) => { + const ready = readyTasks(doctor, a.taskIds); + return ( +
  1. + + {a.count} + + {" · "} + + {ready > 0 && ( + + {ready} PR ready + + )} +
  2. + ); + })} +
+
)} + +
+

Skill failures

+
+ + +
+
+ + Show + + {present.map((k) => { + const pressed = !active || active.has(k); + return ( + + ); + })} + setQuery(e.target.value)} + placeholder="Search task, cause, PR…" + aria-label="Search failures" + className="min-w-0 max-w-xs flex-1 rounded-md border border-gray-200 px-2.5 py-1 text-sm" + /> +
+
+ {groups.length === 0 ? ( +

No failures match this filter.

+ ) : ( + groups.map((g) => ( + + setOpenSkills((cur) => { + const next = new Set(cur); + if (next.has(g.skill)) next.delete(g.skill); + else next.add(g.skill); + return next; + }) + } + doctor={doctor} + ctx={ctx} + /> + )) + )} +
+
); } +// ---------- block ---------- + +function BuildLinks({ status }: { status: DoctorStatus }) { + return ( + <> + {status.buildUrl && ( + <> + {" · "} + + build + + + )} + {status.trigger && ( + <> + {" · "} + + nightly + + + )} + + ); +} + +/** + * The Task Failure Analysis block: shaped like the AI summary panel (bordered + * section, header button, collapsed by default), right under it. The header + * carries a one-line digest; only a done report with a triage has a body. + */ export function TaskFailureAnalysis({ doctor, runId, @@ -300,111 +724,69 @@ export function TaskFailureAnalysis({ runId: string; sourceId: string; }) { - const { status, triage } = doctor; - if (!triage) return null; - // Per-task triage state (new / changed / known), counted for the header. - const byState: Record = {}; - for (const t of triage.tasks) { - if (t.state) byState[t.state] = (byState[t.state] ?? 0) + 1; - } - const vh = triage.summary.vsHistory; + const [open, setOpen] = useState(false); + const hasBody = doctor.status.state === "done" && doctor.triage !== null; + const ctx: FixCtx = { + runId, + sourceId, + dir: doctor.status.fixesDir, + linked: patchIds(doctor.fixes), + }; + const label = ( + + Task failure analysis + + ); + const line = ( + + {digest(doctor)} + + ); return (
-
-
-

- Task Failure Analysis -

- - skill-doctor · {triage.summary.rowsFailed} failed,{" "} - {triage.summary.rowsDiagnosed} diagnosed · diagnosis{" "} - {fmtUsd(status.costUsd ?? triage.costUsd)} - {status.buildUrl && ( - <> - {" · "} - - build - - - )} - {status.trigger && ( + {hasBody ? ( + + ) : ( +
+ {label} + {line} + + + +
+ )} + {hasBody && open && ( + <> +
+ skill-doctor · diagnosis {fmtUsd(doctor.status.costUsd)} + {doctor.fixes && ( <> - {" · "} - - nightly - + {" · fixes "} + {fmtUsd(doctor.fixes.costUsd)} + {doctor.fixes.fixMode && ` (${doctor.fixes.fixMode})`} )} - -
- - - - {vh && ( -

- vs {vh.baseRun}: {vh.new} new, {vh.persisting}{" "} - persisting, {vh.fixed} fixed -

- )} - {triage.disclaimer && ( -

- {triage.disclaimer} -

- )} -
- - {triage.actions.length > 0 && ( -
-

- Actions -

-
    - {triage.actions.map((a, i) => ( -
  • - [{a.kind}]{" "} - {a.text} -
  • - ))} -
-
+ +
+ + )} - - {/* Collapsed by default: a nightly can list 80+ rows, and the - task grid below should stay in reach. */} -
- - Tasks ({triage.tasks.length}) - - - - - - - - - - - - - - - - {triage.tasks.map((t, i) => ( - - ))} - -
TaskSkillLayerConfidenceWhyFixExisting PRsOwners
-
- -
); } diff --git a/evalboard/lib/__tests__/doctor-blob.test.ts b/evalboard/lib/__tests__/doctor-blob.test.ts index bd6f10d2..4f161004 100644 --- a/evalboard/lib/__tests__/doctor-blob.test.ts +++ b/evalboard/lib/__tests__/doctor-blob.test.ts @@ -181,7 +181,7 @@ describe("readRunDoctor", () => { const { readRunDoctor } = await import("../runs"); const d = await readRunDoctor(RUN); expect(d?.status.state).toBe("done"); - expect(d?.triage?.tasks).toHaveLength(3); + expect(d?.triage?.tasks).toHaveLength(5); expect(d?.fixes?.candidates).toHaveLength(2); }); diff --git a/evalboard/lib/__tests__/doctor.test.ts b/evalboard/lib/__tests__/doctor.test.ts index 0143645e..8108941f 100644 --- a/evalboard/lib/__tests__/doctor.test.ts +++ b/evalboard/lib/__tests__/doctor.test.ts @@ -133,15 +133,38 @@ describe("parseDoctorStatus", () => { }); describe("parseTriage / parseFixes", () => { - test("reads the fixture", () => { + test("reads the fixture: the card fields", () => { const t = parseTriage(TRIAGE)!; - expect(t.tasks.map((x) => x.taskId)).toEqual(["example-e1", "example-t1", "example-t2"]); - expect(t.summary.byLayer).toEqual({ env: 1, grader: 1, skill: 1 }); - expect(t.tasks[1].nightsSeen).toBe(3); - expect(t.tasks.map((x) => x.state)).toEqual(["new", "known", "new"]); - expect(t.summary.vsHistory?.baseRun).toBe("2099-01-02_04-25-52"); - expect(t.tasks[1].existingPrs).toHaveLength(3); - expect(t.actions).toHaveLength(2); + expect(t.tasks.map((x) => x.taskId)).toEqual([ + "example-e1", + "example-t1", + "example-t2", + "example-e2", + "example-e3", + ]); + expect(t.rowsRun).toBe(10); + expect(t.rowsFailed).toBe(5); + expect(t.actions[0]).toEqual({ + count: 3, + text: expect.stringContaining("Reauthorize"), + taskIds: ["example-e1", "example-e2", "example-e3"], + }); + const t1 = t.tasks[1]; + expect(t1).toMatchObject({ + layer: "grader", + solvable: "yes-grader", + runStatus: "FAILURE", + confidence: "high", + criterion: "file_contains", + passedBefore: "2099-01-01_04-20-00 (claude-code)", + related: "example-t3 fails the same check", + owners: ["Example Owner", "Second Owner"], + envGroup: null, + doctorPr: null, + }); + expect(t1.existingPrs).toHaveLength(3); + expect(t1.relatedSkillPrs[0].url).toBe("https://github.com/UiPath/skills/pull/125"); + expect(t.tasks[0].envGroup).toBe("connection example-conn"); }); test("schema_version 2 or a broken triage is absent", () => { @@ -150,18 +173,6 @@ describe("parseTriage / parseFixes", () => { expect(parseTriage(JSON.stringify({ schema_version: 1 }))).toBeNull(); }); - test("vs_history without a base run is not shown", () => { - const doc = JSON.parse(TRIAGE); - doc.summary.vs_history.base_run = null; - expect(parseTriage(JSON.stringify(doc))!.summary.vsHistory).toBeNull(); - }); - - test("an unknown task state is null", () => { - const doc = JSON.parse(TRIAGE); - doc.tasks[0].state = "persisting"; - expect(parseTriage(JSON.stringify(doc))!.tasks[0].state).toBeNull(); - }); - test("fixes.json with schema_version 2, or none, is absent", () => { const doc = JSON.parse(FIXES); expect(parseFixes(JSON.stringify({ ...doc, schema_version: 2 }))).toBeNull(); @@ -254,48 +265,44 @@ describe("taskAnalysisEnabled (gating + kill switch)", () => { }); describe("planAnalysis (states)", () => { - test("absent: today's view, no status line", () => { - expect(planAnalysis(MD, null)).toEqual({ summary: MD, panel: null, statusLine: null }); - expect(planAnalysis(null, null)).toEqual({ summary: null, panel: null, statusLine: null }); + test("absent: today's view, no block", () => { + expect(planAnalysis(MD, null)).toEqual({ summary: MD, doctor: null }); + expect(planAnalysis(null, null)).toEqual({ summary: null, doctor: null }); }); - test("done with a split: summary keeps the top, the panel replaces the rest", () => { + test("done with a split: summary keeps the top, the block replaces the rest", () => { const d = doctor(); const p = planAnalysis(MD, d); expect(p.summary).toBe("# Analysis\n\n## TL;DR\nTop part.\n\n"); - expect(p.summary).not.toContain("Findings"); - expect(p.panel).toBe(d); - expect(p.statusLine).toBeNull(); + expect(p.doctor).toBe(d); }); - test("done without a split: the full analysis, then the panel", () => { + test("done without a split: the full analysis, then the block", () => { const md = "# A\n\n## TL;DR\nonly\n"; const d = doctor(); - expect(planAnalysis(md, d)).toEqual({ summary: md, panel: d, statusLine: null }); + expect(planAnalysis(md, d)).toEqual({ summary: md, doctor: d }); }); - test("done without analysis.md: the panel alone", () => { + test("done without analysis.md: the block alone", () => { const d = doctor(); - expect(planAnalysis(null, d)).toEqual({ summary: null, panel: d, statusLine: null }); + expect(planAnalysis(null, d)).toEqual({ summary: null, doctor: d }); }); - test("an analysis.md that starts with Findings: no summary, the panel", () => { + test("an analysis.md that starts with Findings: no summary", () => { const d = doctor(); - expect(planAnalysis("## Findings\nx\n", d)).toEqual({ summary: null, panel: d, statusLine: null }); + expect(planAnalysis("## Findings\nx\n", d)).toEqual({ summary: null, doctor: d }); }); - test("done but the triage did not load: no panel", () => { + test("done but the triage did not load: the full analysis", () => { const d = { ...doctor(), triage: null }; - expect(planAnalysis(MD, d)).toEqual({ summary: MD, panel: null, statusLine: d.status }); + expect(planAnalysis(MD, d)).toEqual({ summary: MD, doctor: d }); }); test.each(["queued", "running", "failed", "skipped", "retracted"])( - "%s: the full analysis (Findings included) plus a status line", + "%s: the full analysis (Findings included) and the block", (state) => { - const p = planAnalysis(MD, doctor({ state, reason: "why" })); - expect(p.summary).toBe(MD); - expect(p.panel).toBeNull(); - expect(p.statusLine?.state).toBe(state); + const d = doctor({ state, reason: "why" }); + expect(planAnalysis(MD, d)).toEqual({ summary: MD, doctor: d }); }, ); }); diff --git a/evalboard/lib/__tests__/fixtures/skill-doctor/status.json b/evalboard/lib/__tests__/fixtures/skill-doctor/status.json index 316a3341..d7031bcf 100644 --- a/evalboard/lib/__tests__/fixtures/skill-doctor/status.json +++ b/evalboard/lib/__tests__/fixtures/skill-doctor/status.json @@ -17,7 +17,7 @@ "build_url": "https://dev.azure.com/example-org/example-project/_build/results?buildId=1000001", "cost_usd": 1.25, "cost_complete": true, - "rows_failed": 3, + "rows_failed": 5, "rows_diagnosed": 2, "triage_file": "triage-1000001.json", "trigger": "https://dev.azure.com/example-org/example-project/_build/results?buildId=1000000", diff --git a/evalboard/lib/__tests__/fixtures/skill-doctor/triage-1000001.json b/evalboard/lib/__tests__/fixtures/skill-doctor/triage-1000001.json index bc5d96e3..9a28c3a8 100644 --- a/evalboard/lib/__tests__/fixtures/skill-doctor/triage-1000001.json +++ b/evalboard/lib/__tests__/fixtures/skill-doctor/triage-1000001.json @@ -2,53 +2,240 @@ "schema_version": 1, "disclaimer": "Probable causes, not verified fixes. Existing PRs are context; only a doctor PR is a proposed fix.", "run": { - "run_id": "2099-01-03_04-23-30", "harness": "claude-code", "model": "example-model", "skills_git_commit": "0a1b2c3d", - "cli_version": "1.0.0", "tasks_run": 10, "rows_executed": 10, "adhoc": false, "history_base_run": "2099-01-02_04-25-52", - "run_json_etag": "\"0x8D000000000000A\"", "build_id": "1000001", + "run_id": "2099-01-03_04-23-30", + "harness": "claude-code", + "model": "example-model", + "skills_git_commit": "0a1b2c3d", + "cli_version": "1.0.0", + "tasks_run": 10, + "rows_executed": 10, + "adhoc": false, + "history_base_run": "2099-01-02_04-25-52", + "run_json_etag": "\"0x8D000000000000A\"", + "build_id": "1000001", "build_url": "https://dev.azure.com/example-org/example-project/_build/results?buildId=1000001", - "doctor_version": "0.3.0a0", "generated_at": "2099-01-03T09:12:00Z", "mode": "diagnosed", - "mode_reason": "example", "cost_usd": 1.25 + "doctor_version": "0.3.0a0", + "generated_at": "2099-01-03T09:12:00Z", + "mode": "diagnosed", + "mode_reason": "example", + "cost_usd": 1.25 }, "summary": { - "rows_failed": 3, "rows_diagnosed": 2, - "by_class": {"env": 1, "failure": 2}, - "by_layer": {"env": 1, "grader": 1, "skill": 1}, - "vs_history": {"new": 2, "persisting": 1, "fixed": 0, "fixed_tasks": [], "base_run": "2099-01-02_04-25-52"}, - "env_groups": [{"cause": "connection example-conn", "count": 1, "task_ids": ["example-e1"]}] + "rows_failed": 5, + "rows_diagnosed": 2, + "by_class": { + "env": 3, + "failure": 2 + }, + "by_layer": { + "env": 3, + "grader": 1, + "skill": 1 + }, + "vs_history": { + "new": 4, + "persisting": 1, + "fixed": 0, + "fixed_tasks": [], + "base_run": "2099-01-02_04-25-52" + }, + "env_groups": [ + { + "cause": "connection example-conn", + "count": 3, + "task_ids": [ + "example-e1", + "example-e2", + "example-e3" + ] + } + ] }, "tasks": [ { - "task_id": "example-e1", "skill": "uipath-example", "class": "env", "family": "preflight", "state": "new", - "nights_seen": 1, "owners": ["Example Owner"], "run_status": "ERROR", "criterion": null, "error": "pre-run check failed", - "contested": null, "env_group": "connection example-conn", "diagnosed": false, "layer": "env", "confidence": null, - "why": "example-conn: no enabled connection", "evidence": null, "solvable": "env-fix", - "fix": "Reauthorize the connection, then re-run.", "related": null, "passed_before": null, - "existing_prs": null, "doctor_pr": null + "task_id": "example-e1", + "skill": "uipath-example", + "class": "env", + "family": "preflight", + "state": "new", + "nights_seen": 1, + "owners": [ + "Example Owner" + ], + "run_status": "ERROR", + "criterion": null, + "error": "pre-run check failed", + "contested": null, + "env_group": "connection example-conn", + "diagnosed": false, + "layer": "env", + "confidence": null, + "why": "example-conn: no enabled connection", + "evidence": null, + "solvable": "env-fix", + "fix": "Reauthorize the connection, then re-run.", + "related": null, + "passed_before": null, + "existing_prs": null, + "doctor_pr": null }, { - "task_id": "example-t1", "skill": "uipath-example", "class": "failure", "family": "grader", "state": "known", - "nights_seen": 3, "owners": ["Example Owner", "Second Owner"], "run_status": "FAILURE", "criterion": "file_contains", - "error": null, "contested": null, "env_group": null, "diagnosed": true, "layer": "grader", "confidence": "high", - "why": "The grader expects a literal the task prompt never asks for.", "evidence": "example evidence", - "solvable": "grader-fix", "fix": "Match the documented output instead of the literal.", "related": null, - "passed_before": null, + "task_id": "example-t1", + "skill": "uipath-example", + "class": "failure", + "family": "grader", + "state": "known", + "nights_seen": 3, + "owners": [ + "Example Owner", + "Second Owner" + ], + "run_status": "FAILURE", + "criterion": "file_contains", + "error": null, + "contested": null, + "env_group": null, + "diagnosed": true, + "layer": "grader", + "confidence": "high", + "why": "The grader expects a literal the task prompt never asks for.", + "evidence": "grader log: pattern `literal` not found", + "solvable": "yes-grader", + "fix": "Match the documented output instead of the literal.", + "related": "example-t3 fails the same check", + "passed_before": "2099-01-01_04-20-00 (claude-code)", "existing_prs": [ - {"url": "https://github.com/UiPath/skills/pull/123", "title": "fix: example grader", "state": "open"}, - {"url": "https://github.com.example.invalid/UiPath/skills/pull/124", "title": "look-alike host", "state": "open"}, - {"url": "javascript:alert(1)", "title": "script url", "state": "open"} + { + "url": "https://github.com/UiPath/skills/pull/123", + "title": "fix: example grader", + "state": "open" + }, + { + "url": "https://github.com.example.invalid/UiPath/skills/pull/124", + "title": "look-alike host", + "state": "open" + }, + { + "url": "javascript:alert(1)", + "title": "script url", + "state": "open" + } ], + "doctor_pr": null, + "related_skill_prs": [ + { + "url": "https://github.com/UiPath/skills/pull/125", + "title": "docs(example): document the output", + "state": "merged" + } + ] + }, + { + "task_id": "example-t2", + "skill": "uipath-example-two", + "class": "failure", + "family": "skill", + "state": "new", + "nights_seen": 1, + "owners": [], + "run_status": "FAILURE", + "criterion": "run_command", + "error": null, + "contested": null, + "env_group": null, + "diagnosed": true, + "layer": "skill", + "confidence": "medium", + "why": "The skill documents a removed flag.", + "evidence": null, + "solvable": "yes-skill", + "fix": "Document the current flag.", + "related": null, + "passed_before": null, + "existing_prs": [], "doctor_pr": null }, { - "task_id": "example-t2", "skill": "uipath-example-two", "class": "failure", "family": "skill", "state": "new", - "nights_seen": 1, "owners": [], "run_status": "FAILURE", "criterion": "run_command", "error": null, - "contested": null, "env_group": null, "diagnosed": true, "layer": "skill", "confidence": "medium", - "why": "The skill documents a removed flag.", "evidence": null, "solvable": "skill-fix", - "fix": "Document the current flag.", "related": null, "passed_before": null, "existing_prs": [], "doctor_pr": null + "task_id": "example-e2", + "skill": "uipath-example-two", + "class": "env", + "family": "preflight", + "state": "new", + "nights_seen": 1, + "owners": [ + "Example Owner" + ], + "run_status": "ERROR", + "criterion": null, + "error": "pre-run check failed", + "contested": null, + "env_group": "connection example-conn", + "diagnosed": false, + "layer": "env", + "confidence": null, + "why": "example-conn: no enabled connection", + "evidence": null, + "solvable": "env-fix", + "fix": "Reauthorize the connection, then re-run.", + "related": null, + "passed_before": null, + "existing_prs": null, + "doctor_pr": null + }, + { + "task_id": "example-e3", + "skill": "uipath-example-two", + "class": "env", + "family": "preflight", + "state": "new", + "nights_seen": 1, + "owners": [ + "Example Owner" + ], + "run_status": "ERROR", + "criterion": null, + "error": "pre-run check failed", + "contested": null, + "env_group": "connection example-conn", + "diagnosed": false, + "layer": "env", + "confidence": null, + "why": "example-conn: no enabled connection", + "evidence": null, + "solvable": "env-fix", + "fix": "Reauthorize the connection, then re-run.", + "related": null, + "passed_before": null, + "existing_prs": null, + "doctor_pr": null } ], "actions": [ - {"kind": "env", "count": 1, "text": "Environment: 1 task stopped at the pre-run check. Reauthorize the connection, then re-run.", "task_ids": ["example-e1"]}, - {"kind": "layer", "count": 1, "text": "uipath-example: 1 task diagnosed as a grader problem.", "task_ids": ["example-t1"]} + { + "kind": "env", + "count": 3, + "text": "Environment: 1 task stopped at the pre-run check. Reauthorize the connection, then re-run.", + "task_ids": [ + "example-e1", + "example-e2", + "example-e3" + ] + }, + { + "kind": "open-pr", + "count": 1, + "text": "Open PR https://github.com/UiPath/skills/pull/123 (fix: example grader) touches the files of 1 failing task. See javascript:alert(1) and https://github.com.example.invalid/x.", + "task_ids": [ + "example-t1" + ] + }, + { + "kind": "layer", + "count": 1, + "text": "uipath-example: 1 task diagnosed as a grader problem.", + "task_ids": [ + "example-t1" + ] + } ] } diff --git a/evalboard/lib/doctor.ts b/evalboard/lib/doctor.ts index 13991264..8da668a6 100644 --- a/evalboard/lib/doctor.ts +++ b/evalboard/lib/doctor.ts @@ -53,44 +53,38 @@ export interface DoctorPr { state: string; } -// triage_v1 task.state: new tonight, changed since the last night, or known. -export type DoctorTaskState = "new" | "changed" | "known"; - +// One failed row, with the fields the skill-doctor page shows on its card. export interface DoctorTask { taskId: string; skill: string; layer: string; - cls: string; - state: DoctorTaskState | null; - nightsSeen: number; + solvable: string | null; + runStatus: string | null; confidence: string | null; why: string | null; fix: string | null; + criterion: string | null; + evidence: string | null; + passedBefore: string | null; + related: string | null; owners: string[]; existingPrs: DoctorPr[]; + relatedSkillPrs: DoctorPr[]; + envGroup: string | null; + doctorPr: string | null; } export interface DoctorAction { - kind: string; + count: number; text: string; + taskIds: string[]; } export interface DoctorTriage { - disclaimer: string; - costUsd: number | null; - summary: { - rowsFailed: number; - rowsDiagnosed: number; - byClass: Record; - byLayer: Record; - // Only set when the doctor compared against a base run. - vsHistory: { - new: number; - persisting: number; - fixed: number; - baseRun: string; - } | null; - }; + // Rows the nightly executed (null when the run did not say). + rowsRun: number | null; + rowsFailed: number; + rowsDiagnosed: number; tasks: DoctorTask[]; actions: DoctorAction[]; } @@ -145,14 +139,6 @@ function int(v: unknown, dflt = 0): number { function strList(v: unknown): string[] { return Array.isArray(v) ? v.filter((x): x is string => typeof x === "string") : []; } -function countMap(v: unknown): Record { - const out: Record = {}; - if (!isObj(v)) return out; - for (const [k, n] of Object.entries(v)) { - if (typeof n === "number" && Number.isInteger(n) && n >= 0) out[k] = n; - } - return out; -} function parseJson(raw: string | null | undefined): unknown { if (raw == null) return null; @@ -190,8 +176,8 @@ function parsePr(v: unknown): DoctorPr | null { return { url, title: str(v.title) ?? "", state: str(v.state) ?? "unknown" }; } -function taskState(v: unknown): DoctorTaskState | null { - return v === "new" || v === "changed" || v === "known" ? v : null; +function prList(v: unknown): DoctorPr[] { + return Array.isArray(v) ? v.map(parsePr).filter((p): p is DoctorPr => p !== null) : []; } function parseTask(v: unknown): DoctorTask | null { @@ -202,16 +188,20 @@ function parseTask(v: unknown): DoctorTask | null { taskId, skill: str(v.skill) ?? "", layer: str(v.layer) ?? "undiagnosed", - cls: str(v.class) ?? "", - state: taskState(v.state), - nightsSeen: int(v.nights_seen, 1), + solvable: str(v.solvable), + runStatus: str(v.run_status), confidence: str(v.confidence), why: str(v.why), fix: str(v.fix), + criterion: str(v.criterion), + evidence: str(v.evidence), + passedBefore: str(v.passed_before), + related: str(v.related), owners: strList(v.owners), - existingPrs: Array.isArray(v.existing_prs) - ? v.existing_prs.map(parsePr).filter((p): p is DoctorPr => p !== null) - : [], + existingPrs: prList(v.existing_prs), + relatedSkillPrs: prList(v.related_skill_prs), + envGroup: str(v.env_group), + doctorPr: str(v.doctor_pr), }; } @@ -221,31 +211,15 @@ export function parseTriage(raw: string | null | undefined): DoctorTriage | null if (!isObj(j.summary) || !Array.isArray(j.tasks)) return null; const s = j.summary; const run = isObj(j.run) ? j.run : {}; - const vh = isObj(s.vs_history) ? s.vs_history : null; - const baseRun = vh ? str(vh.base_run) : null; return { - disclaimer: str(j.disclaimer) ?? "", - costUsd: num(run.cost_usd), - summary: { - rowsFailed: int(s.rows_failed), - rowsDiagnosed: int(s.rows_diagnosed), - byClass: countMap(s.by_class), - byLayer: countMap(s.by_layer), - vsHistory: - vh && baseRun - ? { - new: int(vh.new), - persisting: int(vh.persisting), - fixed: int(vh.fixed), - baseRun, - } - : null, - }, + rowsRun: typeof run.rows_executed === "number" ? int(run.rows_executed) : null, + rowsFailed: int(s.rows_failed), + rowsDiagnosed: int(s.rows_diagnosed), tasks: j.tasks.map(parseTask).filter((t): t is DoctorTask => t !== null), actions: Array.isArray(j.actions) ? j.actions.flatMap((a): DoctorAction[] => isObj(a) && typeof a.text === "string" - ? [{ kind: str(a.kind) ?? "", text: a.text }] + ? [{ count: int(a.count), text: a.text, taskIds: strList(a.task_ids) }] : [], ) : [], @@ -386,31 +360,25 @@ export function taskAnalysisEnabled( export interface AnalysisPlan { // Markdown for the AI summary panel (null: no panel). summary: string | null; - // The Task Failure Analysis panel, after the summary (null: none). - panel: RunDoctor | null; - // One-line doctor status when there is no panel (null: none). - statusLine: DoctorStatus | null; + // The Task Failure Analysis block, right under the summary (null: none). + doctor: RunDoctor | null; } /** - * What the run page shows. A done report with a parsed triage gets the panel; - * when analysis.md has a split point, the summary keeps only the part above - * Findings (the panel replaces the rest), else the whole file. Every other - * state keeps today's view, plus a status line. + * What the run page shows. With a done report whose triage parsed, the + * summary stops at analysis.md's `## Findings` line (the block replaces the + * rest), or keeps the whole file when there is no such line. In every other + * state the summary is unchanged (Findings included) and the block shows only + * the doctor's state. */ export function planAnalysis( analysis: string | null, doctor: RunDoctor | null, ): AnalysisPlan { - if (!doctor) return { summary: analysis, panel: null, statusLine: null }; - if (doctor.status.state === "done" && doctor.triage) { + if (doctor && doctor.status.state === "done" && doctor.triage) { const split = analysis ? splitAnalysis(analysis) : null; const summary = split ? split.top : analysis; - return { - summary: summary && summary.trim() ? summary : null, - panel: doctor, - statusLine: null, - }; + return { summary: summary && summary.trim() ? summary : null, doctor }; } - return { summary: analysis, panel: null, statusLine: doctor.status }; + return { summary: analysis, doctor }; } From 56cf46107f9f6f6b3b4d6a08452d4f9bdcd0ef1d Mon Sep 17 00:00:00 2001 From: CarlesUIPath Date: Thu, 19 Nov 2026 16:25:25 +0000 Subject: [PATCH 05/13] feat(evalboard): order skill accordions by proposed PRs ready, then failures Co-Authored-By: Claude Opus 5.5 --- .../__tests__/task-failure-analysis.test.tsx | 31 +++++++++++++++++++ .../app/runs/[id]/task-failure-analysis.tsx | 12 +++++-- 2 files changed, 40 insertions(+), 3 deletions(-) diff --git a/evalboard/app/runs/[id]/__tests__/task-failure-analysis.test.tsx b/evalboard/app/runs/[id]/__tests__/task-failure-analysis.test.tsx index 3f776ca4..fd495d62 100644 --- a/evalboard/app/runs/[id]/__tests__/task-failure-analysis.test.tsx +++ b/evalboard/app/runs/[id]/__tests__/task-failure-analysis.test.tsx @@ -276,6 +276,37 @@ describe("TaskFailureAnalysis block", () => { }); describe("groupBySkill", () => { + test("skills sort by PRs ready, then failed tasks, then name", () => { + // The fixture: uipath-example has 1 ready of 2 failed, -two 0 of 3. + expect(groupBySkill(doctor(), null, "").map((g) => g.skill)).toEqual([ + "uipath-example", + "uipath-example-two", + ]); + // No PR ready anywhere: more failed tasks first. + const none = { ...doctor(), fixes: null }; + expect(groupBySkill(none, null, "").map((g) => g.skill)).toEqual([ + "uipath-example-two", + "uipath-example", + ]); + // Equal on both: by name. + const tie = doctor(); + tie.fixes = null; + tie.triage!.tasks = [ + { ...tie.triage!.tasks[1], skill: "b-skill", taskId: "x1" }, + { ...tie.triage!.tasks[1], skill: "a-skill", taskId: "x2" }, + ]; + expect(groupBySkill(tie, null, "").map((g) => g.skill)).toEqual(["a-skill", "b-skill"]); + }); + + test("the rendered accordions follow the same order", () => { + renderBlock({ ...doctor(), fixes: null }); + openBlock(); + const order = Array.from(document.querySelectorAll("[data-skill]")).map((e) => + e.getAttribute("data-skill"), + ); + expect(order).toEqual(["uipath-example-two", "uipath-example"]); + }); + test("cards sorted by layer rank, env tasks grouped", () => { const g = groupBySkill(doctor(), null, ""); expect(g.map((x) => x.skill)).toEqual(["uipath-example", "uipath-example-two"]); diff --git a/evalboard/app/runs/[id]/task-failure-analysis.tsx b/evalboard/app/runs/[id]/task-failure-analysis.tsx index 4d2354e4..7f0c82b8 100644 --- a/evalboard/app/runs/[id]/task-failure-analysis.tsx +++ b/evalboard/app/runs/[id]/task-failure-analysis.tsx @@ -225,7 +225,10 @@ export function readyTasks(doctor: RunDoctor, taskIds: string[]): number { return taskIds.filter((id) => fixesForTask(doctor, id).some(isReady)).length; } -/** Visible failures by skill, as the doctor's page groups them. */ +/** + * Visible failures by skill, as the doctor's page groups them. Skills with the + * most proposed PRs ready come first; ties go to more failed tasks, then name. + */ export function groupBySkill( doctor: RunDoctor, active: Set | null, @@ -243,7 +246,6 @@ export function groupBySkill( by.set(k, [...(by.get(k) ?? []), t]); } return [...by.keys()] - .sort((a, b) => a.localeCompare(b)) .map((skill) => { const items = by.get(skill)!; const groups = new Map(); @@ -265,7 +267,11 @@ export function groupBySkill( ).length, total: items.length, }; - }); + }) + .sort( + (a, b) => + b.ready - a.ready || b.total - a.total || a.skill.localeCompare(b.skill), + ); } // ---------- cards ---------- From 661fd6fb4cb2c09bdafee13f1c65883ded8e435e Mon Sep 17 00:00:00 2001 From: CarlesUIPath Date: Thu, 19 Nov 2026 16:28:40 +0000 Subject: [PATCH 06/13] feat(evalboard): sort task cards by proposed PR ready, then fix candidates Co-Authored-By: Claude Opus 5.5 --- .../__tests__/task-failure-analysis.test.tsx | 30 +++++++++++++++++++ .../app/runs/[id]/task-failure-analysis.tsx | 14 ++++++++- 2 files changed, 43 insertions(+), 1 deletion(-) diff --git a/evalboard/app/runs/[id]/__tests__/task-failure-analysis.test.tsx b/evalboard/app/runs/[id]/__tests__/task-failure-analysis.test.tsx index fd495d62..72feeea6 100644 --- a/evalboard/app/runs/[id]/__tests__/task-failure-analysis.test.tsx +++ b/evalboard/app/runs/[id]/__tests__/task-failure-analysis.test.tsx @@ -298,6 +298,36 @@ describe("groupBySkill", () => { expect(groupBySkill(tie, null, "").map((g) => g.skill)).toEqual(["a-skill", "b-skill"]); }); + test("cards: PR ready, then other candidates, then the rest, then layer rank", () => { + const d = doctor(); + const base = d.triage!.tasks[1]; + d.triage!.tasks = [ + { ...base, taskId: "z-plain-skill", layer: "skill", doctorPr: null }, + { ...base, taskId: "a-plain-grader", layer: "grader", doctorPr: null }, + { ...base, taskId: "m-live", layer: "grader", doctorPr: null }, + { ...base, taskId: "y-ready", layer: "grader", doctorPr: null }, + { ...base, taskId: "x-doctor-pr", layer: "grader", doctorPr: "https://github.com/UiPath/skills/pull/7" }, + { ...base, taskId: "b-bad-pr", layer: "grader", doctorPr: "https://example.invalid/pull/7" }, + { ...d.triage!.tasks[0], skill: base.skill }, + ]; + d.fixes!.candidates = [ + { ...d.fixes!.candidates[0], id: "f1", taskId: "y-ready", covers: ["y-ready"] }, + { ...d.fixes!.candidates[1], id: "f2", taskId: "m-live", covers: ["m-live"] }, + ]; + const [g] = groupBySkill(d, null, ""); + expect(g.cards.map((t) => t.taskId)).toEqual([ + "x-doctor-pr", + "y-ready", + "m-live", + // The rest: skill ranks after grader on the doctor page, so first. + "z-plain-skill", + "a-plain-grader", + "b-bad-pr", + ]); + // Env tasks stay in their group, rendered after the cards. + expect(g.envGroups.map((e) => e.tasks.map((t) => t.taskId))).toEqual([["example-e1"]]); + }); + test("the rendered accordions follow the same order", () => { renderBlock({ ...doctor(), fixes: null }); openBlock(); diff --git a/evalboard/app/runs/[id]/task-failure-analysis.tsx b/evalboard/app/runs/[id]/task-failure-analysis.tsx index 7f0c82b8..d74a2ad6 100644 --- a/evalboard/app/runs/[id]/task-failure-analysis.tsx +++ b/evalboard/app/runs/[id]/task-failure-analysis.tsx @@ -254,7 +254,19 @@ export function groupBySkill( if (t.envGroup) groups.set(t.envGroup, [...(groups.get(t.envGroup) ?? []), t]); else cards.push(t); } - cards.sort((a, b) => rank(b.layer) - rank(a.layer) || a.taskId.localeCompare(b.taskId)); + // Proposed PR ready first, then other fix candidates, then the rest; + // within each, the doctor page's layer rank and task id. + const tier = (t: DoctorTask) => { + const c = fixesForTask(doctor, t.taskId); + if (isSkillsPrUrl(t.doctorPr) || c.some(isReady)) return 0; + return c.length ? 1 : 2; + }; + cards.sort( + (a, b) => + tier(a) - tier(b) || + rank(b.layer) - rank(a.layer) || + a.taskId.localeCompare(b.taskId), + ); const lc = new Map(); for (const t of items) lc.set(t.layer, (lc.get(t.layer) ?? 0) + 1); return { From 9621fb0d63da43b5e07b62f7a2e489dfe414dc8e Mon Sep 17 00:00:00 2001 From: CarlesUIPath Date: Thu, 19 Nov 2026 16:35:47 +0000 Subject: [PATCH 07/13] feat(evalboard): "PR with fix" and "Proposed PR" rows on task cards - A pr-ready fix candidate reads "PR with fix" with its title, patch, gates and cost; the title expands to the prepared PR body (/.md, loaded by name for ids in patchIds, rendered with ReactMarkdown without raw HTML; only allowlisted links, images as alt text), then the patch download. A doctor PR shows as "PR with fix" linking to GitHub instead. Other outcomes stay "Fix candidate". - "Proposed PR": an open, allowlisted PR from the task's existing PRs that its fix text names (#N or /pull/N). - Header, skill accordions and Fix first badges count distinct PRs with fix (both kinds); skill and card sorts use the same model. - The Owner row is gone and owners are no longer parsed. Co-Authored-By: Claude Opus 5.5 --- .../__tests__/task-failure-analysis.test.tsx | 165 ++++++++-- .../app/runs/[id]/task-failure-analysis.tsx | 283 +++++++++++++----- evalboard/lib/__tests__/doctor-blob.test.ts | 20 ++ evalboard/lib/__tests__/doctor.test.ts | 1 - .../fixes-1000001/example-grader-fix.md | 15 + .../fixtures/skill-doctor/triage-1000001.json | 15 +- evalboard/lib/doctor.ts | 16 +- evalboard/lib/runs.ts | 22 +- 8 files changed, 430 insertions(+), 107 deletions(-) create mode 100644 evalboard/lib/__tests__/fixtures/skill-doctor/fixes-1000001/example-grader-fix.md diff --git a/evalboard/app/runs/[id]/__tests__/task-failure-analysis.test.tsx b/evalboard/app/runs/[id]/__tests__/task-failure-analysis.test.tsx index 72feeea6..af597d83 100644 --- a/evalboard/app/runs/[id]/__tests__/task-failure-analysis.test.tsx +++ b/evalboard/app/runs/[id]/__tests__/task-failure-analysis.test.tsx @@ -3,8 +3,10 @@ import path from "node:path"; import { describe, expect, test } from "vitest"; import { fireEvent, render, screen, within } from "@testing-library/react"; import { + countFixPrs, digest, groupBySkill, + proposedPrs, TaskFailureAnalysis, } from "../task-failure-analysis"; import { @@ -28,6 +30,7 @@ function doctor(patch: Record = {}): RunDoctor { status: s, triage: s.state === "done" ? parseTriage(read("triage-1000001.json")) : null, fixes: parseFixes(read("fixes-1000001/fixes.json")), + fixBodies: { "example-grader-fix": read("fixes-1000001/example-grader-fix.md") }, }; } @@ -43,13 +46,14 @@ const skillButton = (skill: string) => const openSkill = (skill: string) => fireEvent.click(skillButton(skill)); describe("digest", () => { - test("done: errors and PR-ready fixes", () => { - expect(digest(doctor())).toBe("5 errors · 1 proposed PR ready"); + test("done: errors and PRs with fix (prepared + proposed)", () => { + // example-grader-fix (pr-ready) + open PR #126 named by example-t2's fix. + expect(digest(doctor())).toBe("5 errors · 2 PRs with fix"); }); - test("no PR-ready fix: the errors only", () => { + test("one kind only, and none", () => { const d = doctor(); - d.fixes!.candidates = d.fixes!.candidates.filter((c) => c.outcome !== "pr-ready"); - expect(digest(d)).toBe("5 errors"); + expect(digest({ ...d, fixes: null })).toBe("5 errors · 1 PR with fix"); + d.triage!.tasks[2].fix = "Document the current flag."; expect(digest({ ...d, fixes: null })).toBe("5 errors"); }); test("not done: the state", () => { @@ -69,7 +73,7 @@ describe("TaskFailureAnalysis block", () => { renderBlock(); expect(header()).toHaveAttribute("aria-expanded", "false"); expect(header()).toHaveTextContent("Task failure analysis"); - expect(header()).toHaveTextContent("5 errors · 1 proposed PR ready"); + expect(header()).toHaveTextContent("5 errors · 2 PRs with fix"); expect(screen.queryByText("Fix first")).toBeNull(); expect(screen.queryByText("Skill failures")).toBeNull(); }); @@ -106,7 +110,7 @@ describe("TaskFailureAnalysis block", () => { expect(screen.getByRole("img", { name: "Failures by layer" }).children).toHaveLength(3); }); - test("Fix first: ranked actions, allowlisted links, a PR-ready badge", () => { + test("Fix first: ranked actions, allowlisted links, a PRs-with-fix badge", () => { renderBlock(); openBlock(); const fixFirst = screen.getByRole("region", { name: "Fix first" }); @@ -120,7 +124,7 @@ describe("TaskFailureAnalysis block", () => { // javascript: and look-alike hosts stay text. expect(within(items[1]).getAllByRole("link")).toHaveLength(1); expect(items[1]).toHaveTextContent("javascript:alert(1)"); - expect(within(items[1]).getByTestId("action-ready")).toHaveTextContent("1 PR ready"); + expect(within(items[1]).getByTestId("action-ready")).toHaveTextContent("1 PR with fix"); expect(within(items[0]).queryByTestId("action-ready")).toBeNull(); }); @@ -131,11 +135,10 @@ describe("TaskFailureAnalysis block", () => { const b = skillButton("uipath-example-two"); expect(a).toHaveAttribute("aria-expanded", "false"); expect(b).toHaveAttribute("aria-expanded", "false"); - expect(a).toHaveTextContent("2 failed · 1 proposed PR ready"); + expect(a).toHaveTextContent("2 failed · 1 PR with fix"); expect(a).toHaveTextContent("1 environment"); expect(a).toHaveTextContent("1 grader"); - expect(b).toHaveTextContent("3 failed"); - expect(b).not.toHaveTextContent("ready"); + expect(b).toHaveTextContent("3 failed · 1 PR with fix"); expect(screen.queryAllByTestId("task-card")).toHaveLength(0); }); @@ -172,8 +175,9 @@ describe("TaskFailureAnalysis block", () => { expect(within(card).getByText(pill)).toBeInTheDocument(); } expect(card).toHaveTextContent("The grader expects a literal"); - const fix = within(card).getByTestId("fix-candidate"); - expect(fix).toHaveTextContent("Proposed PR: pr-ready"); + const fix = within(card).getByTestId("pr-with-fix"); + expect(fix).toHaveTextContent("PR with fix"); + expect(fix).toHaveTextContent("fix(example): match the documented output"); expect(fix).toHaveTextContent("replay pass · mutants 9/10 · review SEND"); expect(fix).toHaveTextContent("$1.50"); const patch = within(fix).getByRole("link", { name: "+3/-1" }); @@ -182,6 +186,9 @@ describe("TaskFailureAnalysis block", () => { expect(href.searchParams.get("path")).toBe( "skill-doctor/fixes-1000001/example-grader-fix.patch", ); + expect(within(card).queryByTestId("fix-candidate")).toBeNull(); + // #123 is open on these files, but the fix text does not name it. + expect(within(card).queryByTestId("proposed-pr")).toBeNull(); const dts = within(card).getAllByRole("term").map((d) => d.textContent); expect(dts).toEqual([ "Fix", @@ -191,13 +198,12 @@ describe("TaskFailureAnalysis block", () => { "Open PRs on these files", "Related skill PRs", "Related", - "Owner", ]); expect(within(card).getByText("literal").tagName).toBe("CODE"); expect(within(card).getByRole("link", { name: "#123" })).toBeInTheDocument(); expect(within(card).getByRole("link", { name: "#125" })).toBeInTheDocument(); expect(within(card).getByText("javascript:alert(1)").tagName).toBe("SPAN"); - expect(within(card).getByText("Example Owner, Second Owner")).toBeInTheDocument(); + expect(card).not.toHaveTextContent("Example Owner"); }); test("a fix candidate that is not ready, and an unsafe patch id", () => { @@ -209,7 +215,73 @@ describe("TaskFailureAnalysis block", () => { const t1 = screen.getAllByTestId("task-card").find((c) => c.textContent!.includes("example-t1"))!; expect(within(t1).getByText("+3/-1").closest("a")).toBeNull(); const t2 = screen.getAllByTestId("task-card").find((c) => c.textContent!.includes("example-t2"))!; - expect(within(t2).getByTestId("fix-candidate")).toHaveTextContent("Proposed PR: needs-live-run"); + expect(within(t2).getByTestId("fix-candidate")).toHaveTextContent("Fix candidate: needs-live-run"); + }); + + test("the PR with fix title expands to the prepared body, then the patch", () => { + renderBlock(); + openBlock(); + openSkill("uipath-example"); + const fix = screen.getByTestId("pr-with-fix"); + expect(within(fix).queryByTestId("pr-body")).toBeNull(); + const title = within(fix).getByRole("button", { name: "fix(example): match the documented output" }); + expect(title).toHaveAttribute("aria-expanded", "false"); + fireEvent.click(title); + const body = within(fix).getByTestId("pr-body"); + expect(within(body).getByRole("heading", { name: "Summary" })).toBeInTheDocument(); + expect(within(body).getByRole("table")).toBeInTheDocument(); + const links = within(body).getAllByRole("link").map((a) => a.getAttribute("href")); + expect(links).toEqual([ + "https://github.com/UiPath/skills/pull/123", + "https://dev.azure.com/example-org/example-project/_build/results?buildId=1000001", + ]); + // Off-list and script links are text; images are alt text; raw HTML is dropped. + expect(within(body).getByText("an off-list link").tagName).toBe("SPAN"); + expect(within(body).getByText("a script link").tagName).toBe("SPAN"); + expect(within(body).queryByRole("img")).toBeNull(); + expect(within(body).getByText("an image")).toBeInTheDocument(); + expect(body.querySelector("b")).toBeNull(); + const dl = within(fix).getByRole("link", { name: "Download patch (+3/-1)" }); + expect(new URL(dl.getAttribute("href")!, "http://x").searchParams.get("path")).toBe( + "skill-doctor/fixes-1000001/example-grader-fix.patch", + ); + }); + + test("no body: the title is plain text", () => { + renderBlock({ ...doctor(), fixBodies: {} }); + openBlock(); + openSkill("uipath-example"); + const fix = screen.getByTestId("pr-with-fix"); + expect(within(fix).queryByRole("button")).toBeNull(); + expect(fix).toHaveTextContent("fix(example): match the documented output"); + }); + + test("a doctor PR replaces the prepared fix with a GitHub link", () => { + const d = doctor(); + d.triage!.tasks[1].doctorPr = "https://github.com/UiPath/skills/pull/200"; + renderBlock(d); + openBlock(); + openSkill("uipath-example"); + const rows = screen.getAllByTestId("pr-with-fix"); + expect(rows).toHaveLength(1); + expect(rows[0]).toHaveTextContent("PR with fix: #200"); + expect(rows[0]).toHaveAttribute("href", "https://github.com/UiPath/skills/pull/200"); + }); + + test("a third-party open PR named in the fix text: a Proposed PR row", () => { + renderBlock(); + openBlock(); + openSkill("uipath-example-two"); + const card = screen.getAllByTestId("task-card").find((c) => c.textContent!.includes("example-t2"))!; + const rows = within(card).getAllByTestId("proposed-pr"); + // #127 is named too, but merged: it does not count. + expect(rows).toHaveLength(1); + expect(rows[0]).toHaveTextContent("Proposed PR#126 docs(example-two): document the current flagopen"); + expect(within(rows[0]).getByRole("link")).toHaveAttribute( + "href", + "https://github.com/UiPath/skills/pull/126", + ); + expect(within(card).getByTestId("fix-candidate")).toHaveTextContent("Fix candidate: needs-live-run"); }); test("a task id the task page would reject is plain text", () => { @@ -277,16 +349,22 @@ describe("TaskFailureAnalysis block", () => { describe("groupBySkill", () => { test("skills sort by PRs ready, then failed tasks, then name", () => { - // The fixture: uipath-example has 1 ready of 2 failed, -two 0 of 3. + // uipath-example: 1 PR with fix of 2 failed; -two: 1 (proposed #126) of 3. expect(groupBySkill(doctor(), null, "").map((g) => g.skill)).toEqual([ - "uipath-example", "uipath-example-two", + "uipath-example", ]); - // No PR ready anywhere: more failed tasks first. - const none = { ...doctor(), fixes: null }; - expect(groupBySkill(none, null, "").map((g) => g.skill)).toEqual([ - "uipath-example-two", + // More PRs with fix wins over more failures. + const more = doctor(); + more.fixes!.candidates.push({ + ...more.fixes!.candidates[0], + id: "second-ready", + taskId: "example-e1", + covers: ["example-e1"], + }); + expect(groupBySkill(more, null, "").map((g) => g.skill)).toEqual([ "uipath-example", + "uipath-example-two", ]); // Equal on both: by name. const tie = doctor(); @@ -298,7 +376,7 @@ describe("groupBySkill", () => { expect(groupBySkill(tie, null, "").map((g) => g.skill)).toEqual(["a-skill", "b-skill"]); }); - test("cards: PR ready, then other candidates, then the rest, then layer rank", () => { + test("cards: own fix, then proposed PR, then other candidates, then the rest", () => { const d = doctor(); const base = d.triage!.tasks[1]; d.triage!.tasks = [ @@ -308,6 +386,7 @@ describe("groupBySkill", () => { { ...base, taskId: "y-ready", layer: "grader", doctorPr: null }, { ...base, taskId: "x-doctor-pr", layer: "grader", doctorPr: "https://github.com/UiPath/skills/pull/7" }, { ...base, taskId: "b-bad-pr", layer: "grader", doctorPr: "https://example.invalid/pull/7" }, + { ...base, taskId: "p-proposed", layer: "grader", doctorPr: null, fix: "see #123" }, { ...d.triage!.tasks[0], skill: base.skill }, ]; d.fixes!.candidates = [ @@ -318,6 +397,7 @@ describe("groupBySkill", () => { expect(g.cards.map((t) => t.taskId)).toEqual([ "x-doctor-pr", "y-ready", + "p-proposed", "m-live", // The rest: skill ranks after grader on the doctor page, so first. "z-plain-skill", @@ -329,22 +409,47 @@ describe("groupBySkill", () => { }); test("the rendered accordions follow the same order", () => { - renderBlock({ ...doctor(), fixes: null }); + const d = doctor(); + d.triage!.tasks[2].fix = "no PR named"; + renderBlock(d); openBlock(); const order = Array.from(document.querySelectorAll("[data-skill]")).map((e) => e.getAttribute("data-skill"), ); - expect(order).toEqual(["uipath-example-two", "uipath-example"]); + expect(order).toEqual(["uipath-example", "uipath-example-two"]); }); test("cards sorted by layer rank, env tasks grouped", () => { const g = groupBySkill(doctor(), null, ""); - expect(g.map((x) => x.skill)).toEqual(["uipath-example", "uipath-example-two"]); - expect(g[0].cards.map((t) => t.taskId)).toEqual(["example-t1"]); - expect(g[0].envGroups).toEqual([ + const ex = g.find((x) => x.skill === "uipath-example")!; + expect(ex.cards.map((t) => t.taskId)).toEqual(["example-t1"]); + expect(ex.envGroups).toEqual([ { cause: "connection example-conn", tasks: [expect.objectContaining({ taskId: "example-e1" })] }, ]); - expect(g[0].ready).toBe(1); - expect(g[1].ready).toBe(0); + expect(g.map((x) => x.ready)).toEqual([1, 1]); + }); + + test("proposedPrs: named in the fix text, open, allowlisted", () => { + const t = doctor().triage!.tasks[2]; + expect(proposedPrs(t).map((p) => p.url)).toEqual(["https://github.com/UiPath/skills/pull/126"]); + expect(proposedPrs({ ...t, fix: "see https://github.com/UiPath/skills/pull/126" })).toHaveLength(1); + expect(proposedPrs({ ...t, fix: "see #1260 and #12" })).toHaveLength(0); + expect(proposedPrs({ ...t, fix: null })).toHaveLength(0); + expect( + proposedPrs({ + ...t, + existingPrs: [{ url: "https://github.com.example.invalid/UiPath/skills/pull/126", title: "", state: "open" }], + }), + ).toHaveLength(0); + expect( + proposedPrs({ ...t, existingPrs: [{ ...t.existingPrs[0], state: "unknown" }] }), + ).toHaveLength(0); + }); + + test("countFixPrs counts a PR once across the tasks it covers", () => { + const d = doctor(); + d.fixes!.candidates[0].covers = ["example-t1", "example-t2"]; + // example-grader-fix covers both; #126 on t2: 2 distinct PRs. + expect(countFixPrs(d, d.triage!.tasks)).toBe(2); }); }); diff --git a/evalboard/app/runs/[id]/task-failure-analysis.tsx b/evalboard/app/runs/[id]/task-failure-analysis.tsx index d74a2ad6..16fda432 100644 --- a/evalboard/app/runs/[id]/task-failure-analysis.tsx +++ b/evalboard/app/runs/[id]/task-failure-analysis.tsx @@ -2,6 +2,8 @@ import Link from "next/link"; import { useMemo, useState, type ReactNode } from "react"; +import ReactMarkdown, { type Components } from "react-markdown"; +import remarkGfm from "remark-gfm"; import { withSource } from "@/app/_lib/source-param"; import { fmtTimestamp, fmtUsd } from "@/lib/format"; import { @@ -139,7 +141,7 @@ function PrList({ prs }: { prs: DoctorPr[] }) { // ---------- digest + state ---------- export function digest(doctor: RunDoctor): string { - const { status, triage, fixes } = doctor; + const { status, triage } = doctor; switch (status.state) { case "queued": case "running": @@ -153,11 +155,9 @@ export function digest(doctor: RunDoctor): string { return "the report for this run was retracted"; case "done": { if (!triage) return "report done, but its triage file is missing or did not parse"; - const ready = (fixes?.candidates ?? []).filter(isReady).length; + const n = countFixPrs(doctor, triage.tasks); const errors = `${triage.rowsFailed} error${triage.rowsFailed === 1 ? "" : "s"}`; - return ready - ? `${errors} · ${ready} proposed PR${ready === 1 ? "" : "s"} ready` - : errors; + return n ? `${errors} · ${prsWithFix(n)}` : errors; } } } @@ -209,7 +209,7 @@ export interface SkillGroup { cards: DoctorTask[]; envGroups: { cause: string; tasks: DoctorTask[] }[]; layerCounts: [string, number][]; - // Tasks with a proposed PR ready: a pr-ready fix candidate or a doctor PR. + // Distinct PRs with a fix among the skill's tasks (countFixPrs). ready: number; total: number; } @@ -220,14 +220,62 @@ export function fixesForTask(doctor: RunDoctor, taskId: string): DoctorFix[] { ); } -/** How many of these tasks have a pr-ready fix candidate. */ -export function readyTasks(doctor: RunDoctor, taskIds: string[]): number { - return taskIds.filter((id) => fixesForTask(doctor, id).some(isReady)).length; +// "#N" or ".../pull/N" in a task's fix text. +const PR_REF_RE = /(?:#|\/pull\/)(\d+)\b/g; + +/** + * A third-party proposed PR: the task's fix text names it AND it is one of the + * task's existing PRs, open, with an allowlisted URL. Merged or unknown PRs + * do not count. + */ +export function proposedPrs(t: DoctorTask): DoctorPr[] { + if (!t.fix) return []; + const named = new Set([...t.fix.matchAll(PR_REF_RE)].map((m) => m[1])); + return t.existingPrs.filter((p) => { + if (p.state !== "open" || !isSkillsPrUrl(p.url)) return false; + return named.has(/(\d+)$/.exec(p.url)![1]); + }); +} + +export interface TaskFixes { + // The doctor's own fix: its opened PR (allowlisted), else its prepared + // pr-ready candidates. Both render as "PR with fix". + doctorPr: string | null; + ready: DoctorFix[]; + // Third-party open PRs the fix text points at ("Proposed PR"). + proposed: DoctorPr[]; + // Other fix candidates (needs-live-run, ...). + others: DoctorFix[]; + // One key per distinct PR with a fix, for counting across tasks. + keys: string[]; + // Card sort: own fix, then proposed PR, then other candidates, then rest. + tier: number; +} + +export function taskFixes(doctor: RunDoctor, t: DoctorTask): TaskFixes { + const cands = fixesForTask(doctor, t.taskId); + const doctorPr = isSkillsPrUrl(t.doctorPr) ? t.doctorPr : null; + const ready = doctorPr ? [] : cands.filter(isReady); + const others = cands.filter((c) => !isReady(c)); + const proposed = proposedPrs(t); + const keys = [ + ...(doctorPr ? [`pr:${doctorPr}`] : ready.map((c) => `fix:${c.id}`)), + ...proposed.map((p) => `pr:${p.url}`), + ]; + const tier = doctorPr || ready.length ? 0 : proposed.length ? 1 : others.length ? 2 : 3; + return { doctorPr, ready, proposed, others, keys, tier }; +} + +/** Distinct PRs with a fix (own or proposed) over these tasks. */ +export function countFixPrs(doctor: RunDoctor, tasks: DoctorTask[]): number { + return new Set(tasks.flatMap((t) => taskFixes(doctor, t).keys)).size; } +const prsWithFix = (n: number) => `${n} PR${n === 1 ? "" : "s"} with fix`; + /** * Visible failures by skill, as the doctor's page groups them. Skills with the - * most proposed PRs ready come first; ties go to more failed tasks, then name. + * most PRs with fix come first; ties go to more failed tasks, then name. */ export function groupBySkill( doctor: RunDoctor, @@ -254,13 +302,9 @@ export function groupBySkill( if (t.envGroup) groups.set(t.envGroup, [...(groups.get(t.envGroup) ?? []), t]); else cards.push(t); } - // Proposed PR ready first, then other fix candidates, then the rest; - // within each, the doctor page's layer rank and task id. - const tier = (t: DoctorTask) => { - const c = fixesForTask(doctor, t.taskId); - if (isSkillsPrUrl(t.doctorPr) || c.some(isReady)) return 0; - return c.length ? 1 : 2; - }; + // The doctor's own fix first, then a proposed PR, then other fix + // candidates, then the rest; within each, layer rank and task id. + const tier = (t: DoctorTask) => taskFixes(doctor, t).tier; cards.sort( (a, b) => tier(a) - tier(b) || @@ -274,9 +318,7 @@ export function groupBySkill( cards, envGroups: [...groups.entries()].map(([cause, tasks]) => ({ cause, tasks })), layerCounts: [...lc.entries()].sort((a, b) => rank(a[0]) - rank(b[0])), - ready: items.filter( - (t) => isSkillsPrUrl(t.doctorPr) || fixesForTask(doctor, t.taskId).some(isReady), - ).length, + ready: countFixPrs(doctor, items), total: items.length, }; }) @@ -317,7 +359,7 @@ function TaskId({ id, ctx }: { id: string; ctx: FixCtx }) { function TaskCard({ t, doctor, ctx }: { t: DoctorTask; doctor: RunDoctor; ctx: FixCtx }) { const layer = layerOf(t.layer); - const cands = fixesForTask(doctor, t.taskId); + const fx = taskFixes(doctor, t); const rows: [string, ReactNode, boolean?][] = []; if (t.fix) rows.push(["Fix", ]); if (t.criterion) rows.push(["Failing check", ]); @@ -327,7 +369,6 @@ function TaskCard({ t, doctor, ctx }: { t: DoctorTask; doctor: RunDoctor; ctx: F if (t.relatedSkillPrs.length) rows.push(["Related skill PRs", ]); if (t.related) rows.push(["Related", ]); - if (t.owners.length) rows.push(["Owner", t.owners.join(", ")]); return (

)} - + {rows.length > 0 && (
{rows.map(([k, v, mono]) => ( @@ -369,54 +410,156 @@ function TaskCard({ t, doctor, ctx }: { t: DoctorTask; doctor: RunDoctor; ctx: F ); } -// page.html's "Proposed PR" row: the doctor PR when there is one, else each -// fix candidate for this task (outcome, patch, gates, cost), else "none yet" -// for a repo-fixable failure. -function ProposedFix({ t, cands, ctx }: { t: DoctorTask; cands: DoctorFix[]; ctx: FixCtx }) { - const box = "inline-flex flex-wrap items-baseline gap-x-2 rounded-md border px-3 py-1 text-xs"; - if (isSkillsPrUrl(t.doctorPr) || cands.length) { - return ( -
- {isSkillsPrUrl(t.doctorPr) && ( - + isSkillsPrUrl(href) || isAdoUrl(href) ? ( + + {children} + + ) : ( + {children} + ), + img: ({ alt }) => {alt ?? ""}, +}; + +const BOX = "rounded-md border px-3 py-1.5 text-xs"; + +function FixMeta({ c, ctx }: { c: DoctorFix; ctx: FixCtx }) { + return ( + <> + {c.patchLines && ( + + patch + + )} + {gateText(c)} + {fmtUsd(c.costUsd)} + + ); +} + +/** The doctor's prepared, unopened fix: title expands to the PR body, then the patch. */ +function PreparedFix({ c, body, ctx }: { c: DoctorFix; body: string | null; ctx: FixCtx }) { + const [open, setOpen] = useState(false); + const title = c.title || c.id; + return ( +
+
+ PR with fix + {body ? ( + + ) : ( + {title} )} - {cands.map((c) => ( - - - Proposed PR: {c.outcome} - - {c.patchLines && ( - - patch - - )} - {gateText(c)} - {fmtUsd(c.costUsd)} - {c.title && {c.title}} - - ))} +
+ {open && body && ( +
+
+ + {body} + +
+ {c.patchLines && ctx.dir && ctx.linked.has(c.id) && ( + + Download patch (+{c.patchLines.added}/-{c.patchLines.removed}) + + )} +
+ )} +
+ ); +} + +// page.html's "Proposed PR" row, split by who made the fix: "PR with fix" +// (the doctor's opened PR or prepared fix), "Proposed PR" (a third party's open +// PR the fix text names), "Fix candidate: " (not ready), or "none yet" +// for a repo-fixable failure. +function FixRows({ + t, + fx, + doctor, + ctx, +}: { + t: DoctorTask; + fx: TaskFixes; + doctor: RunDoctor; + ctx: FixCtx; +}) { + const rows: ReactNode[] = []; + if (fx.doctorPr) { + rows.push( + + PR with fix: {prLabel(fx.doctorPr)} ↗ + , ); } - if (!isFixable(t)) return null; - return ( -
- + for (const c of fx.ready) { + rows.push( + , + ); + } + for (const p of fx.proposed) { + rows.push( +
+ Proposed PR + + {prLabel(p.url)} {p.title} + + {p.state} +
, + ); + } + for (const c of fx.others) { + rows.push( +
+ Fix candidate: {c.outcome} + + {c.title && {c.title}} +
, + ); + } + if (rows.length === 0 && isFixable(t)) { + rows.push( + Proposed PR: none yet - -
- ); + , + ); + } + if (rows.length === 0) return null; + return
{rows}
; } function EnvGroup({ cause, tasks, ctx }: { cause: string; tasks: DoctorTask[]; ctx: FixCtx }) { @@ -489,8 +632,7 @@ function SkillAccordion({ {g.skill} {g.total} failed - {g.ready > 0 && - ` · ${g.ready} proposed PR${g.ready === 1 ? "" : "s"} ready`} + {g.ready > 0 && ` · ${prsWithFix(g.ready)}`} {g.layerCounts.map(([k, n]) => ( @@ -606,7 +748,10 @@ function Body({ doctor, ctx }: { doctor: RunDoctor; ctx: FixCtx }) {

Fix first

    {triage.actions.map((a, i) => { - const ready = readyTasks(doctor, a.taskIds); + const ready = countFixPrs( + doctor, + triage.tasks.filter((t) => a.taskIds.includes(t.taskId)), + ); return (
  1. @@ -619,7 +764,7 @@ function Body({ doctor, ctx }: { doctor: RunDoctor; ctx: FixCtx }) { className="ml-2 whitespace-nowrap rounded bg-studio-blue/10 px-1.5 py-px text-[11px] font-semibold text-studio-blue" data-testid="action-ready" > - {ready} PR ready + {prsWithFix(ready)} )}
  2. diff --git a/evalboard/lib/__tests__/doctor-blob.test.ts b/evalboard/lib/__tests__/doctor-blob.test.ts index 4f161004..1b9a6c10 100644 --- a/evalboard/lib/__tests__/doctor-blob.test.ts +++ b/evalboard/lib/__tests__/doctor-blob.test.ts @@ -48,6 +48,9 @@ function seedBlobs(status: string = STATUS) { [`${d}/fixes-1000001/example-grader-fix.patch`]: fixture( "fixes-1000001/example-grader-fix.patch", ), + [`${d}/fixes-1000001/example-grader-fix.md`]: fixture( + "fixes-1000001/example-grader-fix.md", + ), }; } @@ -115,6 +118,8 @@ describe("readRunDoctor: what it fetches", () => { `${D}/triage-1000001.json`, `${D}/fixes-1000001/fixes.json`, `${D}/fixes-1000001/example-grader-fix.patch`, + // The pr-ready candidate's prepared PR body. + `${D}/fixes-1000001/example-grader-fix.md`, ].sort(), ); }); @@ -183,6 +188,19 @@ describe("readRunDoctor", () => { expect(d?.status.state).toBe("done"); expect(d?.triage?.tasks).toHaveLength(5); expect(d?.fixes?.candidates).toHaveLength(2); + // Only the pr-ready candidate in patchIds gets its body. + expect(Object.keys(d!.fixBodies!)).toEqual(["example-grader-fix"]); + expect(d!.fixBodies!["example-grader-fix"]).toContain("## Summary"); + }); + + test("a body that is not in patchIds is never fetched", async () => { + const doc = JSON.parse(fixture("fixes-1000001/fixes.json")); + doc.candidates[0].patch_lines = null; + state.blobs[`${D}/fixes-1000001/fixes.json`] = JSON.stringify(doc); + const { readRunDoctor } = await import("../runs"); + const d = await readRunDoctor(RUN); + expect(d!.fixBodies).toEqual({}); + expect(state.downloads.some((n) => n.endsWith(".md"))).toBe(false); }); test("never throws: a storage error is no report", async () => { @@ -276,6 +294,8 @@ describe("readRunDoctor", () => { "skill-doctor/status.json", "skill-doctor/triage-1000001.json", "skill-doctor/fixes-1000001/fixes.json", + // Bodies travel in the page data; /api/file never serves a .md. + "skill-doctor/fixes-1000001/example-grader-fix.md", "skill-doctor/fixes-1000001/../status.json", "skill-doctor/other/x.patch", "skill-doctor/fixes-1000001/a/b.patch", diff --git a/evalboard/lib/__tests__/doctor.test.ts b/evalboard/lib/__tests__/doctor.test.ts index 8108941f..221837f9 100644 --- a/evalboard/lib/__tests__/doctor.test.ts +++ b/evalboard/lib/__tests__/doctor.test.ts @@ -158,7 +158,6 @@ describe("parseTriage / parseFixes", () => { criterion: "file_contains", passedBefore: "2099-01-01_04-20-00 (claude-code)", related: "example-t3 fails the same check", - owners: ["Example Owner", "Second Owner"], envGroup: null, doctorPr: null, }); diff --git a/evalboard/lib/__tests__/fixtures/skill-doctor/fixes-1000001/example-grader-fix.md b/evalboard/lib/__tests__/fixtures/skill-doctor/fixes-1000001/example-grader-fix.md new file mode 100644 index 00000000..80614cfb --- /dev/null +++ b/evalboard/lib/__tests__/fixtures/skill-doctor/fixes-1000001/example-grader-fix.md @@ -0,0 +1,15 @@ +## Summary + +The grader expects a literal the task prompt never asks for. This matches the +documented output instead. See https://github.com/UiPath/skills/pull/123 and +[the build](https://dev.azure.com/example-org/example-project/_build/results?buildId=1000001). + +- [an off-list link](https://example.invalid/phish) +- [a script link](javascript:alert(1)) +- ![an image](https://example.invalid/pixel.png) + +raw html + +| Before | After | +| --- | --- | +| fail | pass | diff --git a/evalboard/lib/__tests__/fixtures/skill-doctor/triage-1000001.json b/evalboard/lib/__tests__/fixtures/skill-doctor/triage-1000001.json index 9a28c3a8..0c0a03d3 100644 --- a/evalboard/lib/__tests__/fixtures/skill-doctor/triage-1000001.json +++ b/evalboard/lib/__tests__/fixtures/skill-doctor/triage-1000001.json @@ -149,10 +149,21 @@ "why": "The skill documents a removed flag.", "evidence": null, "solvable": "yes-skill", - "fix": "Document the current flag.", + "fix": "Document the current flag. Open PR #126 already does this; #127 tried before.", "related": null, "passed_before": null, - "existing_prs": [], + "existing_prs": [ + { + "url": "https://github.com/UiPath/skills/pull/126", + "title": "docs(example-two): document the current flag", + "state": "open" + }, + { + "url": "https://github.com/UiPath/skills/pull/127", + "title": "docs: older attempt", + "state": "merged" + } + ], "doctor_pr": null }, { diff --git a/evalboard/lib/doctor.ts b/evalboard/lib/doctor.ts index 8da668a6..de03199a 100644 --- a/evalboard/lib/doctor.ts +++ b/evalboard/lib/doctor.ts @@ -67,7 +67,6 @@ export interface DoctorTask { evidence: string | null; passedBefore: string | null; related: string | null; - owners: string[]; existingPrs: DoctorPr[]; relatedSkillPrs: DoctorPr[]; envGroup: string | null; @@ -118,6 +117,9 @@ export interface RunDoctor { // Set only when the status is `done` and its triage file parsed. triage: DoctorTriage | null; fixes: DoctorFixes | null; + // Prepared PR bodies (`/.md`) of pr-ready candidates in + // patchIds, by candidate id. Markdown text, rendered without raw HTML. + fixBodies?: Record; } // ---------- coercion helpers ---------- @@ -197,7 +199,6 @@ function parseTask(v: unknown): DoctorTask | null { evidence: str(v.evidence), passedBefore: str(v.passed_before), related: str(v.related), - owners: strList(v.owners), existingPrs: prList(v.existing_prs), relatedSkillPrs: prList(v.related_skill_prs), envGroup: str(v.env_group), @@ -284,6 +285,17 @@ export function patchIds(fixes: DoctorFixes | null): Set { return ids; } +/** The patchIds whose prepared PR body (`.md`) the board loads: pr-ready only. */ +export function bodyIds(fixes: DoctorFixes | null): string[] { + const ids = patchIds(fixes); + return (fixes?.candidates ?? []) + .filter((c) => c.outcome === "pr-ready" && ids.has(c.id)) + .map((c) => c.id); +} + +// A prepared PR body is a few KB; anything far larger is cut. +export const MAX_BODY_CHARS = 64_000; + /** `/.patch` → the id, or null for any other doctor path. */ export function patchIdFromPath(rel: string): { fixesDir: string; id: string } | null { const parts = rel.split("/"); diff --git a/evalboard/lib/runs.ts b/evalboard/lib/runs.ts index a459d48a..2db47c2f 100644 --- a/evalboard/lib/runs.ts +++ b/evalboard/lib/runs.ts @@ -25,6 +25,8 @@ import { parseDoctorStatus, parseFixes, parseTriage, + bodyIds, + MAX_BODY_CHARS, patchIdFromPath, patchIds, taskAnalysisEnabled, @@ -2745,13 +2747,27 @@ export async function readRunDoctor( triageRel ? load(triageRel, parseTriage) : null, fixesRel ? load(fixesRel, parseFixes) : null, ]); - // Cached patches are not fetched again (downloadBlob skips them). + // Cached patches and bodies are not fetched again (downloadBlob skips + // them). Bodies are read here and travel in the page data, so + // /api/file never serves a .md. + const fixBodies: Record = {}; if (status.fixesDir) { + const fdir = status.fixesDir; + const bodies = bodyIds(fixes); + await Promise.all([ + ...[...patchIds(fixes)].map((id) => get(`${fdir}/${id}.patch`)), + ...bodies.map((id) => get(`${fdir}/${id}.md`)), + ]); await Promise.all( - [...patchIds(fixes)].map((id) => get(`${status.fixesDir}/${id}.patch`)), + bodies.map(async (id) => { + const raw = await fs + .readFile(path.join(base, fdir, `${id}.md`), "utf-8") + .catch(() => null); + if (raw !== null) fixBodies[id] = raw.slice(0, MAX_BODY_CHARS); + }), ); } - return { status, triage, fixes }; + return { status, triage, fixes, fixBodies }; } catch (err) { console.error(`readRunDoctor failed for run ${runId}:`, err); return null; From bd3eed3dcba63d427fee93b6b26a2819ce520a48 Mon Sep 17 00:00:00 2001 From: CarlesUIPath Date: Thu, 19 Nov 2026 16:42:56 +0000 Subject: [PATCH 08/13] feat(evalboard): PR with fix links real PRs; cards show only fixing PRs; no costs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - "PR with fix: #N ↗" links the doctor's opened PR (/opened.json, fetched by name and refreshed like status.json, allowlisted URLs only) or the triage's doctor_pr, with the gate verdicts. - A pr-ready fix with no opened PR reads "Fix prepared, PR not opened yet" with the patch download; it is not a PR with fix and sorts right after linked PRs. - "PRs with fix" (header, skill accordions, Fix first badges, both sorts) counts only linked PRs: opened or doctor PRs, and third-party open PRs the fix text names. - Cards drop "Open PRs on these files" and "Related skill PRs"; related_skill_prs is no longer parsed. Fix first drops open-pr actions. - The inline PR body and its .md loading are gone; /api/file still serves .patch only. - No costs anywhere; cost fields are no longer parsed. Co-Authored-By: Claude Opus 5.5 --- .../__tests__/task-failure-analysis.test.tsx | 291 +++++++++--------- .../app/runs/[id]/task-failure-analysis.tsx | 231 ++++++-------- evalboard/lib/__tests__/doctor-blob.test.ts | 44 ++- evalboard/lib/__tests__/doctor.test.ts | 28 +- .../fixes-1000001/example-grader-fix.md | 15 - evalboard/lib/doctor.ts | 50 ++- evalboard/lib/runs.ts | 25 +- 7 files changed, 320 insertions(+), 364 deletions(-) delete mode 100644 evalboard/lib/__tests__/fixtures/skill-doctor/fixes-1000001/example-grader-fix.md diff --git a/evalboard/app/runs/[id]/__tests__/task-failure-analysis.test.tsx b/evalboard/app/runs/[id]/__tests__/task-failure-analysis.test.tsx index af597d83..c866de72 100644 --- a/evalboard/app/runs/[id]/__tests__/task-failure-analysis.test.tsx +++ b/evalboard/app/runs/[id]/__tests__/task-failure-analysis.test.tsx @@ -19,21 +19,29 @@ import { const FIX = path.join(__dirname, "..", "..", "..", "..", "lib", "__tests__", "fixtures", "skill-doctor"); const read = (p: string) => readFileSync(path.join(FIX, p), "utf-8"); const RUN = "2099-01-03_04-23-30"; +const OPENED = "https://github.com/UiPath/skills/pull/300"; function status(patch: Record = {}) { return parseDoctorStatus(JSON.stringify({ ...JSON.parse(read("status.json")), ...patch }))!; } +// The fixture: example-t1 has a pr-ready fix with no PR opened yet; example-t2 +// has a needs-live-run candidate and its fix text names open PR #126 (and +// merged #127). function doctor(patch: Record = {}): RunDoctor { const s = status(patch); return { status: s, triage: s.state === "done" ? parseTriage(read("triage-1000001.json")) : null, fixes: parseFixes(read("fixes-1000001/fixes.json")), - fixBodies: { "example-grader-fix": read("fixes-1000001/example-grader-fix.md") }, + opened: {}, }; } +function withOpened(d: RunDoctor = doctor()): RunDoctor { + return { ...d, opened: { "example-t1": OPENED } }; +} + function renderBlock(d: RunDoctor = doctor(), sourceId = "skills") { return render(); } @@ -44,17 +52,19 @@ const openBlock = () => fireEvent.click(header()); const skillButton = (skill: string) => screen.getByRole("button", { name: new RegExp(`^▸?▾? ?${skill} `) }); const openSkill = (skill: string) => fireEvent.click(skillButton(skill)); +const card = (taskId: string) => + screen.getAllByTestId("task-card").find((c) => c.textContent!.includes(taskId))!; describe("digest", () => { - test("done: errors and PRs with fix (prepared + proposed)", () => { - // example-grader-fix (pr-ready) + open PR #126 named by example-t2's fix. - expect(digest(doctor())).toBe("5 errors · 2 PRs with fix"); + test("done: errors and linked PRs with fix; a prepared fix does not count", () => { + // Only #126 (proposed) is linked; example-t1's fix is not opened yet. + expect(digest(doctor())).toBe("5 errors · 1 PR with fix"); + expect(digest(withOpened())).toBe("5 errors · 2 PRs with fix"); }); - test("one kind only, and none", () => { + test("no linked PR: the errors only", () => { const d = doctor(); - expect(digest({ ...d, fixes: null })).toBe("5 errors · 1 PR with fix"); d.triage!.tasks[2].fix = "Document the current flag."; - expect(digest({ ...d, fixes: null })).toBe("5 errors"); + expect(digest(d)).toBe("5 errors"); }); test("not done: the state", () => { expect(digest(doctor({ state: "running" }))).toMatch(/^running since /); @@ -73,7 +83,7 @@ describe("TaskFailureAnalysis block", () => { renderBlock(); expect(header()).toHaveAttribute("aria-expanded", "false"); expect(header()).toHaveTextContent("Task failure analysis"); - expect(header()).toHaveTextContent("5 errors · 2 PRs with fix"); + expect(header()).toHaveTextContent("5 errors · 1 PR with fix"); expect(screen.queryByText("Fix first")).toBeNull(); expect(screen.queryByText("Skill failures")).toBeNull(); }); @@ -94,10 +104,21 @@ describe("TaskFailureAnalysis block", () => { expect(at("rows run")).toBeLessThan(at("Failures by layer")); expect(at("Failures by layer")).toBeLessThan(at("Fix first")); expect(at("Fix first")).toBeLessThan(at("Skill failures")); - // No separate fix table or task table any more. expect(screen.queryByRole("table")).toBeNull(); }); + test("no costs anywhere in the panel", () => { + renderBlock(withOpened()); + openBlock(); + fireEvent.click(screen.getByRole("button", { name: "Expand all" })); + for (const b of within(block()).getAllByTestId("env-group")) { + fireEvent.click(within(b).getByRole("button")); + } + expect(block().textContent).not.toContain("$"); + renderBlock(doctor({ state: "running" })); + expect(document.body.textContent).not.toContain("$"); + }); + test("stats as on the doctor page", () => { renderBlock(); openBlock(); @@ -110,32 +131,49 @@ describe("TaskFailureAnalysis block", () => { expect(screen.getByRole("img", { name: "Failures by layer" }).children).toHaveLength(3); }); - test("Fix first: ranked actions, allowlisted links, a PRs-with-fix badge", () => { - renderBlock(); + test("Fix first: env and layer actions only, allowlisted links, a PRs-with-fix badge", () => { + const d = withOpened(); + d.triage!.actions[1].text += + " See https://github.com/UiPath/skills/pull/123, javascript:alert(1) and https://github.com.example.invalid/x."; + renderBlock(d); openBlock(); - const fixFirst = screen.getByRole("region", { name: "Fix first" }); - const items = within(fixFirst).getAllByRole("listitem"); - expect(items).toHaveLength(3); + const items = within(screen.getByRole("region", { name: "Fix first" })).getAllByRole("listitem"); + // The fixture's open-pr action is left out. + expect(items).toHaveLength(2); + expect(block()).not.toHaveTextContent("touches the files"); expect(items[0]).toHaveTextContent(/^3 · Environment/); - expect(within(items[1]).getByRole("link")).toHaveAttribute( - "href", + expect(within(items[1]).getAllByRole("link").map((a) => a.getAttribute("href"))).toEqual([ "https://github.com/UiPath/skills/pull/123", - ); - // javascript: and look-alike hosts stay text. - expect(within(items[1]).getAllByRole("link")).toHaveLength(1); + ]); expect(items[1]).toHaveTextContent("javascript:alert(1)"); expect(within(items[1]).getByTestId("action-ready")).toHaveTextContent("1 PR with fix"); expect(within(items[0]).queryByTestId("action-ready")).toBeNull(); }); - test("skills: grouped and sorted, all collapsed, with counts and layer dots", () => { + test("a prepared fix gives no badge", () => { + renderBlock(); + openBlock(); + expect(screen.queryByTestId("action-ready")).toBeNull(); + }); + + test("no env or layer actions: no Fix first section", () => { + const d = doctor(); + d.triage!.actions = []; + renderBlock(d); + openBlock(); + expect(screen.queryByRole("region", { name: "Fix first" })).toBeNull(); + }); + + test("skills: grouped, all collapsed, with linked-PR counts and layer dots", () => { renderBlock(); openBlock(); const a = skillButton("uipath-example"); const b = skillButton("uipath-example-two"); expect(a).toHaveAttribute("aria-expanded", "false"); expect(b).toHaveAttribute("aria-expanded", "false"); - expect(a).toHaveTextContent("2 failed · 1 PR with fix"); + // example-t1's fix is only prepared: it does not count. + expect(a).toHaveTextContent("2 failed"); + expect(a).not.toHaveTextContent("with fix"); expect(a).toHaveTextContent("1 environment"); expect(a).toHaveTextContent("1 grader"); expect(b).toHaveTextContent("3 failed · 1 PR with fix"); @@ -156,124 +194,87 @@ describe("TaskFailureAnalysis block", () => { "example-e2", "example-e3", ]); - // The one non-env task is a card. expect(screen.getAllByTestId("task-card")).toHaveLength(1); }); - test("a card: task link, pills, diagnosis, proposed PR row, dl fields", () => { + test("a card: task link, pills, diagnosis, dl fields; no file-touching PRs, no owner", () => { renderBlock(); openBlock(); openSkill("uipath-example"); - const card = screen - .getAllByTestId("task-card") - .find((c) => c.textContent!.includes("example-t1"))!; - expect(within(card).getByRole("link", { name: "example-t1" })).toHaveAttribute( + const c = card("example-t1"); + expect(within(c).getByRole("link", { name: "example-t1" })).toHaveAttribute( "href", `/runs/${RUN}/example-t1`, ); for (const pill of ["Grader", "Fixable: grader", "failure", "high confidence"]) { - expect(within(card).getByText(pill)).toBeInTheDocument(); + expect(within(c).getByText(pill)).toBeInTheDocument(); } - expect(card).toHaveTextContent("The grader expects a literal"); - const fix = within(card).getByTestId("pr-with-fix"); - expect(fix).toHaveTextContent("PR with fix"); - expect(fix).toHaveTextContent("fix(example): match the documented output"); - expect(fix).toHaveTextContent("replay pass · mutants 9/10 · review SEND"); - expect(fix).toHaveTextContent("$1.50"); - const patch = within(fix).getByRole("link", { name: "+3/-1" }); - const href = new URL(patch.getAttribute("href")!, "http://x"); - expect(href.pathname).toBe("/api/file"); - expect(href.searchParams.get("path")).toBe( - "skill-doctor/fixes-1000001/example-grader-fix.patch", - ); - expect(within(card).queryByTestId("fix-candidate")).toBeNull(); - // #123 is open on these files, but the fix text does not name it. - expect(within(card).queryByTestId("proposed-pr")).toBeNull(); - const dts = within(card).getAllByRole("term").map((d) => d.textContent); - expect(dts).toEqual([ + expect(c).toHaveTextContent("The grader expects a literal"); + expect(within(c).getAllByRole("term").map((d) => d.textContent)).toEqual([ "Fix", "Failing check", "Evidence", "Passed before", - "Open PRs on these files", - "Related skill PRs", "Related", ]); - expect(within(card).getByText("literal").tagName).toBe("CODE"); - expect(within(card).getByRole("link", { name: "#123" })).toBeInTheDocument(); - expect(within(card).getByRole("link", { name: "#125" })).toBeInTheDocument(); - expect(within(card).getByText("javascript:alert(1)").tagName).toBe("SPAN"); - expect(card).not.toHaveTextContent("Example Owner"); + expect(within(c).getByText("literal").tagName).toBe("CODE"); + // #123 (open on these files) and #125 (related skill PR) are not shown. + expect(c).not.toHaveTextContent("#123"); + expect(c).not.toHaveTextContent("#125"); + expect(c).not.toHaveTextContent("Example Owner"); + expect(within(c).queryByTestId("proposed-pr")).toBeNull(); }); - test("a fix candidate that is not ready, and an unsafe patch id", () => { - const d = doctor(); - d.fixes!.candidates[0].id = "../bad"; - renderBlock(d); - openBlock(); - fireEvent.click(screen.getByRole("button", { name: "Expand all" })); - const t1 = screen.getAllByTestId("task-card").find((c) => c.textContent!.includes("example-t1"))!; - expect(within(t1).getByText("+3/-1").closest("a")).toBeNull(); - const t2 = screen.getAllByTestId("task-card").find((c) => c.textContent!.includes("example-t2"))!; - expect(within(t2).getByTestId("fix-candidate")).toHaveTextContent("Fix candidate: needs-live-run"); - }); - - test("the PR with fix title expands to the prepared body, then the patch", () => { + test("a pr-ready fix with no PR opened: plain text, patch and gates, no link", () => { renderBlock(); openBlock(); openSkill("uipath-example"); - const fix = screen.getByTestId("pr-with-fix"); - expect(within(fix).queryByTestId("pr-body")).toBeNull(); - const title = within(fix).getByRole("button", { name: "fix(example): match the documented output" }); - expect(title).toHaveAttribute("aria-expanded", "false"); - fireEvent.click(title); - const body = within(fix).getByTestId("pr-body"); - expect(within(body).getByRole("heading", { name: "Summary" })).toBeInTheDocument(); - expect(within(body).getByRole("table")).toBeInTheDocument(); - const links = within(body).getAllByRole("link").map((a) => a.getAttribute("href")); - expect(links).toEqual([ - "https://github.com/UiPath/skills/pull/123", - "https://dev.azure.com/example-org/example-project/_build/results?buildId=1000001", - ]); - // Off-list and script links are text; images are alt text; raw HTML is dropped. - expect(within(body).getByText("an off-list link").tagName).toBe("SPAN"); - expect(within(body).getByText("a script link").tagName).toBe("SPAN"); - expect(within(body).queryByRole("img")).toBeNull(); - expect(within(body).getByText("an image")).toBeInTheDocument(); - expect(body.querySelector("b")).toBeNull(); - const dl = within(fix).getByRole("link", { name: "Download patch (+3/-1)" }); - expect(new URL(dl.getAttribute("href")!, "http://x").searchParams.get("path")).toBe( + const c = card("example-t1"); + expect(within(c).queryByTestId("pr-with-fix")).toBeNull(); + const row = within(c).getByTestId("fix-prepared"); + expect(row).toHaveTextContent("Fix prepared, PR not opened yet"); + expect(row).toHaveTextContent("replay pass · mutants 9/10 · review SEND"); + expect(within(row).getAllByRole("link")).toHaveLength(1); + const patch = within(row).getByRole("link", { name: "+3/-1" }); + const href = new URL(patch.getAttribute("href")!, "http://x"); + expect(href.pathname).toBe("/api/file"); + expect(href.searchParams.get("path")).toBe( "skill-doctor/fixes-1000001/example-grader-fix.patch", ); }); - test("no body: the title is plain text", () => { - renderBlock({ ...doctor(), fixBodies: {} }); + test("an opened PR (opened.json): 'PR with fix: #N ↗' with the gates", () => { + renderBlock(withOpened()); openBlock(); openSkill("uipath-example"); - const fix = screen.getByTestId("pr-with-fix"); - expect(within(fix).queryByRole("button")).toBeNull(); - expect(fix).toHaveTextContent("fix(example): match the documented output"); + const c = card("example-t1"); + const row = within(c).getByTestId("pr-with-fix"); + expect(within(row).getByRole("link", { name: "PR with fix: #300 ↗" })).toHaveAttribute( + "href", + OPENED, + ); + expect(row).toHaveTextContent("replay pass · mutants 9/10 · review SEND"); + expect(within(c).queryByTestId("fix-prepared")).toBeNull(); }); - test("a doctor PR replaces the prepared fix with a GitHub link", () => { + test("a doctor PR from the triage counts the same way", () => { const d = doctor(); d.triage!.tasks[1].doctorPr = "https://github.com/UiPath/skills/pull/200"; + expect(digest(d)).toBe("5 errors · 2 PRs with fix"); renderBlock(d); openBlock(); openSkill("uipath-example"); - const rows = screen.getAllByTestId("pr-with-fix"); - expect(rows).toHaveLength(1); - expect(rows[0]).toHaveTextContent("PR with fix: #200"); - expect(rows[0]).toHaveAttribute("href", "https://github.com/UiPath/skills/pull/200"); + expect( + within(card("example-t1")).getByRole("link", { name: "PR with fix: #200 ↗" }), + ).toBeInTheDocument(); }); test("a third-party open PR named in the fix text: a Proposed PR row", () => { renderBlock(); openBlock(); openSkill("uipath-example-two"); - const card = screen.getAllByTestId("task-card").find((c) => c.textContent!.includes("example-t2"))!; - const rows = within(card).getAllByTestId("proposed-pr"); + const c = card("example-t2"); + const rows = within(c).getAllByTestId("proposed-pr"); // #127 is named too, but merged: it does not count. expect(rows).toHaveLength(1); expect(rows[0]).toHaveTextContent("Proposed PR#126 docs(example-two): document the current flagopen"); @@ -281,7 +282,16 @@ describe("TaskFailureAnalysis block", () => { "href", "https://github.com/UiPath/skills/pull/126", ); - expect(within(card).getByTestId("fix-candidate")).toHaveTextContent("Fix candidate: needs-live-run"); + expect(within(c).getByTestId("fix-candidate")).toHaveTextContent("Fix candidate: needs-live-run"); + }); + + test("an unsafe patch id shows its size without a link", () => { + const d = doctor(); + d.fixes!.candidates[0].id = "../bad"; + renderBlock(d); + openBlock(); + openSkill("uipath-example"); + expect(within(card("example-t1")).getByText("+3/-1").closest("a")).toBeNull(); }); test("a task id the task page would reject is plain text", () => { @@ -348,64 +358,68 @@ describe("TaskFailureAnalysis block", () => { }); describe("groupBySkill", () => { - test("skills sort by PRs ready, then failed tasks, then name", () => { - // uipath-example: 1 PR with fix of 2 failed; -two: 1 (proposed #126) of 3. + test("skills: linked PRs first, then prepared fixes, then failed tasks, then name", () => { + // -two: 1 linked (#126) of 3; uipath-example: 0 linked, 1 prepared, of 2. expect(groupBySkill(doctor(), null, "").map((g) => g.skill)).toEqual([ "uipath-example-two", "uipath-example", ]); - // More PRs with fix wins over more failures. - const more = doctor(); - more.fixes!.candidates.push({ - ...more.fixes!.candidates[0], - id: "second-ready", - taskId: "example-e1", - covers: ["example-e1"], - }); - expect(groupBySkill(more, null, "").map((g) => g.skill)).toEqual([ + // Without #126, the prepared fix beats more failed tasks. + const d = doctor(); + d.triage!.tasks[2].fix = "no PR named"; + expect(groupBySkill(d, null, "").map((g) => g.skill)).toEqual([ "uipath-example", "uipath-example-two", ]); - // Equal on both: by name. - const tie = doctor(); - tie.fixes = null; - tie.triage!.tasks = [ - { ...tie.triage!.tasks[1], skill: "b-skill", taskId: "x1" }, - { ...tie.triage!.tasks[1], skill: "a-skill", taskId: "x2" }, - ]; + // Nothing at all: more failed tasks first. + const none = { ...d, fixes: null }; + expect(groupBySkill(none, null, "").map((g) => g.skill)).toEqual([ + "uipath-example-two", + "uipath-example", + ]); + // Equal on everything: by name. + const tie = { ...none, triage: { ...none.triage!, tasks: [ + { ...none.triage!.tasks[1], skill: "b-skill", taskId: "x1" }, + { ...none.triage!.tasks[1], skill: "a-skill", taskId: "x2" }, + ] } }; expect(groupBySkill(tie, null, "").map((g) => g.skill)).toEqual(["a-skill", "b-skill"]); }); - test("cards: own fix, then proposed PR, then other candidates, then the rest", () => { + test("cards: own PR, proposed PR, prepared fix, other candidates, the rest", () => { const d = doctor(); - const base = d.triage!.tasks[1]; + const base = { ...d.triage!.tasks[1], doctorPr: null, fix: "plain" }; d.triage!.tasks = [ - { ...base, taskId: "z-plain-skill", layer: "skill", doctorPr: null }, - { ...base, taskId: "a-plain-grader", layer: "grader", doctorPr: null }, - { ...base, taskId: "m-live", layer: "grader", doctorPr: null }, - { ...base, taskId: "y-ready", layer: "grader", doctorPr: null }, + { ...base, taskId: "z-plain-skill", layer: "skill" }, + { ...base, taskId: "a-plain-grader", layer: "grader" }, + { ...base, taskId: "m-live", layer: "grader" }, + { ...base, taskId: "y-prepared", layer: "grader" }, + { ...base, taskId: "w-opened", layer: "grader" }, { ...base, taskId: "x-doctor-pr", layer: "grader", doctorPr: "https://github.com/UiPath/skills/pull/7" }, { ...base, taskId: "b-bad-pr", layer: "grader", doctorPr: "https://example.invalid/pull/7" }, - { ...base, taskId: "p-proposed", layer: "grader", doctorPr: null, fix: "see #123" }, + { ...base, taskId: "p-proposed", layer: "grader", fix: "see #123" }, { ...d.triage!.tasks[0], skill: base.skill }, ]; d.fixes!.candidates = [ - { ...d.fixes!.candidates[0], id: "f1", taskId: "y-ready", covers: ["y-ready"] }, + { ...d.fixes!.candidates[0], id: "f1", taskId: "y-prepared", covers: ["y-prepared"] }, + { ...d.fixes!.candidates[0], id: "f3", taskId: "w-opened", covers: ["w-opened"] }, { ...d.fixes!.candidates[1], id: "f2", taskId: "m-live", covers: ["m-live"] }, ]; + d.opened = { "w-opened": "https://github.com/UiPath/skills/pull/8" }; const [g] = groupBySkill(d, null, ""); expect(g.cards.map((t) => t.taskId)).toEqual([ + "w-opened", "x-doctor-pr", - "y-ready", "p-proposed", + "y-prepared", "m-live", // The rest: skill ranks after grader on the doctor page, so first. "z-plain-skill", "a-plain-grader", "b-bad-pr", ]); - // Env tasks stay in their group, rendered after the cards. expect(g.envGroups.map((e) => e.tasks.map((t) => t.taskId))).toEqual([["example-e1"]]); + // Linked: #8, #7, #123. The prepared f1 is not. + expect(g.ready).toBe(3); }); test("the rendered accordions follow the same order", () => { @@ -419,16 +433,6 @@ describe("groupBySkill", () => { expect(order).toEqual(["uipath-example", "uipath-example-two"]); }); - test("cards sorted by layer rank, env tasks grouped", () => { - const g = groupBySkill(doctor(), null, ""); - const ex = g.find((x) => x.skill === "uipath-example")!; - expect(ex.cards.map((t) => t.taskId)).toEqual(["example-t1"]); - expect(ex.envGroups).toEqual([ - { cause: "connection example-conn", tasks: [expect.objectContaining({ taskId: "example-e1" })] }, - ]); - expect(g.map((x) => x.ready)).toEqual([1, 1]); - }); - test("proposedPrs: named in the fix text, open, allowlisted", () => { const t = doctor().triage!.tasks[2]; expect(proposedPrs(t).map((p) => p.url)).toEqual(["https://github.com/UiPath/skills/pull/126"]); @@ -441,15 +445,14 @@ describe("groupBySkill", () => { existingPrs: [{ url: "https://github.com.example.invalid/UiPath/skills/pull/126", title: "", state: "open" }], }), ).toHaveLength(0); - expect( - proposedPrs({ ...t, existingPrs: [{ ...t.existingPrs[0], state: "unknown" }] }), - ).toHaveLength(0); + expect(proposedPrs({ ...t, existingPrs: [{ ...t.existingPrs[0], state: "unknown" }] })).toHaveLength(0); }); - test("countFixPrs counts a PR once across the tasks it covers", () => { + test("countFixPrs counts each linked PR once, and never a prepared fix", () => { const d = doctor(); - d.fixes!.candidates[0].covers = ["example-t1", "example-t2"]; - // example-grader-fix covers both; #126 on t2: 2 distinct PRs. - expect(countFixPrs(d, d.triage!.tasks)).toBe(2); + d.triage!.tasks[1].fix = "see #126"; + d.triage!.tasks[1].existingPrs = d.triage!.tasks[2].existingPrs; + // t1 and t2 both name #126: one PR. + expect(countFixPrs(d, d.triage!.tasks)).toBe(1); }); }); diff --git a/evalboard/app/runs/[id]/task-failure-analysis.tsx b/evalboard/app/runs/[id]/task-failure-analysis.tsx index 16fda432..56f96bf1 100644 --- a/evalboard/app/runs/[id]/task-failure-analysis.tsx +++ b/evalboard/app/runs/[id]/task-failure-analysis.tsx @@ -2,10 +2,8 @@ import Link from "next/link"; import { useMemo, useState, type ReactNode } from "react"; -import ReactMarkdown, { type Components } from "react-markdown"; -import remarkGfm from "remark-gfm"; import { withSource } from "@/app/_lib/source-param"; -import { fmtTimestamp, fmtUsd } from "@/lib/format"; +import { fmtTimestamp } from "@/lib/format"; import { DOCTOR_DIR, isAdoUrl, @@ -121,23 +119,6 @@ function prLabel(url: string): string { return m ? `#${m[1]}` : url; } -function PrList({ prs }: { prs: DoctorPr[] }) { - return ( - <> - {prs.map((p, i) => ( - - {i > 0 && "; "} - - {isSkillsPrUrl(p.url) ? prLabel(p.url) : p.url} - - {p.title && ` ${p.title}`} - ({p.state}) - - ))} - - ); -} - // ---------- digest + state ---------- export function digest(doctor: RunDoctor): string { @@ -209,8 +190,10 @@ export interface SkillGroup { cards: DoctorTask[]; envGroups: { cause: string; tasks: DoctorTask[] }[]; layerCounts: [string, number][]; - // Distinct PRs with a fix among the skill's tasks (countFixPrs). + // Distinct linked PRs with a fix among the skill's tasks (countFixPrs). ready: number; + // Tasks with a fix prepared but no PR yet: sorts right after linked PRs. + prepared: number; total: number; } @@ -238,35 +221,51 @@ export function proposedPrs(t: DoctorTask): DoctorPr[] { } export interface TaskFixes { - // The doctor's own fix: its opened PR (allowlisted), else its prepared - // pr-ready candidates. Both render as "PR with fix". - doctorPr: string | null; + // The doctor's own PR for this task, opened (opened.json) or recorded in + // the triage (doctor_pr); allowlisted. "PR with fix". + ownPr: string | null; + // The pr-ready candidates behind it, for their gate verdicts. ready: DoctorFix[]; + // pr-ready candidates with no PR opened yet: "Fix prepared, PR not + // opened yet". Not a PR with fix. + prepared: DoctorFix[]; // Third-party open PRs the fix text points at ("Proposed PR"). proposed: DoctorPr[]; // Other fix candidates (needs-live-run, ...). others: DoctorFix[]; - // One key per distinct PR with a fix, for counting across tasks. + // One key per distinct linked PR (own or proposed), for counting. keys: string[]; - // Card sort: own fix, then proposed PR, then other candidates, then rest. + // Card sort: own PR, proposed PR, prepared fix, other candidates, rest. tier: number; } export function taskFixes(doctor: RunDoctor, t: DoctorTask): TaskFixes { const cands = fixesForTask(doctor, t.taskId); - const doctorPr = isSkillsPrUrl(t.doctorPr) ? t.doctorPr : null; - const ready = doctorPr ? [] : cands.filter(isReady); + const opened = doctor.opened?.[t.taskId]; + const ownPr = isSkillsPrUrl(opened) ? opened : isSkillsPrUrl(t.doctorPr) ? t.doctorPr : null; + const ready = cands.filter(isReady); + const prepared = ownPr ? [] : ready; const others = cands.filter((c) => !isReady(c)); const proposed = proposedPrs(t); - const keys = [ - ...(doctorPr ? [`pr:${doctorPr}`] : ready.map((c) => `fix:${c.id}`)), - ...proposed.map((p) => `pr:${p.url}`), - ]; - const tier = doctorPr || ready.length ? 0 : proposed.length ? 1 : others.length ? 2 : 3; - return { doctorPr, ready, proposed, others, keys, tier }; + const keys = [...(ownPr ? [ownPr] : []), ...proposed.map((p) => p.url)]; + const tier = ownPr + ? 0 + : proposed.length + ? 1 + : prepared.length + ? 2 + : others.length + ? 3 + : 4; + return { ownPr, ready, prepared, proposed, others, keys, tier }; +} + +/** Tasks with a fix prepared but no PR opened (sort tie-break only). */ +export function countPrepared(doctor: RunDoctor, tasks: DoctorTask[]): number { + return tasks.filter((t) => taskFixes(doctor, t).prepared.length > 0).length; } -/** Distinct PRs with a fix (own or proposed) over these tasks. */ +/** Distinct linked PRs with a fix (own or proposed) over these tasks. */ export function countFixPrs(doctor: RunDoctor, tasks: DoctorTask[]): number { return new Set(tasks.flatMap((t) => taskFixes(doctor, t).keys)).size; } @@ -275,7 +274,8 @@ const prsWithFix = (n: number) => `${n} PR${n === 1 ? "" : "s"} with fix`; /** * Visible failures by skill, as the doctor's page groups them. Skills with the - * most PRs with fix come first; ties go to more failed tasks, then name. + * most PRs with fix come first, then the most prepared fixes; ties go to more + * failed tasks, then name. */ export function groupBySkill( doctor: RunDoctor, @@ -319,12 +319,16 @@ export function groupBySkill( envGroups: [...groups.entries()].map(([cause, tasks]) => ({ cause, tasks })), layerCounts: [...lc.entries()].sort((a, b) => rank(a[0]) - rank(b[0])), ready: countFixPrs(doctor, items), + prepared: countPrepared(doctor, items), total: items.length, }; }) .sort( (a, b) => - b.ready - a.ready || b.total - a.total || a.skill.localeCompare(b.skill), + b.ready - a.ready || + b.prepared - a.prepared || + b.total - a.total || + a.skill.localeCompare(b.skill), ); } @@ -365,9 +369,6 @@ function TaskCard({ t, doctor, ctx }: { t: DoctorTask; doctor: RunDoctor; ctx: F if (t.criterion) rows.push(["Failing check", ]); if (t.evidence) rows.push(["Evidence", , true]); if (t.passedBefore) rows.push(["Passed before", ]); - if (t.existingPrs.length) rows.push(["Open PRs on these files", ]); - if (t.relatedSkillPrs.length) - rows.push(["Related skill PRs", ]); if (t.related) rows.push(["Related", ]); return (

    )} - + {rows.length > 0 && (
    {rows.map(([k, v, mono]) => ( @@ -410,112 +411,37 @@ function TaskCard({ t, doctor, ctx }: { t: DoctorTask; doctor: RunDoctor; ctx: F ); } -// The prepared PR body is model-written markdown: no raw HTML (ReactMarkdown -// drops it by default), links only for allowlisted URLs, images as alt text. -const BODY_COMPONENTS: Components = { - a: ({ href, children }) => - isSkillsPrUrl(href) || isAdoUrl(href) ? ( - - {children} - - ) : ( - {children} - ), - img: ({ alt }) => {alt ?? ""}, -}; - const BOX = "rounded-md border px-3 py-1.5 text-xs"; -function FixMeta({ c, ctx }: { c: DoctorFix; ctx: FixCtx }) { - return ( - <> - {c.patchLines && ( - - patch - - )} - {gateText(c)} - {fmtUsd(c.costUsd)} - - ); -} - -/** The doctor's prepared, unopened fix: title expands to the PR body, then the patch. */ -function PreparedFix({ c, body, ctx }: { c: DoctorFix; body: string | null; ctx: FixCtx }) { - const [open, setOpen] = useState(false); - const title = c.title || c.id; - return ( -
    -
    - PR with fix - {body ? ( - - ) : ( - {title} - )} - -
    - {open && body && ( -
    -
    - - {body} - -
    - {c.patchLines && ctx.dir && ctx.linked.has(c.id) && ( - - Download patch (+{c.patchLines.added}/-{c.patchLines.removed}) - - )} -
    - )} -
    - ); +function Gates({ cands }: { cands: DoctorFix[] }) { + const g = cands.map(gateText).filter((x) => x !== "—"); + if (!g.length) return null; + return {g.join("; ")}; } // page.html's "Proposed PR" row, split by who made the fix: "PR with fix" // (the doctor's opened PR or prepared fix), "Proposed PR" (a third party's open // PR the fix text names), "Fix candidate: " (not ready), or "none yet" // for a repo-fixable failure. -function FixRows({ - t, - fx, - doctor, - ctx, -}: { - t: DoctorTask; - fx: TaskFixes; - doctor: RunDoctor; - ctx: FixCtx; -}) { +function FixRows({ t, fx, ctx }: { t: DoctorTask; fx: TaskFixes; ctx: FixCtx }) { const rows: ReactNode[] = []; - if (fx.doctorPr) { + if (fx.ownPr) { rows.push( - - PR with fix: {prLabel(fx.doctorPr)} ↗ - , - ); - } - for (const c of fx.ready) { - rows.push( - , + + PR with fix: {prLabel(fx.ownPr)} ↗ + + +
, ); } for (const p of fx.proposed) { @@ -538,6 +464,23 @@ function FixRows({ , ); } + for (const c of fx.prepared) { + rows.push( +
+ Fix prepared, PR not opened yet + {c.patchLines && ( + + patch + + )} + +
, + ); + } for (const c of fx.others) { rows.push(
Fix candidate: {c.outcome} - - {c.title && {c.title}} + {c.patchLines && ( + + patch + + )} +
, ); } @@ -937,14 +884,8 @@ export function TaskFailureAnalysis({ {hasBody && open && ( <>
- skill-doctor · diagnosis {fmtUsd(doctor.status.costUsd)} - {doctor.fixes && ( - <> - {" · fixes "} - {fmtUsd(doctor.fixes.costUsd)} - {doctor.fixes.fixMode && ` (${doctor.fixes.fixMode})`} - - )} + skill-doctor + {doctor.fixes?.fixMode && ` · fix mode ${doctor.fixes.fixMode}`}
diff --git a/evalboard/lib/__tests__/doctor-blob.test.ts b/evalboard/lib/__tests__/doctor-blob.test.ts index 1b9a6c10..4d05bb00 100644 --- a/evalboard/lib/__tests__/doctor-blob.test.ts +++ b/evalboard/lib/__tests__/doctor-blob.test.ts @@ -48,9 +48,7 @@ function seedBlobs(status: string = STATUS) { [`${d}/fixes-1000001/example-grader-fix.patch`]: fixture( "fixes-1000001/example-grader-fix.patch", ), - [`${d}/fixes-1000001/example-grader-fix.md`]: fixture( - "fixes-1000001/example-grader-fix.md", - ), + [`${d}/fixes-1000001/opened.json`]: JSON.stringify({ opened: [] }), }; } @@ -118,8 +116,7 @@ describe("readRunDoctor: what it fetches", () => { `${D}/triage-1000001.json`, `${D}/fixes-1000001/fixes.json`, `${D}/fixes-1000001/example-grader-fix.patch`, - // The pr-ready candidate's prepared PR body. - `${D}/fixes-1000001/example-grader-fix.md`, + `${D}/fixes-1000001/opened.json`, ].sort(), ); }); @@ -188,19 +185,35 @@ describe("readRunDoctor", () => { expect(d?.status.state).toBe("done"); expect(d?.triage?.tasks).toHaveLength(5); expect(d?.fixes?.candidates).toHaveLength(2); - // Only the pr-ready candidate in patchIds gets its body. - expect(Object.keys(d!.fixBodies!)).toEqual(["example-grader-fix"]); - expect(d!.fixBodies!["example-grader-fix"]).toContain("## Summary"); + expect(d!.opened).toEqual({}); }); - test("a body that is not in patchIds is never fetched", async () => { - const doc = JSON.parse(fixture("fixes-1000001/fixes.json")); - doc.candidates[0].patch_lines = null; - state.blobs[`${D}/fixes-1000001/fixes.json`] = JSON.stringify(doc); + test("opened.json maps opened PRs by task; a bad host or a problem entry does not", async () => { + state.blobs[`${D}/fixes-1000001/opened.json`] = JSON.stringify({ + opened: [ + { task_id: "example-t1", pr_url: "https://github.com/UiPath/skills/pull/300" }, + { task_id: "example-t2", pr_url: "https://evil.example/UiPath/skills/pull/301" }, + { task_id: "example-e1", problem: "push refused" }, + ], + }); const { readRunDoctor } = await import("../runs"); const d = await readRunDoctor(RUN); - expect(d!.fixBodies).toEqual({}); - expect(state.downloads.some((n) => n.endsWith(".md"))).toBe(false); + expect(d!.opened).toEqual({ "example-t1": "https://github.com/UiPath/skills/pull/300" }); + }); + + test("no or malformed opened.json: no opened PRs, no throw", async () => { + delete state.blobs[`${D}/fixes-1000001/opened.json`]; + const { readRunDoctor } = await import("../runs"); + expect((await readRunDoctor(RUN))!.opened).toEqual({}); + vi.resetModules(); + state.blobs[`${D}/fixes-1000001/opened.json`] = "{not json"; + const warn = vi.spyOn(console, "warn").mockImplementation(() => {}); + const { readRunDoctor: again } = await import("../runs"); + // Cached copy from before is fresh, so drop it to read the bad one. + await fs.rm(path.join(root, RUN, "skill-doctor", "fixes-1000001", "opened.json"), { force: true }); + expect((await again(RUN))!.opened).toEqual({}); + expect(warn).toHaveBeenCalled(); + warn.mockRestore(); }); test("never throws: a storage error is no report", async () => { @@ -294,8 +307,9 @@ describe("readRunDoctor", () => { "skill-doctor/status.json", "skill-doctor/triage-1000001.json", "skill-doctor/fixes-1000001/fixes.json", - // Bodies travel in the page data; /api/file never serves a .md. + // /api/file serves .patch files only. "skill-doctor/fixes-1000001/example-grader-fix.md", + "skill-doctor/fixes-1000001/opened.json", "skill-doctor/fixes-1000001/../status.json", "skill-doctor/other/x.patch", "skill-doctor/fixes-1000001/a/b.patch", diff --git a/evalboard/lib/__tests__/doctor.test.ts b/evalboard/lib/__tests__/doctor.test.ts index 221837f9..661340a6 100644 --- a/evalboard/lib/__tests__/doctor.test.ts +++ b/evalboard/lib/__tests__/doctor.test.ts @@ -8,6 +8,7 @@ import { isSkillsPrUrl, parseDoctorStatus, parseFixes, + parseOpened, parseTriage, patchIds, planAnalysis, @@ -93,7 +94,6 @@ describe("parseDoctorStatus", () => { expect(s.state).toBe("done"); expect(s.triageFile).toBe("triage-1000001.json"); expect(s.fixesDir).toBe("fixes-1000001"); - expect(s.costUsd).toBe(1.25); }); test("schema_version 2 is absent", () => { @@ -162,7 +162,10 @@ describe("parseTriage / parseFixes", () => { doctorPr: null, }); expect(t1.existingPrs).toHaveLength(3); - expect(t1.relatedSkillPrs[0].url).toBe("https://github.com/UiPath/skills/pull/125"); + expect(t1).not.toHaveProperty("relatedSkillPrs"); + // open-pr actions (PRs that merely touch the files) are left out. + expect(t.actions.map((a) => a.text)).not.toContainEqual(expect.stringMatching(/^Open PR/)); + expect(t.actions).toHaveLength(2); expect(t.tasks[0].envGroup).toBe("connection example-conn"); }); @@ -172,6 +175,27 @@ describe("parseTriage / parseFixes", () => { expect(parseTriage(JSON.stringify({ schema_version: 1 }))).toBeNull(); }); + test("parseOpened: allowlisted PR URLs map by task; the rest is skipped", () => { + expect( + parseOpened( + JSON.stringify({ + opened: [ + { task_id: "a", pr_url: "https://github.com/UiPath/skills/pull/300", branch: "x" }, + { task_id: "b", pr_url: "https://github.com.example.invalid/UiPath/skills/pull/301" }, + { task_id: "c", pr_url: "javascript:alert(1)" }, + { task_id: "d", problem: "push refused" }, + { pr_url: "https://github.com/UiPath/skills/pull/302" }, + "junk", + ], + error: "1 of 5 failed", + }), + ), + ).toEqual({ a: "https://github.com/UiPath/skills/pull/300" }); + expect(parseOpened("{oops")).toBeNull(); + expect(parseOpened(JSON.stringify({ opened: "x" }))).toBeNull(); + expect(parseOpened(JSON.stringify({ opened: [] }))).toEqual({}); + }); + test("fixes.json with schema_version 2, or none, is absent", () => { const doc = JSON.parse(FIXES); expect(parseFixes(JSON.stringify({ ...doc, schema_version: 2 }))).toBeNull(); diff --git a/evalboard/lib/__tests__/fixtures/skill-doctor/fixes-1000001/example-grader-fix.md b/evalboard/lib/__tests__/fixtures/skill-doctor/fixes-1000001/example-grader-fix.md deleted file mode 100644 index 80614cfb..00000000 --- a/evalboard/lib/__tests__/fixtures/skill-doctor/fixes-1000001/example-grader-fix.md +++ /dev/null @@ -1,15 +0,0 @@ -## Summary - -The grader expects a literal the task prompt never asks for. This matches the -documented output instead. See https://github.com/UiPath/skills/pull/123 and -[the build](https://dev.azure.com/example-org/example-project/_build/results?buildId=1000001). - -- [an off-list link](https://example.invalid/phish) -- [a script link](javascript:alert(1)) -- ![an image](https://example.invalid/pixel.png) - -raw html - -| Before | After | -| --- | --- | -| fail | pass | diff --git a/evalboard/lib/doctor.ts b/evalboard/lib/doctor.ts index de03199a..714e7d06 100644 --- a/evalboard/lib/doctor.ts +++ b/evalboard/lib/doctor.ts @@ -41,7 +41,6 @@ export interface DoctorStatus { reason: string; startedAt: string | null; buildUrl: string | null; - costUsd: number | null; triageFile: string | null; fixesDir: string | null; trigger: string | null; @@ -67,8 +66,8 @@ export interface DoctorTask { evidence: string | null; passedBefore: string | null; related: string | null; + // Only the source of a third-party "Proposed PR" (an open PR the fix names). existingPrs: DoctorPr[]; - relatedSkillPrs: DoctorPr[]; envGroup: string | null; doctorPr: string | null; } @@ -99,7 +98,6 @@ export interface DoctorFix { taskId: string; covers: string[]; outcome: string; - costUsd: number | null; patchLines: { added: number; removed: number } | null; title: string; gates: DoctorGates; @@ -107,8 +105,6 @@ export interface DoctorFix { export interface DoctorFixes { fixMode: string | null; - costUsd: number | null; - totalCapUsd: number | null; candidates: DoctorFix[]; } @@ -117,9 +113,9 @@ export interface RunDoctor { // Set only when the status is `done` and its triage file parsed. triage: DoctorTriage | null; fixes: DoctorFixes | null; - // Prepared PR bodies (`/.md`) of pr-ready candidates in - // patchIds, by candidate id. Markdown text, rendered without raw HTML. - fixBodies?: Record; + // PRs the doctor opened (`/opened.json`): task id → PR URL, + // allowlisted URLs only. + opened?: Record; } // ---------- coercion helpers ---------- @@ -132,9 +128,6 @@ function isObj(v: unknown): v is Obj { function str(v: unknown): string | null { return typeof v === "string" ? v : null; } -function num(v: unknown): number | null { - return typeof v === "number" && Number.isFinite(v) ? v : null; -} function int(v: unknown, dflt = 0): number { return typeof v === "number" && Number.isInteger(v) && v >= 0 ? v : dflt; } @@ -163,7 +156,6 @@ export function parseDoctorStatus(raw: string | null | undefined): DoctorStatus reason: str(j.reason) ?? "", startedAt: str(j.started_at), buildUrl: str(j.build_url), - costUsd: num(j.cost_usd), // A name outside the pattern is treated as absent: it lands in a path. triageFile: triageFile && TRIAGE_FILE_RE.test(triageFile) ? triageFile : null, fixesDir: fixesDir && FIXES_DIR_RE.test(fixesDir) ? fixesDir : null, @@ -200,7 +192,6 @@ function parseTask(v: unknown): DoctorTask | null { passedBefore: str(v.passed_before), related: str(v.related), existingPrs: prList(v.existing_prs), - relatedSkillPrs: prList(v.related_skill_prs), envGroup: str(v.env_group), doctorPr: str(v.doctor_pr), }; @@ -219,7 +210,9 @@ export function parseTriage(raw: string | null | undefined): DoctorTriage | null tasks: j.tasks.map(parseTask).filter((t): t is DoctorTask => t !== null), actions: Array.isArray(j.actions) ? j.actions.flatMap((a): DoctorAction[] => - isObj(a) && typeof a.text === "string" + // "open-pr" actions list PRs that merely touch the files, not + // fixes: the board leaves them out. + isObj(a) && typeof a.text === "string" && a.kind !== "open-pr" ? [{ count: int(a.count), text: a.text, taskIds: strList(a.task_ids) }] : [], ) @@ -243,8 +236,6 @@ export function parseFixes(raw: string | null | undefined): DoctorFixes | null { if (!isObj(j) || j.schema_version !== 1 || !Array.isArray(j.candidates)) return null; return { fixMode: str(j.fix_mode), - costUsd: num(j.cost_usd), - totalCapUsd: num(j.total_cap_usd), candidates: j.candidates.flatMap((c): DoctorFix[] => { if (!isObj(c)) return []; const id = str(c.id); @@ -257,7 +248,6 @@ export function parseFixes(raw: string | null | undefined): DoctorFixes | null { taskId, covers: strList(c.covers), outcome: str(c.outcome) ?? "unknown", - costUsd: num(c.cost_usd), patchLines: pl ? { added: int(pl.added), removed: int(pl.removed) } : null, @@ -285,17 +275,25 @@ export function patchIds(fixes: DoctorFixes | null): Set { return ids; } -/** The patchIds whose prepared PR body (`.md`) the board loads: pr-ready only. */ -export function bodyIds(fixes: DoctorFixes | null): string[] { - const ids = patchIds(fixes); - return (fixes?.candidates ?? []) - .filter((c) => c.outcome === "pr-ready" && ids.has(c.id)) - .map((c) => c.id); +/** + * `/opened.json` from the doctor's open step: + * `{"opened": [{"task_id", "pr_url", ...} | {"task_id", "problem"}], "error"?}`. + * Untrusted: only entries with a task id and an allowlisted PR URL map; + * problem entries and anything else are skipped. Null when it does not parse. + */ +export function parseOpened(raw: string | null | undefined): Record | null { + const j = parseJson(raw); + if (!isObj(j) || !Array.isArray(j.opened)) return null; + const out: Record = {}; + for (const e of j.opened) { + if (!isObj(e)) continue; + const id = str(e.task_id); + const url = str(e.pr_url); + if (id && isSkillsPrUrl(url) && !(id in out)) out[id] = url; + } + return out; } -// A prepared PR body is a few KB; anything far larger is cut. -export const MAX_BODY_CHARS = 64_000; - /** `/.patch` → the id, or null for any other doctor path. */ export function patchIdFromPath(rel: string): { fixesDir: string; id: string } | null { const parts = rel.split("/"); diff --git a/evalboard/lib/runs.ts b/evalboard/lib/runs.ts index 2db47c2f..45d372f3 100644 --- a/evalboard/lib/runs.ts +++ b/evalboard/lib/runs.ts @@ -24,9 +24,8 @@ import { DOCTOR_DIR, parseDoctorStatus, parseFixes, + parseOpened, parseTriage, - bodyIds, - MAX_BODY_CHARS, patchIdFromPath, patchIds, taskAnalysisEnabled, @@ -2747,27 +2746,19 @@ export async function readRunDoctor( triageRel ? load(triageRel, parseTriage) : null, fixesRel ? load(fixesRel, parseFixes) : null, ]); - // Cached patches and bodies are not fetched again (downloadBlob skips - // them). Bodies are read here and travel in the page data, so - // /api/file never serves a .md. - const fixBodies: Record = {}; + // Cached patches are not fetched again (downloadBlob skips them). + // opened.json is written by the doctor's open step after the fixes, so + // it may appear or change later: it is refetched like status.json. + let opened: Record = {}; if (status.fixesDir) { const fdir = status.fixesDir; - const bodies = bodyIds(fixes); await Promise.all([ ...[...patchIds(fixes)].map((id) => get(`${fdir}/${id}.patch`)), - ...bodies.map((id) => get(`${fdir}/${id}.md`)), + get(`${fdir}/opened.json`, DOCTOR_STATUS_REFRESH_MS), ]); - await Promise.all( - bodies.map(async (id) => { - const raw = await fs - .readFile(path.join(base, fdir, `${id}.md`), "utf-8") - .catch(() => null); - if (raw !== null) fixBodies[id] = raw.slice(0, MAX_BODY_CHARS); - }), - ); + opened = (await load(`${fdir}/opened.json`, parseOpened)) ?? {}; } - return { status, triage, fixes, fixBodies }; + return { status, triage, fixes, opened }; } catch (err) { console.error(`readRunDoctor failed for run ${runId}:`, err); return null; From ac241b1d3c6308fa37ed1e5884d018495e9c7dae Mon Sep 17 00:00:00 2001 From: CarlesUIPath Date: Thu, 19 Nov 2026 17:44:00 +0000 Subject: [PATCH 09/13] fix(evalboard): review round for the redesigned Task failure analysis MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Layer and solvable labels are Maps and opened.json lookups use Object.hasOwn, so report keys such as "constructor" or "__proto__" cannot reach the object prototype; parseOpened builds through a Map. - A task covered by a pr-ready candidate whose PR was opened gets that PR (opened.json is keyed by the candidate's primary task) instead of "Fix prepared". - The doctor's own PR is filtered out of the Proposed PR list and counted once. - PR URLs (existing_prs, doctor_pr, opened.json) are allowlisted in the parsers; the view links them without re-checking. - Cards drop the Related row (no longer parsed) and the "none yet" placeholder. - One layerCounts() helper and one FixRow component; Body split into Stats / LayerBar / FixFirst / SkillFailures; fixTier uses early returns; SkillGroup.ready is now fixPrs; dead "—" sentinels and the ExternalLink fallback are gone. - Tests: prototype keys, the covered-task PR, own PR vs proposed, the opened.json refresh, parser allowlisting. Co-Authored-By: Claude Opus 5.5 --- .../__tests__/task-failure-analysis.test.tsx | 72 ++- .../app/runs/[id]/task-failure-analysis.tsx | 611 +++++++++--------- evalboard/lib/__tests__/doctor-blob.test.ts | 25 +- evalboard/lib/__tests__/doctor.test.ts | 29 +- evalboard/lib/doctor.ts | 17 +- 5 files changed, 438 insertions(+), 316 deletions(-) diff --git a/evalboard/app/runs/[id]/__tests__/task-failure-analysis.test.tsx b/evalboard/app/runs/[id]/__tests__/task-failure-analysis.test.tsx index c866de72..9aa593b7 100644 --- a/evalboard/app/runs/[id]/__tests__/task-failure-analysis.test.tsx +++ b/evalboard/app/runs/[id]/__tests__/task-failure-analysis.test.tsx @@ -215,7 +215,6 @@ describe("TaskFailureAnalysis block", () => { "Failing check", "Evidence", "Passed before", - "Related", ]); expect(within(c).getByText("literal").tagName).toBe("CODE"); // #123 (open on these files) and #125 (related skill PR) are not shown. @@ -357,6 +356,65 @@ describe("TaskFailureAnalysis block", () => { }); }); +describe("untrusted keys", () => { + test.each(["constructor", "__proto__", "toString", "hasOwnProperty"])( + "layer / solvable / opened task id %j do not crash and render as text", + (k) => { + const d = doctor(); + d.triage!.tasks[1] = { ...d.triage!.tasks[1], layer: k, solvable: k }; + d.opened = JSON.parse(JSON.stringify({ [k]: OPENED })); + renderBlock(d); + openBlock(); + fireEvent.click(screen.getByRole("button", { name: "Expand all" })); + const c = card("example-t1"); + expect(within(c).getAllByText(k).length).toBeGreaterThan(0); + expect(screen.getByRole("toolbar", { name: "Filter failures" })).toHaveTextContent(k); + }, + ); +}); + +describe("doctor PR resolution", () => { + test("a task covered by an opened candidate gets that PR, not 'Fix prepared'", () => { + const d = doctor(); + // example-grader-fix's primary task is example-t1; it also covers t2. + d.fixes!.candidates[0].covers = ["example-t1", "example-t2"]; + d.opened = { "example-t1": OPENED }; + renderBlock(d); + openBlock(); + fireEvent.click(screen.getByRole("button", { name: "Expand all" })); + for (const id of ["example-t1", "example-t2"]) { + const c = card(id); + expect(within(c).getByRole("link", { name: "PR with fix: #300 ↗" })).toBeInTheDocument(); + expect(within(c).queryByTestId("fix-prepared")).toBeNull(); + } + // #300 counted once, plus #126 on t2. + expect(digest(d)).toBe("5 errors · 2 PRs with fix"); + }); + + test("the doctor's own PR is not also a Proposed PR, and counts once", () => { + const d = doctor(); + const pr126 = "https://github.com/UiPath/skills/pull/126"; + d.triage!.tasks[2].doctorPr = pr126; + renderBlock(d); + openBlock(); + openSkill("uipath-example-two"); + const c = card("example-t2"); + expect(within(c).getByRole("link", { name: "PR with fix: #126 ↗" })).toBeInTheDocument(); + expect(within(c).queryByTestId("proposed-pr")).toBeNull(); + expect(digest(d)).toBe("5 errors · 1 PR with fix"); + }); + + test("no placeholder row for a fixable task without any PR or candidate", () => { + const d = doctor(); + d.fixes = null; + d.triage!.tasks[2].fix = "no PR named"; + renderBlock(d); + openBlock(); + fireEvent.click(screen.getByRole("button", { name: "Expand all" })); + expect(block()).not.toHaveTextContent("none yet"); + }); +}); + describe("groupBySkill", () => { test("skills: linked PRs first, then prepared fixes, then failed tasks, then name", () => { // -two: 1 linked (#126) of 3; uipath-example: 0 linked, 1 prepared, of 2. @@ -395,7 +453,6 @@ describe("groupBySkill", () => { { ...base, taskId: "y-prepared", layer: "grader" }, { ...base, taskId: "w-opened", layer: "grader" }, { ...base, taskId: "x-doctor-pr", layer: "grader", doctorPr: "https://github.com/UiPath/skills/pull/7" }, - { ...base, taskId: "b-bad-pr", layer: "grader", doctorPr: "https://example.invalid/pull/7" }, { ...base, taskId: "p-proposed", layer: "grader", fix: "see #123" }, { ...d.triage!.tasks[0], skill: base.skill }, ]; @@ -415,11 +472,10 @@ describe("groupBySkill", () => { // The rest: skill ranks after grader on the doctor page, so first. "z-plain-skill", "a-plain-grader", - "b-bad-pr", ]); expect(g.envGroups.map((e) => e.tasks.map((t) => t.taskId))).toEqual([["example-e1"]]); // Linked: #8, #7, #123. The prepared f1 is not. - expect(g.ready).toBe(3); + expect(g.fixPrs).toBe(3); }); test("the rendered accordions follow the same order", () => { @@ -433,18 +489,12 @@ describe("groupBySkill", () => { expect(order).toEqual(["uipath-example", "uipath-example-two"]); }); - test("proposedPrs: named in the fix text, open, allowlisted", () => { + test("proposedPrs: named in the fix text and open (URLs are allowlisted by the parser)", () => { const t = doctor().triage!.tasks[2]; expect(proposedPrs(t).map((p) => p.url)).toEqual(["https://github.com/UiPath/skills/pull/126"]); expect(proposedPrs({ ...t, fix: "see https://github.com/UiPath/skills/pull/126" })).toHaveLength(1); expect(proposedPrs({ ...t, fix: "see #1260 and #12" })).toHaveLength(0); expect(proposedPrs({ ...t, fix: null })).toHaveLength(0); - expect( - proposedPrs({ - ...t, - existingPrs: [{ url: "https://github.com.example.invalid/UiPath/skills/pull/126", title: "", state: "open" }], - }), - ).toHaveLength(0); expect(proposedPrs({ ...t, existingPrs: [{ ...t.existingPrs[0], state: "unknown" }] })).toHaveLength(0); }); diff --git a/evalboard/app/runs/[id]/task-failure-analysis.tsx b/evalboard/app/runs/[id]/task-failure-analysis.tsx index 56f96bf1..0a76e0e0 100644 --- a/evalboard/app/runs/[id]/task-failure-analysis.tsx +++ b/evalboard/app/runs/[id]/task-failure-analysis.tsx @@ -14,6 +14,7 @@ import { type DoctorPr, type DoctorStatus, type DoctorTask, + type DoctorTriage, type RunDoctor, } from "@/lib/doctor"; @@ -22,36 +23,45 @@ import { // skill failures), never by embedding that page. Every string is plain text; // the only links are this board's task pages and URLs of an exact known shape. -// Layer order, labels and colors, as on the doctor's page. -export const LAYERS: Record = { - env: { label: "Environment", color: "#56708f" }, - grader: { label: "Grader", color: "#b07410" }, - task: { label: "Task setup", color: "#7a4fb0" }, - agent: { label: "Agent mistake", color: "#c2413a" }, - skill: { label: "Skill", color: "#d0661c" }, - upstream: { label: "Upstream (CLI)", color: "#2f6fd0" }, - flaky: { label: "Flaky", color: "#7d8794" }, - undiagnosed: { label: "Not diagnosed", color: "#9aa5b3" }, -}; -const ORDER = Object.keys(LAYERS); -const UNKNOWN_LAYER = { label: "Other", color: "#9aa5b3" }; -const layerOf = (k: string) => LAYERS[k] ?? { ...UNKNOWN_LAYER, label: k }; +// Layer order, labels and colors, as on the doctor's page. Maps, not object +// literals: layer and solvable come from the report, and a key such as +// "constructor" or "__proto__" must not reach the object prototype. +const LAYERS = new Map([ + ["env", { label: "Environment", color: "#56708f" }], + ["grader", { label: "Grader", color: "#b07410" }], + ["task", { label: "Task setup", color: "#7a4fb0" }], + ["agent", { label: "Agent mistake", color: "#c2413a" }], + ["skill", { label: "Skill", color: "#d0661c" }], + ["upstream", { label: "Upstream (CLI)", color: "#2f6fd0" }], + ["flaky", { label: "Flaky", color: "#7d8794" }], + ["undiagnosed", { label: "Not diagnosed", color: "#9aa5b3" }], +]); +const ORDER = [...LAYERS.keys()]; +const OTHER_COLOR = "#9aa5b3"; +const layerOf = (k: string) => LAYERS.get(k) ?? { label: k, color: OTHER_COLOR }; const rank = (k: string) => { const i = ORDER.indexOf(k); return i === -1 ? ORDER.length : i; }; -const SOLVABLE: Record = { - "env-fix": "Fix the tenant", - "yes-grader": "Fixable: grader", - "yes-task": "Fixable: task", - "yes-skill": "Fixable: skill", - "upstream-report": "Upstream report", - no: "Not a repo fix", -}; +const SOLVABLE = new Map([ + ["env-fix", "Fix the tenant"], + ["yes-grader", "Fixable: grader"], + ["yes-task", "Fixable: task"], + ["yes-skill", "Fixable: skill"], + ["upstream-report", "Upstream report"], + ["no", "Not a repo fix"], +]); const isFixable = (t: DoctorTask) => /^yes/.test(t.solvable ?? ""); const isReady = (c: DoctorFix) => c.outcome === "pr-ready"; +/** Tasks per layer, in the doctor page's layer order. */ +export function layerCounts(tasks: DoctorTask[]): [string, number][] { + const m = new Map(); + for (const t of tasks) m.set(t.layer, (m.get(t.layer) ?? 0) + 1); + return [...m.entries()].sort((a, b) => rank(a[0]) - rank(b[0])); +} + // ---------- text ---------- /** @@ -91,17 +101,9 @@ export function RichText({ text }: { text: string }) { return <>{out}; } -function ExternalLink({ - href, - ok, - children, -}: { - href: string | null; - ok: (u: unknown) => boolean; - children?: ReactNode; -}) { - if (!href) return null; - if (!ok(href)) return {children ?? href}; +/** A build link only for a dev.azure.com URL; anything else is the label as text. */ +function BuildLink({ href, label }: { href: string; label: string }) { + if (!isAdoUrl(href)) return {label}; return ( - {children ?? href} + {label} ); } @@ -158,7 +160,7 @@ function gateText(f: DoctorFix): string { if (g.replay) parts.push(`replay ${g.replay}`); if (g.mutants) parts.push(`mutants ${g.mutants.killed}/${g.mutants.n}`); if (g.review) parts.push(`review ${g.review}`); - return parts.length ? parts.join(" · ") : "—"; + return parts.join(" · "); } interface FixCtx { @@ -170,7 +172,7 @@ interface FixCtx { } function PatchCell({ fix, ctx }: { fix: DoctorFix; ctx: FixCtx }) { - if (!fix.patchLines) return <>—; + if (!fix.patchLines) return null; const text = `+${fix.patchLines.added}/-${fix.patchLines.removed}`; if (!ctx.dir || !ctx.linked.has(fix.id)) return {text}; return ( @@ -190,8 +192,8 @@ export interface SkillGroup { cards: DoctorTask[]; envGroups: { cause: string; tasks: DoctorTask[] }[]; layerCounts: [string, number][]; - // Distinct linked PRs with a fix among the skill's tasks (countFixPrs). - ready: number; + // Distinct PRs with fix among the skill's tasks (countFixPrs). + fixPrs: number; // Tasks with a fix prepared but no PR yet: sorts right after linked PRs. prepared: number; total: number; @@ -214,50 +216,64 @@ const PR_REF_RE = /(?:#|\/pull\/)(\d+)\b/g; export function proposedPrs(t: DoctorTask): DoctorPr[] { if (!t.fix) return []; const named = new Set([...t.fix.matchAll(PR_REF_RE)].map((m) => m[1])); - return t.existingPrs.filter((p) => { - if (p.state !== "open" || !isSkillsPrUrl(p.url)) return false; - return named.has(/(\d+)$/.exec(p.url)![1]); - }); + // existing_prs URLs are allowlisted by the parser, so each ends in /pull/N. + return t.existingPrs.filter( + (p) => p.state === "open" && named.has(/(\d+)$/.exec(p.url)?.[1] ?? ""), + ); } export interface TaskFixes { - // The doctor's own PR for this task, opened (opened.json) or recorded in - // the triage (doctor_pr); allowlisted. "PR with fix". + // The doctor's own PR: opened for this task or for a pr-ready candidate + // that covers it (opened.json is keyed by the candidate's primary task), + // else the triage's doctor_pr. "PR with fix". ownPr: string | null; // The pr-ready candidates behind it, for their gate verdicts. ready: DoctorFix[]; // pr-ready candidates with no PR opened yet: "Fix prepared, PR not // opened yet". Not a PR with fix. prepared: DoctorFix[]; - // Third-party open PRs the fix text points at ("Proposed PR"). + // Third-party open PRs the fix text names ("Proposed PR"), minus ownPr. proposed: DoctorPr[]; // Other fix candidates (needs-live-run, ...). others: DoctorFix[]; - // One key per distinct linked PR (own or proposed), for counting. + // One key per distinct PR with fix (own or proposed), for counting. keys: string[]; - // Card sort: own PR, proposed PR, prepared fix, other candidates, rest. + // Card sort tier, lower first (see fixTier). tier: number; } +function openedFor(doctor: RunDoctor, taskId: string): string | null { + const o = doctor.opened; + return o && Object.hasOwn(o, taskId) ? o[taskId] : null; +} + +// Own PR, then a proposed PR, then a prepared fix, then other candidates, +// then the rest. +function fixTier(f: Omit): number { + if (f.ownPr) return 0; + if (f.proposed.length) return 1; + if (f.prepared.length) return 2; + if (f.others.length) return 3; + return 4; +} + export function taskFixes(doctor: RunDoctor, t: DoctorTask): TaskFixes { const cands = fixesForTask(doctor, t.taskId); - const opened = doctor.opened?.[t.taskId]; - const ownPr = isSkillsPrUrl(opened) ? opened : isSkillsPrUrl(t.doctorPr) ? t.doctorPr : null; const ready = cands.filter(isReady); - const prepared = ownPr ? [] : ready; - const others = cands.filter((c) => !isReady(c)); - const proposed = proposedPrs(t); - const keys = [...(ownPr ? [ownPr] : []), ...proposed.map((p) => p.url)]; - const tier = ownPr - ? 0 - : proposed.length - ? 1 - : prepared.length - ? 2 - : others.length - ? 3 - : 4; - return { ownPr, ready, prepared, proposed, others, keys, tier }; + // URLs here are allowlisted by the parsers (parseOpened, parseTriage). + const ownPr = + openedFor(doctor, t.taskId) ?? + ready.map((c) => openedFor(doctor, c.taskId)).find((u) => u !== null) ?? + t.doctorPr; + const base = { + ownPr, + ready, + prepared: ownPr ? [] : ready, + proposed: proposedPrs(t).filter((p) => p.url !== ownPr), + others: cands.filter((c) => !isReady(c)), + }; + const keys = [...(ownPr ? [ownPr] : []), ...base.proposed.map((p) => p.url)]; + return { ...base, keys, tier: fixTier(base) }; } /** Tasks with a fix prepared but no PR opened (sort tie-break only). */ @@ -265,7 +281,7 @@ export function countPrepared(doctor: RunDoctor, tasks: DoctorTask[]): number { return tasks.filter((t) => taskFixes(doctor, t).prepared.length > 0).length; } -/** Distinct linked PRs with a fix (own or proposed) over these tasks. */ +/** Distinct PRs with fix (the doctor's own or a proposed one) over these tasks. */ export function countFixPrs(doctor: RunDoctor, tasks: DoctorTask[]): number { return new Set(tasks.flatMap((t) => taskFixes(doctor, t).keys)).size; } @@ -302,8 +318,7 @@ export function groupBySkill( if (t.envGroup) groups.set(t.envGroup, [...(groups.get(t.envGroup) ?? []), t]); else cards.push(t); } - // The doctor's own fix first, then a proposed PR, then other fix - // candidates, then the rest; within each, layer rank and task id. + // By fix tier (fixTier), then layer rank and task id. const tier = (t: DoctorTask) => taskFixes(doctor, t).tier; cards.sort( (a, b) => @@ -311,21 +326,19 @@ export function groupBySkill( rank(b.layer) - rank(a.layer) || a.taskId.localeCompare(b.taskId), ); - const lc = new Map(); - for (const t of items) lc.set(t.layer, (lc.get(t.layer) ?? 0) + 1); return { skill, cards, envGroups: [...groups.entries()].map(([cause, tasks]) => ({ cause, tasks })), - layerCounts: [...lc.entries()].sort((a, b) => rank(a[0]) - rank(b[0])), - ready: countFixPrs(doctor, items), + layerCounts: layerCounts(items), + fixPrs: countFixPrs(doctor, items), prepared: countPrepared(doctor, items), total: items.length, }; }) .sort( (a, b) => - b.ready - a.ready || + b.fixPrs - a.fixPrs || b.prepared - a.prepared || b.total - a.total || a.skill.localeCompare(b.skill), @@ -369,7 +382,6 @@ function TaskCard({ t, doctor, ctx }: { t: DoctorTask; doctor: RunDoctor; ctx: F if (t.criterion) rows.push(["Failing check", ]); if (t.evidence) rows.push(["Evidence", , true]); if (t.passedBefore) rows.push(["Passed before", ]); - if (t.related) rows.push(["Related", ]); return (
{layer.label} {t.solvable && ( - {SOLVABLE[t.solvable.split(" ")[0]] ?? t.solvable} + {SOLVABLE.get(t.solvable.split(" ")[0]) ?? t.solvable} )} {t.runStatus && ( {t.runStatus.replace(/_/g, " ").toLowerCase()} @@ -392,7 +404,7 @@ function TaskCard({ t, doctor, ctx }: { t: DoctorTask; doctor: RunDoctor; ctx: F

)} - + {rows.length > 0 && (
{rows.map(([k, v, mono]) => ( @@ -411,27 +423,55 @@ function TaskCard({ t, doctor, ctx }: { t: DoctorTask; doctor: RunDoctor; ctx: F ); } -const BOX = "rounded-md border px-3 py-1.5 text-xs"; - function Gates({ cands }: { cands: DoctorFix[] }) { - const g = cands.map(gateText).filter((x) => x !== "—"); + const g = cands.map(gateText).filter(Boolean); if (!g.length) return null; return {g.join("; ")}; } -// page.html's "Proposed PR" row, split by who made the fix: "PR with fix" -// (the doctor's opened PR or prepared fix), "Proposed PR" (a third party's open -// PR the fix text names), "Fix candidate: " (not ready), or "none yet" -// for a repo-fixable failure. -function FixRows({ t, fx, ctx }: { t: DoctorTask; fx: TaskFixes; ctx: FixCtx }) { +function Patch({ c, ctx }: { c: DoctorFix; ctx: FixCtx }) { + if (!c.patchLines) return null; + return ( + + patch + + ); +} + +const ROW_BORDER = { + own: "border-studio-blue", + proposed: "border-studio-blue/60", + prepared: "border-dashed border-studio-blue/60", + candidate: "border-dashed border-gray-300", +} as const; + +function FixRow({ + kind, + testId, + children, +}: { + kind: keyof typeof ROW_BORDER; + testId: string; + children: ReactNode; +}) { + return ( +
+ {children} +
+ ); +} + +// The card's PR rows: "PR with fix" (the doctor's PR), "Proposed PR" (someone +// else's open PR that the fix names), "Fix prepared, PR not opened yet", and +// "Fix candidate: " for candidates that are not ready. +function FixRows({ fx, ctx }: { fx: TaskFixes; ctx: FixCtx }) { const rows: ReactNode[] = []; if (fx.ownPr) { rows.push( -
+ -
, + , ); } for (const p of fx.proposed) { rows.push( - , + , ); } for (const c of fx.prepared) { rows.push( -
+ Fix prepared, PR not opened yet - {c.patchLines && ( - - patch - - )} + -
, + , ); } for (const c of fx.others) { rows.push( -
+ Fix candidate: {c.outcome} - {c.patchLines && ( - - patch - - )} + -
, - ); - } - if (rows.length === 0 && isFixable(t)) { - rows.push( - - Proposed PR: none yet - , + , ); } if (rows.length === 0) return null; @@ -515,7 +528,7 @@ function EnvGroup({ cause, tasks, ctx }: { cause: string; tasks: DoctorTask[]; c return (
+
+
+ + Show + + {present.map((k) => { + const pressed = !active || active.has(k); + return ( + + ); + })} + setQuery(e.target.value)} + placeholder="Search task, cause, PR…" + aria-label="Search failures" + className="min-w-0 max-w-xs flex-1 rounded-md border border-gray-200 px-2.5 py-1 text-sm" + /> +
+
+ {groups.length === 0 ? ( +

No failures match this filter.

+ ) : ( + groups.map((g) => ( + + setOpenSkills((cur) => { + const next = new Set(cur); + if (next.has(g.skill)) next.delete(g.skill); + else next.add(g.skill); + return next; + }) + } + doctor={doctor} + ctx={ctx} + /> + )) + )} +
+ + ); +} - {n > 0 && ( -
-
- Failures by layer -
-
- {counts.map(([k, c]) => ( - - ))} -
-
- {counts.map(([k, c]) => ( - - - {layerOf(k).label} {c} - - ))} -
-
- )} - - {triage.actions.length > 0 && ( -
-

Fix first

-
    - {triage.actions.map((a, i) => { - const ready = countFixPrs( - doctor, - triage.tasks.filter((t) => a.taskIds.includes(t.taskId)), - ); - return ( -
  1. - - {a.count} - - {" · "} - - {ready > 0 && ( - - {prsWithFix(ready)} - - )} -
  2. - ); - })} -
-
- )} - -
-

Skill failures

-
- - -
-
- - Show - - {present.map((k) => { - const pressed = !active || active.has(k); - return ( - - ); - })} - setQuery(e.target.value)} - placeholder="Search task, cause, PR…" - aria-label="Search failures" - className="min-w-0 max-w-xs flex-1 rounded-md border border-gray-200 px-2.5 py-1 text-sm" - /> -
-
- {groups.length === 0 ? ( -

No failures match this filter.

- ) : ( - groups.map((g) => ( - - setOpenSkills((cur) => { - const next = new Set(cur); - if (next.has(g.skill)) next.delete(g.skill); - else next.add(g.skill); - return next; - }) - } - doctor={doctor} - ctx={ctx} - /> - )) - )} -
-
+function Body({ doctor, ctx }: { doctor: RunDoctor; ctx: FixCtx }) { + const triage = doctor.triage!; + const counts = useMemo(() => layerCounts(triage.tasks), [triage.tasks]); + return ( +
+ + + + k)} ctx={ctx} />
); } @@ -803,17 +834,13 @@ function BuildLinks({ status }: { status: DoctorStatus }) { {status.buildUrl && ( <> {" · "} - - build - + )} {status.trigger && ( <> {" · "} - - nightly - + )} diff --git a/evalboard/lib/__tests__/doctor-blob.test.ts b/evalboard/lib/__tests__/doctor-blob.test.ts index 4d05bb00..7c9f0330 100644 --- a/evalboard/lib/__tests__/doctor-blob.test.ts +++ b/evalboard/lib/__tests__/doctor-blob.test.ts @@ -151,16 +151,33 @@ describe("readRunDoctor: what it fetches", () => { warn.mockRestore(); }); - test("status.json is refetched after a minute, not within it; the rest never", async () => { + test("status.json and opened.json are refetched after a minute, not within it; the rest never", async () => { await read(); state.downloads = []; await read(); expect(state.downloads).toEqual([]); - const st = path.join(root, RUN, "skill-doctor", "status.json"); const old = new Date(Date.now() - 120_000); - await fs.utimes(st, old, old); + for (const f of ["status.json", "fixes-1000001/opened.json"]) { + await fs.utimes(path.join(root, RUN, "skill-doctor", f), old, old); + } await read(); - expect(state.downloads).toEqual([`${D}/status.json`]); + expect(state.downloads.sort()).toEqual( + [`${D}/status.json`, `${D}/fixes-1000001/opened.json`].sort(), + ); + }); + + test("an opened.json that changes is picked up after the refresh age", async () => { + expect((await read())!.opened).toEqual({}); + state.blobs[`${D}/fixes-1000001/opened.json`] = JSON.stringify({ + opened: [{ task_id: "example-t1", pr_url: "https://github.com/UiPath/skills/pull/300" }], + }); + // Within the minute: the cached copy. + expect((await read())!.opened).toEqual({}); + const old = new Date(Date.now() - 120_000); + await fs.utimes(path.join(root, RUN, "skill-doctor", "fixes-1000001", "opened.json"), old, old); + expect((await read())!.opened).toEqual({ + "example-t1": "https://github.com/UiPath/skills/pull/300", + }); }); test("patch fan-out is capped and ids are checked", async () => { diff --git a/evalboard/lib/__tests__/doctor.test.ts b/evalboard/lib/__tests__/doctor.test.ts index 661340a6..4ead8d2a 100644 --- a/evalboard/lib/__tests__/doctor.test.ts +++ b/evalboard/lib/__tests__/doctor.test.ts @@ -157,11 +157,13 @@ describe("parseTriage / parseFixes", () => { confidence: "high", criterion: "file_contains", passedBefore: "2099-01-01_04-20-00 (claude-code)", - related: "example-t3 fails the same check", envGroup: null, doctorPr: null, }); - expect(t1.existingPrs).toHaveLength(3); + // Only allowlisted PR URLs survive the parser (javascript: and a + // look-alike host are dropped). + expect(t1.existingPrs.map((p) => p.url)).toEqual(["https://github.com/UiPath/skills/pull/123"]); + expect(t1).not.toHaveProperty("related"); expect(t1).not.toHaveProperty("relatedSkillPrs"); // open-pr actions (PRs that merely touch the files) are left out. expect(t.actions.map((a) => a.text)).not.toContainEqual(expect.stringMatching(/^Open PR/)); @@ -175,6 +177,29 @@ describe("parseTriage / parseFixes", () => { expect(parseTriage(JSON.stringify({ schema_version: 1 }))).toBeNull(); }); + test("doctor_pr is kept only when allowlisted", () => { + const doc = JSON.parse(TRIAGE); + doc.tasks[1].doctor_pr = "https://github.com/UiPath/skills/pull/9"; + doc.tasks[2].doctor_pr = "https://github.com.example.invalid/UiPath/skills/pull/9"; + const t = parseTriage(JSON.stringify(doc))!; + expect(t.tasks[1].doctorPr).toBe("https://github.com/UiPath/skills/pull/9"); + expect(t.tasks[2].doctorPr).toBeNull(); + }); + + test("parseOpened: prototype-named task ids are own keys", () => { + const o = parseOpened( + JSON.stringify({ + opened: [ + { task_id: "__proto__", pr_url: "https://github.com/UiPath/skills/pull/1" }, + { task_id: "constructor", pr_url: "https://github.com/UiPath/skills/pull/2" }, + ], + }), + )!; + expect(Object.hasOwn(o, "__proto__")).toBe(true); + expect(o.constructor).toBe("https://github.com/UiPath/skills/pull/2"); + expect(Object.getPrototypeOf(o)).toBe(Object.prototype); + }); + test("parseOpened: allowlisted PR URLs map by task; the rest is skipped", () => { expect( parseOpened( diff --git a/evalboard/lib/doctor.ts b/evalboard/lib/doctor.ts index 714e7d06..099c57b8 100644 --- a/evalboard/lib/doctor.ts +++ b/evalboard/lib/doctor.ts @@ -46,6 +46,8 @@ export interface DoctorStatus { trigger: string | null; } +// Every PR URL the parsers hand out has passed isSkillsPrUrl: views can link +// them without checking again. export interface DoctorPr { url: string; title: string; @@ -65,10 +67,10 @@ export interface DoctorTask { criterion: string | null; evidence: string | null; passedBefore: string | null; - related: string | null; // Only the source of a third-party "Proposed PR" (an open PR the fix names). existingPrs: DoctorPr[]; envGroup: string | null; + // The doctor's PR for this task, allowlisted (else null). doctorPr: string | null; } @@ -166,7 +168,7 @@ export function parseDoctorStatus(raw: string | null | undefined): DoctorStatus function parsePr(v: unknown): DoctorPr | null { if (!isObj(v)) return null; const url = str(v.url); - if (!url) return null; + if (!isSkillsPrUrl(url)) return null; return { url, title: str(v.title) ?? "", state: str(v.state) ?? "unknown" }; } @@ -190,10 +192,9 @@ function parseTask(v: unknown): DoctorTask | null { criterion: str(v.criterion), evidence: str(v.evidence), passedBefore: str(v.passed_before), - related: str(v.related), existingPrs: prList(v.existing_prs), envGroup: str(v.env_group), - doctorPr: str(v.doctor_pr), + doctorPr: isSkillsPrUrl(v.doctor_pr) ? v.doctor_pr : null, }; } @@ -284,14 +285,16 @@ export function patchIds(fixes: DoctorFixes | null): Set { export function parseOpened(raw: string | null | undefined): Record | null { const j = parseJson(raw); if (!isObj(j) || !Array.isArray(j.opened)) return null; - const out: Record = {}; + // Built through a Map so a task id such as "__proto__" or "constructor" + // is an own key, never the prototype; the first entry per task wins. + const out = new Map(); for (const e of j.opened) { if (!isObj(e)) continue; const id = str(e.task_id); const url = str(e.pr_url); - if (id && isSkillsPrUrl(url) && !(id in out)) out[id] = url; + if (id && isSkillsPrUrl(url) && !out.has(id)) out.set(id, url); } - return out; + return Object.fromEntries(out); } /** `/.patch` → the id, or null for any other doctor path. */ From 2217049fc39b0f1e77f21a43617fe5db077f7217 Mon Sep 17 00:00:00 2001 From: CarlesUIPath Date: Thu, 19 Nov 2026 18:27:33 +0000 Subject: [PATCH 10/13] feat(evalboard): third-party PR rows from the doctor's fixed_by_pr and pr_checks The "fix text names #N" rule is gone. A third-party PR now shows only when the triage's fixed_by_pr says "fixes" and the PR is open in the row's existing PRs, joined with fixes.json pr_checks for that row and number: - verified: "Proposed PR" with the replay evidence; counts as a PR with fix and sorts after the doctor's own PR; - untested or no check: "Open PR may fix this (not verified)" with the diagnosis's basis, the evidence, and "needs a Windows run" when the check says so; not counted, sorted after prepared fixes; - not_verified, may_help, unrelated: nothing. Both fields are optional and untrusted: malformed entries parse to nothing, pr_checks URLs are allowlisted and must match the number, and task ids are own keys. Co-Authored-By: Claude Opus 5.5 --- .../__tests__/task-failure-analysis.test.tsx | 114 ++++++++++++---- .../app/runs/[id]/task-failure-analysis.tsx | 123 +++++++++++++----- evalboard/lib/__tests__/doctor.test.ts | 52 ++++++++ .../skill-doctor/fixes-1000001/fixes.json | 86 ++++++++++-- .../fixtures/skill-doctor/triage-1000001.json | 21 ++- evalboard/lib/doctor.ts | 59 ++++++++- 6 files changed, 379 insertions(+), 76 deletions(-) diff --git a/evalboard/app/runs/[id]/__tests__/task-failure-analysis.test.tsx b/evalboard/app/runs/[id]/__tests__/task-failure-analysis.test.tsx index 9aa593b7..6a07aaaa 100644 --- a/evalboard/app/runs/[id]/__tests__/task-failure-analysis.test.tsx +++ b/evalboard/app/runs/[id]/__tests__/task-failure-analysis.test.tsx @@ -6,7 +6,7 @@ import { countFixPrs, digest, groupBySkill, - proposedPrs, + existingPrFix, TaskFailureAnalysis, } from "../task-failure-analysis"; import { @@ -26,8 +26,8 @@ function status(patch: Record = {}) { } // The fixture: example-t1 has a pr-ready fix with no PR opened yet; example-t2 -// has a needs-live-run candidate and its fix text names open PR #126 (and -// merged #127). +// has a needs-live-run candidate, and the diagnosis says open PR #126 fixes it +// (fixed_by_pr "fixes"), which the Fix stage verified (pr_checks). function doctor(patch: Record = {}): RunDoctor { const s = status(patch); return { @@ -63,7 +63,7 @@ describe("digest", () => { }); test("no linked PR: the errors only", () => { const d = doctor(); - d.triage!.tasks[2].fix = "Document the current flag."; + d.triage!.tasks[2].fixedByPr = null; expect(digest(d)).toBe("5 errors"); }); test("not done: the state", () => { @@ -268,15 +268,16 @@ describe("TaskFailureAnalysis block", () => { ).toBeInTheDocument(); }); - test("a third-party open PR named in the fix text: a Proposed PR row", () => { + test("a verified third-party PR: a Proposed PR row with the evidence", () => { renderBlock(); openBlock(); openSkill("uipath-example-two"); const c = card("example-t2"); const rows = within(c).getAllByTestId("proposed-pr"); - // #127 is named too, but merged: it does not count. expect(rows).toHaveLength(1); expect(rows[0]).toHaveTextContent("Proposed PR#126 docs(example-two): document the current flagopen"); + expect(rows[0]).toHaveTextContent("run_command flips to pass on both"); + expect(within(c).queryByTestId("maybe-pr")).toBeNull(); expect(within(rows[0]).getByRole("link")).toHaveAttribute( "href", "https://github.com/UiPath/skills/pull/126", @@ -284,6 +285,76 @@ describe("TaskFailureAnalysis block", () => { expect(within(c).getByTestId("fix-candidate")).toHaveTextContent("Fix candidate: needs-live-run"); }); + test.each([ + ["untested", undefined], + ["untested", "needs-windows-run"], + ["none", undefined], + ])("check %s (%s): 'Open PR may fix this (not verified)', not counted", (verdict, reason) => { + const d = doctor(); + if (verdict === "none") d.fixes!.prChecks = {}; + else + d.fixes!.prChecks["example-t2"] = { + ...d.fixes!.prChecks["example-t2"], + verdict: "untested", + evidence: "no stored run could replay it", + reason: reason ? "needs-windows-run" : null, + }; + expect(digest(d)).toBe("5 errors"); + renderBlock(d); + openBlock(); + openSkill("uipath-example-two"); + const c = card("example-t2"); + expect(within(c).queryByTestId("proposed-pr")).toBeNull(); + const row = within(c).getByTestId("maybe-pr"); + expect(row).toHaveTextContent("Open PR may fix this (not verified)"); + expect(within(row).getByRole("link", { name: /#126/ })).toHaveAttribute( + "href", + "https://github.com/UiPath/skills/pull/126", + ); + expect(row).toHaveTextContent("documents the current flag"); + if (verdict === "none") expect(row).not.toHaveTextContent("replay"); + else expect(row).toHaveTextContent("no stored run could replay it"); + if (reason) expect(row).toHaveTextContent("needs a Windows run"); + else expect(row).not.toHaveTextContent("Windows"); + }); + + test.each([ + ["not_verified check", (d: RunDoctor) => { d.fixes!.prChecks["example-t2"].verdict = "not_verified"; }], + ["may_help", (d: RunDoctor) => { d.triage!.tasks[2].fixedByPr!.verdict = "may_help"; }], + ["unrelated", (d: RunDoctor) => { d.triage!.tasks[2].fixedByPr!.verdict = "unrelated"; }], + ["merged PR", (d: RunDoctor) => { d.triage!.tasks[2].fixedByPr!.number = 127; }], + ["PR not in existing_prs", (d: RunDoctor) => { d.triage!.tasks[2].fixedByPr!.number = 999; }], + ["no fixed_by_pr", (d: RunDoctor) => { d.triage!.tasks[2].fixedByPr = null; }], + ])("%s: no PR row", (_name, mutate) => { + const d = doctor(); + mutate(d); + expect(digest(d)).toBe("5 errors"); + renderBlock(d); + openBlock(); + openSkill("uipath-example-two"); + const c = card("example-t2"); + expect(within(c).queryByTestId("proposed-pr")).toBeNull(); + expect(within(c).queryByTestId("maybe-pr")).toBeNull(); + }); + + test("a PR only cited in the fix text shows nothing (no prose parsing)", () => { + const d = doctor(); + // #123 is open on t1's files and named in its fix text, but there is no fixed_by_pr. + d.triage!.tasks[1].fix = "Open PR #123 (https://github.com/UiPath/skills/pull/123) already fixes this."; + renderBlock(d); + openBlock(); + openSkill("uipath-example"); + const c = card("example-t1"); + expect(within(c).queryByTestId("proposed-pr")).toBeNull(); + expect(within(c).queryByTestId("maybe-pr")).toBeNull(); + }); + + test("a check for another PR number does not verify this one", () => { + const d = doctor(); + d.fixes!.prChecks["example-t2"].number = 4; + expect(existingPrFix(d, d.triage!.tasks[2])).toMatchObject({ verified: false, evidence: null }); + }); + test("an unsafe patch id shows its size without a link", () => { const d = doctor(); d.fixes!.candidates[0].id = "../bad"; @@ -407,7 +478,7 @@ describe("doctor PR resolution", () => { test("no placeholder row for a fixable task without any PR or candidate", () => { const d = doctor(); d.fixes = null; - d.triage!.tasks[2].fix = "no PR named"; + d.triage!.tasks[2].fixedByPr = null; renderBlock(d); openBlock(); fireEvent.click(screen.getByRole("button", { name: "Expand all" })); @@ -424,7 +495,7 @@ describe("groupBySkill", () => { ]); // Without #126, the prepared fix beats more failed tasks. const d = doctor(); - d.triage!.tasks[2].fix = "no PR named"; + d.triage!.tasks[2].fixedByPr = null; expect(groupBySkill(d, null, "").map((g) => g.skill)).toEqual([ "uipath-example", "uipath-example-two", @@ -443,7 +514,7 @@ describe("groupBySkill", () => { expect(groupBySkill(tie, null, "").map((g) => g.skill)).toEqual(["a-skill", "b-skill"]); }); - test("cards: own PR, proposed PR, prepared fix, other candidates, the rest", () => { + test("cards: own PR, verified PR, prepared fix, unverified PR, other candidates, the rest", () => { const d = doctor(); const base = { ...d.triage!.tasks[1], doctorPr: null, fix: "plain" }; d.triage!.tasks = [ @@ -453,7 +524,8 @@ describe("groupBySkill", () => { { ...base, taskId: "y-prepared", layer: "grader" }, { ...base, taskId: "w-opened", layer: "grader" }, { ...base, taskId: "x-doctor-pr", layer: "grader", doctorPr: "https://github.com/UiPath/skills/pull/7" }, - { ...base, taskId: "p-proposed", layer: "grader", fix: "see #123" }, + { ...base, taskId: "p-proposed", layer: "grader", fixedByPr: { number: 123, verdict: "fixes", basis: "" } }, + { ...base, taskId: "q-maybe", layer: "grader", fixedByPr: { number: 123, verdict: "fixes", basis: "" } }, { ...d.triage!.tasks[0], skill: base.skill }, ]; d.fixes!.candidates = [ @@ -462,25 +534,29 @@ describe("groupBySkill", () => { { ...d.fixes!.candidates[1], id: "f2", taskId: "m-live", covers: ["m-live"] }, ]; d.opened = { "w-opened": "https://github.com/UiPath/skills/pull/8" }; + d.fixes!.prChecks = { + "p-proposed": { number: 123, url: "https://github.com/UiPath/skills/pull/123", verdict: "verified", evidence: "", reason: null }, + }; const [g] = groupBySkill(d, null, ""); expect(g.cards.map((t) => t.taskId)).toEqual([ "w-opened", "x-doctor-pr", "p-proposed", "y-prepared", + "q-maybe", "m-live", // The rest: skill ranks after grader on the doctor page, so first. "z-plain-skill", "a-plain-grader", ]); expect(g.envGroups.map((e) => e.tasks.map((t) => t.taskId))).toEqual([["example-e1"]]); - // Linked: #8, #7, #123. The prepared f1 is not. + // Linked: #8, #7, #123 (verified). The prepared f1 and the unverified q-maybe are not. expect(g.fixPrs).toBe(3); }); test("the rendered accordions follow the same order", () => { const d = doctor(); - d.triage!.tasks[2].fix = "no PR named"; + d.triage!.tasks[2].fixedByPr = null; renderBlock(d); openBlock(); const order = Array.from(document.querySelectorAll("[data-skill]")).map((e) => @@ -489,20 +565,12 @@ describe("groupBySkill", () => { expect(order).toEqual(["uipath-example", "uipath-example-two"]); }); - test("proposedPrs: named in the fix text and open (URLs are allowlisted by the parser)", () => { - const t = doctor().triage!.tasks[2]; - expect(proposedPrs(t).map((p) => p.url)).toEqual(["https://github.com/UiPath/skills/pull/126"]); - expect(proposedPrs({ ...t, fix: "see https://github.com/UiPath/skills/pull/126" })).toHaveLength(1); - expect(proposedPrs({ ...t, fix: "see #1260 and #12" })).toHaveLength(0); - expect(proposedPrs({ ...t, fix: null })).toHaveLength(0); - expect(proposedPrs({ ...t, existingPrs: [{ ...t.existingPrs[0], state: "unknown" }] })).toHaveLength(0); - }); - test("countFixPrs counts each linked PR once, and never a prepared fix", () => { const d = doctor(); - d.triage!.tasks[1].fix = "see #126"; + d.triage!.tasks[1].fixedByPr = d.triage!.tasks[2].fixedByPr; d.triage!.tasks[1].existingPrs = d.triage!.tasks[2].existingPrs; - // t1 and t2 both name #126: one PR. + d.fixes!.prChecks["example-t1"] = d.fixes!.prChecks["example-t2"]; + // t1 and t2 both verified against #126: one PR. expect(countFixPrs(d, d.triage!.tasks)).toBe(1); }); }); diff --git a/evalboard/app/runs/[id]/task-failure-analysis.tsx b/evalboard/app/runs/[id]/task-failure-analysis.tsx index 0a76e0e0..167c956f 100644 --- a/evalboard/app/runs/[id]/task-failure-analysis.tsx +++ b/evalboard/app/runs/[id]/task-failure-analysis.tsx @@ -205,21 +205,42 @@ export function fixesForTask(doctor: RunDoctor, taskId: string): DoctorFix[] { ); } -// "#N" or ".../pull/N" in a task's fix text. -const PR_REF_RE = /(?:#|\/pull\/)(\d+)\b/g; +export interface ExistingPrFix { + pr: DoctorPr; + // The Fix stage replayed the PR and it flipped the row ("Proposed PR"). + verified: boolean; + // The diagnosis's reason for naming this PR. + basis: string; + // The replay's evidence, when there was a check. + evidence: string | null; + // The check could not run: it needs a Windows run. + needsWindows: boolean; +} /** - * A third-party proposed PR: the task's fix text names it AND it is one of the - * task's existing PRs, open, with an allowlisted URL. Merged or unknown PRs - * do not count. + * Someone else's open PR that fixes this row, from the doctor's structured + * fields only (no reading of prose): the diagnosis says `fixed_by_pr` with + * verdict "fixes", the PR is open in the row's existing PRs, and the Fix + * stage's pr_checks entry for the row (same number) is verified (counts as a + * PR with fix) or untested / absent (shown as "may fix this", not counted). + * not_verified, may_help and unrelated show nothing. */ -export function proposedPrs(t: DoctorTask): DoctorPr[] { - if (!t.fix) return []; - const named = new Set([...t.fix.matchAll(PR_REF_RE)].map((m) => m[1])); - // existing_prs URLs are allowlisted by the parser, so each ends in /pull/N. - return t.existingPrs.filter( - (p) => p.state === "open" && named.has(/(\d+)$/.exec(p.url)?.[1] ?? ""), - ); +export function existingPrFix(doctor: RunDoctor, t: DoctorTask): ExistingPrFix | null { + const f = t.fixedByPr; + if (!f || f.verdict !== "fixes") return null; + const pr = t.existingPrs.find((p) => p.state === "open" && p.url.endsWith(`/pull/${f.number}`)); + if (!pr) return null; + const checks = doctor.fixes?.prChecks; + const c = checks && Object.hasOwn(checks, t.taskId) ? checks[t.taskId] : null; + const check = c && c.number === f.number ? c : null; + if (check?.verdict === "not_verified") return null; + return { + pr, + verified: check?.verdict === "verified", + basis: f.basis, + evidence: check?.evidence || null, + needsWindows: check?.reason === "needs-windows-run", + }; } export interface TaskFixes { @@ -232,8 +253,11 @@ export interface TaskFixes { // pr-ready candidates with no PR opened yet: "Fix prepared, PR not // opened yet". Not a PR with fix. prepared: DoctorFix[]; - // Third-party open PRs the fix text names ("Proposed PR"), minus ownPr. - proposed: DoctorPr[]; + // Someone else's open PR the doctor verified fixes this row ("Proposed + // PR"; a PR with fix), or one it could not verify ("Open PR may fix + // this", not counted). Never the doctor's own PR. + proposed: ExistingPrFix | null; + maybe: ExistingPrFix | null; // Other fix candidates (needs-live-run, ...). others: DoctorFix[]; // One key per distinct PR with fix (own or proposed), for counting. @@ -247,14 +271,15 @@ function openedFor(doctor: RunDoctor, taskId: string): string | null { return o && Object.hasOwn(o, taskId) ? o[taskId] : null; } -// Own PR, then a proposed PR, then a prepared fix, then other candidates, -// then the rest. +// Own PR, then a verified proposed PR, then a prepared fix, then an +// unverified open PR, then other candidates, then the rest. function fixTier(f: Omit): number { if (f.ownPr) return 0; - if (f.proposed.length) return 1; + if (f.proposed) return 1; if (f.prepared.length) return 2; - if (f.others.length) return 3; - return 4; + if (f.maybe) return 3; + if (f.others.length) return 4; + return 5; } export function taskFixes(doctor: RunDoctor, t: DoctorTask): TaskFixes { @@ -265,14 +290,17 @@ export function taskFixes(doctor: RunDoctor, t: DoctorTask): TaskFixes { openedFor(doctor, t.taskId) ?? ready.map((c) => openedFor(doctor, c.taskId)).find((u) => u !== null) ?? t.doctorPr; + const ext = existingPrFix(doctor, t); + const other = ext && ext.pr.url !== ownPr ? ext : null; const base = { ownPr, ready, prepared: ownPr ? [] : ready, - proposed: proposedPrs(t).filter((p) => p.url !== ownPr), + proposed: other?.verified ? other : null, + maybe: other && !other.verified ? other : null, others: cands.filter((c) => !isReady(c)), }; - const keys = [...(ownPr ? [ownPr] : []), ...base.proposed.map((p) => p.url)]; + const keys = [...(ownPr ? [ownPr] : []), ...(base.proposed ? [base.proposed.pr.url] : [])]; return { ...base, keys, tier: fixTier(base) }; } @@ -281,7 +309,7 @@ export function countPrepared(doctor: RunDoctor, tasks: DoctorTask[]): number { return tasks.filter((t) => taskFixes(doctor, t).prepared.length > 0).length; } -/** Distinct PRs with fix (the doctor's own or a proposed one) over these tasks. */ +/** Distinct PRs with fix (the doctor's own or a verified proposed one) over these tasks. */ export function countFixPrs(doctor: RunDoctor, tasks: DoctorTask[]): number { return new Set(tasks.flatMap((t) => taskFixes(doctor, t).keys)).size; } @@ -464,9 +492,27 @@ function FixRow({ ); } +function PrLink({ pr }: { pr: DoctorPr }) { + return ( +
+ {prLabel(pr.url)} {pr.title} + + ); +} + +function Small({ children }: { children: ReactNode }) { + return {children}; +} + // The card's PR rows: "PR with fix" (the doctor's PR), "Proposed PR" (someone -// else's open PR that the fix names), "Fix prepared, PR not opened yet", and -// "Fix candidate: " for candidates that are not ready. +// else's open PR, verified by the doctor's replay), "Fix prepared, PR not +// opened yet", "Open PR may fix this (not verified)", and "Fix candidate: +// " for candidates that are not ready. function FixRows({ fx, ctx }: { fx: TaskFixes; ctx: FixCtx }) { const rows: ReactNode[] = []; if (fx.ownPr) { @@ -484,19 +530,14 @@ function FixRows({ fx, ctx }: { fx: TaskFixes; ctx: FixCtx }) { , ); } - for (const p of fx.proposed) { + if (fx.proposed) { + const { pr, evidence } = fx.proposed; rows.push( - + Proposed PR - - {prLabel(p.url)} {p.title} - - {p.state} + + {pr.state} + {evidence && {evidence}} , ); } @@ -509,6 +550,18 @@ function FixRows({ fx, ctx }: { fx: TaskFixes; ctx: FixCtx }) { , ); } + if (fx.maybe) { + const { pr, basis, evidence, needsWindows } = fx.maybe; + rows.push( + + Open PR may fix this (not verified) + + {needsWindows && needs a Windows run} + {basis && {basis}} + {evidence && {evidence}} + , + ); + } for (const c of fx.others) { rows.push( diff --git a/evalboard/lib/__tests__/doctor.test.ts b/evalboard/lib/__tests__/doctor.test.ts index 4ead8d2a..c9c18ff3 100644 --- a/evalboard/lib/__tests__/doctor.test.ts +++ b/evalboard/lib/__tests__/doctor.test.ts @@ -177,6 +177,58 @@ describe("parseTriage / parseFixes", () => { expect(parseTriage(JSON.stringify({ schema_version: 1 }))).toBeNull(); }); + test("fixed_by_pr: parsed when well-formed, else null", () => { + const doc = JSON.parse(TRIAGE); + expect(parseTriage(TRIAGE)!.tasks[2].fixedByPr).toEqual({ + number: 126, + verdict: "fixes", + basis: expect.stringContaining("documents the current flag"), + }); + for (const bad of [ + { number: "126", verdict: "fixes", basis: "" }, + { number: 0, verdict: "fixes", basis: "" }, + { number: 1.5, verdict: "fixes", basis: "" }, + { number: 126, verdict: "yes", basis: "" }, + { number: 126 }, + [126, "fixes"], + "126", + ]) { + doc.tasks[2].fixed_by_pr = bad; + expect(parseTriage(JSON.stringify(doc))!.tasks[2].fixedByPr).toBeNull(); + } + doc.tasks[2].fixed_by_pr = { number: 5, verdict: "may_help", basis: 7 }; + expect(parseTriage(JSON.stringify(doc))!.tasks[2].fixedByPr).toEqual({ + number: 5, + verdict: "may_help", + basis: "", + }); + }); + + test("pr_checks: well-formed entries by task; malformed ones skipped", () => { + const doc = JSON.parse(FIXES); + const ok = { number: 5, url: "https://github.com/UiPath/skills/pull/5", verdict: "untested", evidence: "e" }; + doc.pr_checks = { + good: { ...ok, reason: "needs-windows-run" }, + odd_reason: { ...ok, reason: "something-else" }, + bad_host: { ...ok, url: "https://github.com.example.invalid/UiPath/skills/pull/5" }, + wrong_number: { ...ok, url: "https://github.com/UiPath/skills/pull/6" }, + bad_verdict: { ...ok, verdict: "fixed" }, + no_number: { ...ok, number: null }, + not_obj: "x", + }; + // A "__proto__" task id, as JSON text (an object literal would set the prototype). + const raw = JSON.stringify(doc).replace('"pr_checks":{', `"pr_checks":{"__proto__":${JSON.stringify(ok)},`); + const checks = parseFixes(raw)!.prChecks; + expect(Object.keys(checks).sort()).toEqual(["__proto__", "good", "odd_reason"].sort()); + expect(checks.good.reason).toBe("needs-windows-run"); + expect(checks.odd_reason.reason).toBeNull(); + expect(Object.getPrototypeOf(checks)).toBe(Object.prototype); + doc.pr_checks = "nope"; + expect(parseFixes(JSON.stringify(doc))!.prChecks).toEqual({}); + delete doc.pr_checks; + expect(parseFixes(JSON.stringify(doc))!.prChecks).toEqual({}); + }); + test("doctor_pr is kept only when allowlisted", () => { const doc = JSON.parse(TRIAGE); doc.tasks[1].doctor_pr = "https://github.com/UiPath/skills/pull/9"; diff --git a/evalboard/lib/__tests__/fixtures/skill-doctor/fixes-1000001/fixes.json b/evalboard/lib/__tests__/fixtures/skill-doctor/fixes-1000001/fixes.json index b81f04c1..8815d293 100644 --- a/evalboard/lib/__tests__/fixtures/skill-doctor/fixes-1000001/fixes.json +++ b/evalboard/lib/__tests__/fixtures/skill-doctor/fixes-1000001/fixes.json @@ -12,18 +12,82 @@ "skipped": [], "candidates": [ { - "id": "example-grader-fix", "task_id": "example-t1", "covers": ["example-t1", "example-t3"], "layer": "grader", - "outcome": "pr-ready", "cost_usd": 1.5, "patch_lines": {"added": 3, "removed": 1}, "title": "fix(example): match the documented output", - "problem": "", "files": ["tests/tasks/example/example.yaml"], "overlap": [], "open_prs_on_files": [], - "gates": {"replay": {"status": "pass", "table": "", "evidence": ""}, "mutants": {"n": 10, "killed": 9, "survivors": 1, "pre_existing": 0}, - "flip_judge": null, "genuine": null, "review": "SEND", "holes": null, "history": []} + "id": "example-grader-fix", + "task_id": "example-t1", + "covers": [ + "example-t1", + "example-t3" + ], + "layer": "grader", + "outcome": "pr-ready", + "cost_usd": 1.5, + "patch_lines": { + "added": 3, + "removed": 1 + }, + "title": "fix(example): match the documented output", + "problem": "", + "files": [ + "tests/tasks/example/example.yaml" + ], + "overlap": [], + "open_prs_on_files": [], + "gates": { + "replay": { + "status": "pass", + "table": "", + "evidence": "" + }, + "mutants": { + "n": 10, + "killed": 9, + "survivors": 1, + "pre_existing": 0 + }, + "flip_judge": null, + "genuine": null, + "review": "SEND", + "holes": null, + "history": [] + } }, { - "id": "example-skill-fix", "task_id": "example-t2", "covers": ["example-t2"], "layer": "skill", - "outcome": "needs-live-run", "cost_usd": 1.0, "patch_lines": null, "title": "", - "problem": "blocked-env", "files": [], "overlap": [], "open_prs_on_files": [], - "gates": {"replay": {"status": "blocked-env", "table": "", "evidence": ""}, "mutants": null, - "flip_judge": null, "genuine": null, "review": null, "holes": null, "history": []} + "id": "example-skill-fix", + "task_id": "example-t2", + "covers": [ + "example-t2" + ], + "layer": "skill", + "outcome": "needs-live-run", + "cost_usd": 1.0, + "patch_lines": null, + "title": "", + "problem": "blocked-env", + "files": [], + "overlap": [], + "open_prs_on_files": [], + "gates": { + "replay": { + "status": "blocked-env", + "table": "", + "evidence": "" + }, + "mutants": null, + "flip_judge": null, + "genuine": null, + "review": null, + "holes": null, + "history": [] + } } - ] + ], + "pr_checks": { + "example-t2": { + "number": 126, + "url": "https://github.com/UiPath/skills/pull/126", + "verdict": "verified", + "claim": "fixes", + "evidence": "replay of #126 on 2 stored runs: run_command flips to pass on both" + } + } } diff --git a/evalboard/lib/__tests__/fixtures/skill-doctor/triage-1000001.json b/evalboard/lib/__tests__/fixtures/skill-doctor/triage-1000001.json index 0c0a03d3..03dfe861 100644 --- a/evalboard/lib/__tests__/fixtures/skill-doctor/triage-1000001.json +++ b/evalboard/lib/__tests__/fixtures/skill-doctor/triage-1000001.json @@ -77,7 +77,8 @@ "related": null, "passed_before": null, "existing_prs": null, - "doctor_pr": null + "doctor_pr": null, + "fixed_by_pr": null }, { "task_id": "example-t1", @@ -128,7 +129,8 @@ "title": "docs(example): document the output", "state": "merged" } - ] + ], + "fixed_by_pr": null }, { "task_id": "example-t2", @@ -149,7 +151,7 @@ "why": "The skill documents a removed flag.", "evidence": null, "solvable": "yes-skill", - "fix": "Document the current flag. Open PR #126 already does this; #127 tried before.", + "fix": "Document the current flag.", "related": null, "passed_before": null, "existing_prs": [ @@ -164,7 +166,12 @@ "state": "merged" } ], - "doctor_pr": null + "doctor_pr": null, + "fixed_by_pr": { + "number": 126, + "verdict": "fixes", + "basis": "The PR documents the current flag in the skill's reference page." + } }, { "task_id": "example-e2", @@ -191,7 +198,8 @@ "related": null, "passed_before": null, "existing_prs": null, - "doctor_pr": null + "doctor_pr": null, + "fixed_by_pr": null }, { "task_id": "example-e3", @@ -218,7 +226,8 @@ "related": null, "passed_before": null, "existing_prs": null, - "doctor_pr": null + "doctor_pr": null, + "fixed_by_pr": null } ], "actions": [ diff --git a/evalboard/lib/doctor.ts b/evalboard/lib/doctor.ts index 099c57b8..69c0e7f5 100644 --- a/evalboard/lib/doctor.ts +++ b/evalboard/lib/doctor.ts @@ -67,13 +67,30 @@ export interface DoctorTask { criterion: string | null; evidence: string | null; passedBefore: string | null; - // Only the source of a third-party "Proposed PR" (an open PR the fix names). + // Open/merged PRs on the task's files; only used to resolve fixedByPr. existingPrs: DoctorPr[]; + // The diagnosis's judgement of the open PR that best fixes this row. + fixedByPr: DoctorFixedByPr | null; envGroup: string | null; // The doctor's PR for this task, allowlisted (else null). doctorPr: string | null; } +export interface DoctorFixedByPr { + number: number; + verdict: "fixes" | "may_help" | "unrelated"; + basis: string; +} + +// The Fix stage's replay of a row's fixedByPr (fixes.json pr_checks). +export interface DoctorPrCheck { + number: number; + url: string; + verdict: "verified" | "not_verified" | "untested"; + evidence: string; + reason: "needs-windows-run" | null; +} + export interface DoctorAction { count: number; text: string; @@ -107,6 +124,8 @@ export interface DoctorFix { export interface DoctorFixes { fixMode: string | null; + // By task id; built through a Map so task ids are always own keys. + prChecks: Record; candidates: DoctorFix[]; } @@ -172,6 +191,42 @@ function parsePr(v: unknown): DoctorPr | null { return { url, title: str(v.title) ?? "", state: str(v.state) ?? "unknown" }; } +function parseFixedByPr(v: unknown): DoctorFixedByPr | null { + if (!isObj(v)) return null; + const n = v.number; + const verdict = v.verdict; + if (typeof n !== "number" || !Number.isInteger(n) || n < 1) return null; + if (verdict !== "fixes" && verdict !== "may_help" && verdict !== "unrelated") return null; + return { number: n, verdict, basis: (str(v.basis) ?? "").slice(0, 300) }; +} + +function parsePrCheck(v: unknown): DoctorPrCheck | null { + if (!isObj(v)) return null; + const n = v.number; + const verdict = v.verdict; + if (typeof n !== "number" || !Number.isInteger(n) || n < 1) return null; + if (!isSkillsPrUrl(v.url) || !v.url.endsWith(`/pull/${n}`)) return null; + if (verdict !== "verified" && verdict !== "not_verified" && verdict !== "untested") return null; + return { + number: n, + url: v.url, + verdict, + evidence: (str(v.evidence) ?? "").slice(0, 600), + reason: v.reason === "needs-windows-run" ? "needs-windows-run" : null, + }; +} + +function parsePrChecks(v: unknown): Record { + const out = new Map(); + if (isObj(v)) { + for (const [k, c] of Object.entries(v)) { + const p = parsePrCheck(c); + if (p) out.set(k, p); + } + } + return Object.fromEntries(out); +} + function prList(v: unknown): DoctorPr[] { return Array.isArray(v) ? v.map(parsePr).filter((p): p is DoctorPr => p !== null) : []; } @@ -193,6 +248,7 @@ function parseTask(v: unknown): DoctorTask | null { evidence: str(v.evidence), passedBefore: str(v.passed_before), existingPrs: prList(v.existing_prs), + fixedByPr: parseFixedByPr(v.fixed_by_pr), envGroup: str(v.env_group), doctorPr: isSkillsPrUrl(v.doctor_pr) ? v.doctor_pr : null, }; @@ -237,6 +293,7 @@ export function parseFixes(raw: string | null | undefined): DoctorFixes | null { if (!isObj(j) || j.schema_version !== 1 || !Array.isArray(j.candidates)) return null; return { fixMode: str(j.fix_mode), + prChecks: parsePrChecks(j.pr_checks), candidates: j.candidates.flatMap((c): DoctorFix[] => { if (!isObj(c)) return []; const id = str(c.id); From f17956af924691512483ca1c38a9c67f0ec8ec26 Mon Sep 17 00:00:00 2001 From: CarlesUIPath Date: Thu, 19 Nov 2026 22:02:01 +0000 Subject: [PATCH 11/13] feat(evalboard): accept pr_checks head_sha and show the short sha with the evidence head_sha is optional (string, null or absent); only a 40-hex value is kept. Tests cover it and the untested evidence texts of the final contract. Co-Authored-By: Claude Opus 5.5 --- .../__tests__/task-failure-analysis.test.tsx | 23 ++++++++++++++++++- .../app/runs/[id]/task-failure-analysis.tsx | 22 ++++++++++++++---- evalboard/lib/__tests__/doctor.test.ts | 17 ++++++++++++++ evalboard/lib/doctor.ts | 3 +++ 4 files changed, 59 insertions(+), 6 deletions(-) diff --git a/evalboard/app/runs/[id]/__tests__/task-failure-analysis.test.tsx b/evalboard/app/runs/[id]/__tests__/task-failure-analysis.test.tsx index 6a07aaaa..59ae5636 100644 --- a/evalboard/app/runs/[id]/__tests__/task-failure-analysis.test.tsx +++ b/evalboard/app/runs/[id]/__tests__/task-failure-analysis.test.tsx @@ -349,6 +349,27 @@ describe("TaskFailureAnalysis block", () => { expect(within(c).queryByTestId("maybe-pr")).toBeNull(); }); + test.each([ + "looks like a relaxation, not a fix: the PR loosens the check", + "the PR drops the failing criterion run_command: needs review", + "stored failure on a criterion the yaml no longer has", + ])("untested evidence %j renders as plain text under 'may fix'", (evidence) => { + const d = doctor(); + d.fixes!.prChecks["example-t2"] = { + ...d.fixes!.prChecks["example-t2"], + verdict: "untested", + evidence, + headSha: "0123456789abcdef0123456789abcdef01234567", + }; + renderBlock(d); + openBlock(); + openSkill("uipath-example-two"); + const row = within(card("example-t2")).getByTestId("maybe-pr"); + expect(row).toHaveTextContent(evidence); + expect(row).toHaveTextContent("(at 0123456)"); + expect(within(row).getAllByRole("link")).toHaveLength(1); + }); + test("a check for another PR number does not verify this one", () => { const d = doctor(); d.fixes!.prChecks["example-t2"].number = 4; @@ -535,7 +556,7 @@ describe("groupBySkill", () => { ]; d.opened = { "w-opened": "https://github.com/UiPath/skills/pull/8" }; d.fixes!.prChecks = { - "p-proposed": { number: 123, url: "https://github.com/UiPath/skills/pull/123", verdict: "verified", evidence: "", reason: null }, + "p-proposed": { number: 123, url: "https://github.com/UiPath/skills/pull/123", verdict: "verified", evidence: "", reason: null, headSha: null }, }; const [g] = groupBySkill(d, null, ""); expect(g.cards.map((t) => t.taskId)).toEqual([ diff --git a/evalboard/app/runs/[id]/task-failure-analysis.tsx b/evalboard/app/runs/[id]/task-failure-analysis.tsx index 167c956f..25ea4f34 100644 --- a/evalboard/app/runs/[id]/task-failure-analysis.tsx +++ b/evalboard/app/runs/[id]/task-failure-analysis.tsx @@ -211,8 +211,9 @@ export interface ExistingPrFix { verified: boolean; // The diagnosis's reason for naming this PR. basis: string; - // The replay's evidence, when there was a check. + // The replay's evidence, when there was a check, and the PR head it used. evidence: string | null; + headSha: string | null; // The check could not run: it needs a Windows run. needsWindows: boolean; } @@ -239,6 +240,7 @@ export function existingPrFix(doctor: RunDoctor, t: DoctorTask): ExistingPrFix | verified: check?.verdict === "verified", basis: f.basis, evidence: check?.evidence || null, + headSha: check?.headSha ?? null, needsWindows: check?.reason === "needs-windows-run", }; } @@ -509,6 +511,16 @@ function Small({ children }: { children: ReactNode }) { return {children}; } +// The replay's evidence as plain text, with the PR head it ran at. +function Evidence({ text, headSha }: { text: string; headSha: string | null }) { + return ( + + {text} + {headSha && (at {headSha.slice(0, 7)})} + + ); +} + // The card's PR rows: "PR with fix" (the doctor's PR), "Proposed PR" (someone // else's open PR, verified by the doctor's replay), "Fix prepared, PR not // opened yet", "Open PR may fix this (not verified)", and "Fix candidate: @@ -531,13 +543,13 @@ function FixRows({ fx, ctx }: { fx: TaskFixes; ctx: FixCtx }) { ); } if (fx.proposed) { - const { pr, evidence } = fx.proposed; + const { pr, evidence, headSha } = fx.proposed; rows.push( Proposed PR {pr.state} - {evidence && {evidence}} + {evidence && } , ); } @@ -551,14 +563,14 @@ function FixRows({ fx, ctx }: { fx: TaskFixes; ctx: FixCtx }) { ); } if (fx.maybe) { - const { pr, basis, evidence, needsWindows } = fx.maybe; + const { pr, basis, evidence, headSha, needsWindows } = fx.maybe; rows.push( Open PR may fix this (not verified) {needsWindows && needs a Windows run} {basis && {basis}} - {evidence && {evidence}} + {evidence && } , ); } diff --git a/evalboard/lib/__tests__/doctor.test.ts b/evalboard/lib/__tests__/doctor.test.ts index c9c18ff3..7a5b6227 100644 --- a/evalboard/lib/__tests__/doctor.test.ts +++ b/evalboard/lib/__tests__/doctor.test.ts @@ -204,6 +204,21 @@ describe("parseTriage / parseFixes", () => { }); }); + test("pr_checks: head_sha is kept when it is a 40-hex sha", () => { + const doc = JSON.parse(FIXES); + const sha = "0123456789abcdef0123456789abcdef01234567"; + const base = doc.pr_checks["example-t2"]; + doc.pr_checks = { + a: { ...base, head_sha: sha }, + b: { ...base, head_sha: null }, + c: { ...base, head_sha: "not-a-sha" }, + d: { ...base }, + }; + const c = parseFixes(JSON.stringify(doc))!.prChecks; + expect(Object.keys(c).sort()).toEqual(["a", "b", "c", "d"]); + expect([c.a.headSha, c.b.headSha, c.c.headSha, c.d.headSha]).toEqual([sha, null, null, null]); + }); + test("pr_checks: well-formed entries by task; malformed ones skipped", () => { const doc = JSON.parse(FIXES); const ok = { number: 5, url: "https://github.com/UiPath/skills/pull/5", verdict: "untested", evidence: "e" }; @@ -221,6 +236,8 @@ describe("parseTriage / parseFixes", () => { const checks = parseFixes(raw)!.prChecks; expect(Object.keys(checks).sort()).toEqual(["__proto__", "good", "odd_reason"].sort()); expect(checks.good.reason).toBe("needs-windows-run"); + // head_sha is optional: no key, null, or a malformed value all parse to null. + expect(checks.good.headSha).toBeNull(); expect(checks.odd_reason.reason).toBeNull(); expect(Object.getPrototypeOf(checks)).toBe(Object.prototype); doc.pr_checks = "nope"; diff --git a/evalboard/lib/doctor.ts b/evalboard/lib/doctor.ts index 69c0e7f5..57e32d75 100644 --- a/evalboard/lib/doctor.ts +++ b/evalboard/lib/doctor.ts @@ -89,6 +89,8 @@ export interface DoctorPrCheck { verdict: "verified" | "not_verified" | "untested"; evidence: string; reason: "needs-windows-run" | null; + // The PR head the check applied (40 hex), when the doctor recorded it. + headSha: string | null; } export interface DoctorAction { @@ -213,6 +215,7 @@ function parsePrCheck(v: unknown): DoctorPrCheck | null { verdict, evidence: (str(v.evidence) ?? "").slice(0, 600), reason: v.reason === "needs-windows-run" ? "needs-windows-run" : null, + headSha: typeof v.head_sha === "string" && /^[0-9a-f]{40}$/.test(v.head_sha) ? v.head_sha : null, }; } From 0636adcda445459ce0ce7d2a790d0928efbef8c7 Mon Sep 17 00:00:00 2001 From: CarlesUIPath Date: Thu, 19 Nov 2026 22:10:23 +0000 Subject: [PATCH 12/13] fix(evalboard): final review round for the Task failure analysis - Fix candidates with outcome not-run are hidden and no longer lift a card's sort tier. - A pr_checks entry that does not validate is kept as null, so its task shows no PR row instead of "may fix (not verified)". - rows_executed must be a non-negative integer. - One own() helper for report-keyed lookups; one existingPr field (verified or not) instead of proposed + maybe; Patch absorbs PatchCell; one ExtA anchor; DoctorPrCheck.url is validated but not kept; doc comments updated. - Tests: not-run candidates, a malformed not_verified check, task ids named constructor / __proto__ / toString with opened.json and pr_checks entries, rows_executed. Co-Authored-By: Claude Opus 5.5 --- .../__tests__/task-failure-analysis.test.tsx | 81 ++++++++++- .../app/runs/[id]/task-failure-analysis.tsx | 131 +++++++++--------- evalboard/lib/__tests__/doctor.test.ts | 25 +++- evalboard/lib/doctor.ts | 41 ++++-- 4 files changed, 184 insertions(+), 94 deletions(-) diff --git a/evalboard/app/runs/[id]/__tests__/task-failure-analysis.test.tsx b/evalboard/app/runs/[id]/__tests__/task-failure-analysis.test.tsx index 59ae5636..4b072ae4 100644 --- a/evalboard/app/runs/[id]/__tests__/task-failure-analysis.test.tsx +++ b/evalboard/app/runs/[id]/__tests__/task-failure-analysis.test.tsx @@ -294,7 +294,7 @@ describe("TaskFailureAnalysis block", () => { if (verdict === "none") d.fixes!.prChecks = {}; else d.fixes!.prChecks["example-t2"] = { - ...d.fixes!.prChecks["example-t2"], + ...d.fixes!.prChecks["example-t2"]!, verdict: "untested", evidence: "no stored run could replay it", reason: reason ? "needs-windows-run" : null, @@ -319,7 +319,7 @@ describe("TaskFailureAnalysis block", () => { }); test.each([ - ["not_verified check", (d: RunDoctor) => { d.fixes!.prChecks["example-t2"].verdict = "not_verified"; }], + ["not_verified check", (d: RunDoctor) => { d.fixes!.prChecks["example-t2"]!.verdict = "not_verified"; }], ["may_help", (d: RunDoctor) => { d.triage!.tasks[2].fixedByPr!.verdict = "may_help"; }], ["unrelated", (d: RunDoctor) => { d.triage!.tasks[2].fixedByPr!.verdict = "unrelated"; }], ["merged PR", (d: RunDoctor) => { d.triage!.tasks[2].fixedByPr!.number = 127; }], @@ -356,7 +356,7 @@ describe("TaskFailureAnalysis block", () => { ])("untested evidence %j renders as plain text under 'may fix'", (evidence) => { const d = doctor(); d.fixes!.prChecks["example-t2"] = { - ...d.fixes!.prChecks["example-t2"], + ...d.fixes!.prChecks["example-t2"]!, verdict: "untested", evidence, headSha: "0123456789abcdef0123456789abcdef01234567", @@ -370,9 +370,46 @@ describe("TaskFailureAnalysis block", () => { expect(within(row).getAllByRole("link")).toHaveLength(1); }); + test("a malformed check for the task (even a not_verified one) shows nothing", () => { + const d = doctor(); + // What parseFixes keeps for an entry that did not validate. + d.fixes!.prChecks["example-t2"] = null; + expect(existingPrFix(d, d.triage!.tasks[2])).toBeNull(); + const raw = JSON.parse(read("fixes-1000001/fixes.json")); + raw.pr_checks["example-t2"] = { number: 126, url: "https://evil.example/pull/126", verdict: "not_verified", evidence: "" }; + const d2 = { ...doctor(), fixes: parseFixes(JSON.stringify(raw)) }; + renderBlock(d2); + openBlock(); + openSkill("uipath-example-two"); + const c = card("example-t2"); + expect(within(c).queryByTestId("maybe-pr")).toBeNull(); + expect(within(c).queryByTestId("proposed-pr")).toBeNull(); + }); + + test("not-run candidates are hidden and do not lift the sort tier", () => { + const d = doctor(); + d.fixes!.candidates = d.fixes!.candidates.map((c) => ({ + ...c, + outcome: "not-run", + patchLines: null, + gates: { replay: null, mutants: null, review: null }, + })); + d.triage!.tasks[2].fixedByPr = null; + renderBlock(d); + openBlock(); + fireEvent.click(screen.getByRole("button", { name: "Expand all" })); + expect(block()).not.toHaveTextContent("not-run"); + expect(screen.queryAllByTestId("fix-candidate")).toHaveLength(0); + expect(screen.queryAllByTestId("fix-prepared")).toHaveLength(0); + const g = groupBySkill(d, null, ""); + // Nothing lifts either skill: more failed tasks first. + expect(g.map((x) => x.skill)).toEqual(["uipath-example-two", "uipath-example"]); + expect(g.every((x) => x.prepared === 0)).toBe(true); + }); + test("a check for another PR number does not verify this one", () => { const d = doctor(); - d.fixes!.prChecks["example-t2"].number = 4; + d.fixes!.prChecks["example-t2"]!.number = 4; expect(existingPrFix(d, d.triage!.tasks[2])).toMatchObject({ verified: false, evidence: null }); }); @@ -465,6 +502,40 @@ describe("untrusted keys", () => { ); }); +describe("prototype-named task ids", () => { + test.each(["constructor", "__proto__", "toString"])( + "task id %j: opened.json and pr_checks entries resolve to their own rows", + (k) => { + const d = doctor(); + const t1 = { ...d.triage!.tasks[1], taskId: k }; + const t2 = { ...d.triage!.tasks[2], taskId: `${k}-2` }; + d.triage!.tasks = [t1, t2]; + d.fixes!.candidates = [ + { ...d.fixes!.candidates[0], taskId: k, covers: [k] }, + ]; + // As parsed from JSON text, so "__proto__" is an own key. + d.opened = JSON.parse(JSON.stringify({}).replace("{}", `{"${k}": "${OPENED}"}`)); + d.fixes!.prChecks = JSON.parse( + `{"${k}-2": {"number": 126, "verdict": "verified", "evidence": "ok", "reason": null, "headSha": null}}`, + ); + // t1 claims #126 too, but has no check of its own: "may fix". + t1.fixedByPr = { number: 126, verdict: "fixes", basis: "b" }; + t1.existingPrs = t2.existingPrs; + renderBlock(d); + openBlock(); + fireEvent.click(screen.getByRole("button", { name: "Expand all" })); + const c1 = screen.getAllByTestId("task-card").find((c) => within(c).queryByText(k))!; + expect(within(c1).getByRole("link", { name: "PR with fix: #300 ↗" })).toBeInTheDocument(); + expect(within(c1).getByTestId("maybe-pr")).toHaveTextContent("#126"); + const c2 = card(`${k}-2`); + expect(within(c2).getByTestId("proposed-pr")).toHaveTextContent("#126"); + expect(within(c2).queryByTestId("pr-with-fix")).toBeNull(); + // #300 (own) + #126 (verified on the second task). + expect(digest(d)).toBe("5 errors · 2 PRs with fix"); + }, + ); +}); + describe("doctor PR resolution", () => { test("a task covered by an opened candidate gets that PR, not 'Fix prepared'", () => { const d = doctor(); @@ -556,7 +627,7 @@ describe("groupBySkill", () => { ]; d.opened = { "w-opened": "https://github.com/UiPath/skills/pull/8" }; d.fixes!.prChecks = { - "p-proposed": { number: 123, url: "https://github.com/UiPath/skills/pull/123", verdict: "verified", evidence: "", reason: null, headSha: null }, + "p-proposed": { number: 123, verdict: "verified", evidence: "", reason: null, headSha: null }, }; const [g] = groupBySkill(d, null, ""); expect(g.cards.map((t) => t.taskId)).toEqual([ diff --git a/evalboard/app/runs/[id]/task-failure-analysis.tsx b/evalboard/app/runs/[id]/task-failure-analysis.tsx index 25ea4f34..df25371a 100644 --- a/evalboard/app/runs/[id]/task-failure-analysis.tsx +++ b/evalboard/app/runs/[id]/task-failure-analysis.tsx @@ -9,6 +9,7 @@ import { isAdoUrl, isLinkableTaskId, isSkillsPrUrl, + own, patchIds, type DoctorFix, type DoctorPr, @@ -84,15 +85,9 @@ export function RichText({ text }: { text: string }) { const url = seg.replace(/[.,;:]+$/, ""); if (isSkillsPrUrl(url) || isAdoUrl(url)) { out.push( - + {url} - , + , seg.slice(url.length), ); } else out.push(seg); @@ -101,21 +96,29 @@ export function RichText({ text }: { text: string }) { return <>{out}; } -/** A build link only for a dev.azure.com URL; anything else is the label as text. */ -function BuildLink({ href, label }: { href: string; label: string }) { - if (!isAdoUrl(href)) return {label}; +/** An external link, opened in a new tab. Callers pass only allowlisted URLs. */ +function ExtA({ + href, + className = "text-studio-blue hover:underline", + children, +}: { + href: string; + className?: string; + children: ReactNode; +}) { return ( - - {label} + + {children} ); } +/** A build link only for a dev.azure.com URL; anything else is the label as text. */ +function BuildLink({ href, label }: { href: string; label: string }) { + if (!isAdoUrl(href)) return {label}; + return {label}; +} + function prLabel(url: string): string { const m = /\/pull\/(\d+)$/.exec(url); return m ? `#${m[1]}` : url; @@ -171,20 +174,6 @@ interface FixCtx { linked: Set; } -function PatchCell({ fix, ctx }: { fix: DoctorFix; ctx: FixCtx }) { - if (!fix.patchLines) return null; - const text = `+${fix.patchLines.added}/-${fix.patchLines.removed}`; - if (!ctx.dir || !ctx.linked.has(fix.id)) return {text}; - return ( - - {text} - - ); -} - // ---------- grouping ---------- export interface SkillGroup { @@ -199,9 +188,11 @@ export interface SkillGroup { total: number; } +// A candidate whose Fix stage never ran (outcome not-run) has no patch and no +// gates: the board leaves it out entirely. export function fixesForTask(doctor: RunDoctor, taskId: string): DoctorFix[] { return (doctor.fixes?.candidates ?? []).filter( - (c) => c.taskId === taskId || c.covers.includes(taskId), + (c) => c.outcome !== "not-run" && (c.taskId === taskId || c.covers.includes(taskId)), ); } @@ -224,15 +215,15 @@ export interface ExistingPrFix { * verdict "fixes", the PR is open in the row's existing PRs, and the Fix * stage's pr_checks entry for the row (same number) is verified (counts as a * PR with fix) or untested / absent (shown as "may fix this", not counted). - * not_verified, may_help and unrelated show nothing. + * not_verified, a malformed check, may_help and unrelated show nothing. */ export function existingPrFix(doctor: RunDoctor, t: DoctorTask): ExistingPrFix | null { const f = t.fixedByPr; if (!f || f.verdict !== "fixes") return null; const pr = t.existingPrs.find((p) => p.state === "open" && p.url.endsWith(`/pull/${f.number}`)); if (!pr) return null; - const checks = doctor.fixes?.prChecks; - const c = checks && Object.hasOwn(checks, t.taskId) ? checks[t.taskId] : null; + const c = own(doctor.fixes?.prChecks, t.taskId); + if (c === null) return null; // an entry that did not validate const check = c && c.number === f.number ? c : null; if (check?.verdict === "not_verified") return null; return { @@ -255,31 +246,29 @@ export interface TaskFixes { // pr-ready candidates with no PR opened yet: "Fix prepared, PR not // opened yet". Not a PR with fix. prepared: DoctorFix[]; - // Someone else's open PR the doctor verified fixes this row ("Proposed - // PR"; a PR with fix), or one it could not verify ("Open PR may fix - // this", not counted). Never the doctor's own PR. - proposed: ExistingPrFix | null; - maybe: ExistingPrFix | null; + // Someone else's open PR for this row: verified ("Proposed PR"; a PR with + // fix) or not ("Open PR may fix this", not counted). Never the doctor's + // own PR. + existingPr: ExistingPrFix | null; // Other fix candidates (needs-live-run, ...). others: DoctorFix[]; - // One key per distinct PR with fix (own or proposed), for counting. + // One key per distinct PR with fix (own or verified existing), for counting. keys: string[]; // Card sort tier, lower first (see fixTier). tier: number; } function openedFor(doctor: RunDoctor, taskId: string): string | null { - const o = doctor.opened; - return o && Object.hasOwn(o, taskId) ? o[taskId] : null; + return own(doctor.opened, taskId) ?? null; } // Own PR, then a verified proposed PR, then a prepared fix, then an // unverified open PR, then other candidates, then the rest. function fixTier(f: Omit): number { if (f.ownPr) return 0; - if (f.proposed) return 1; + if (f.existingPr?.verified) return 1; if (f.prepared.length) return 2; - if (f.maybe) return 3; + if (f.existingPr) return 3; if (f.others.length) return 4; return 5; } @@ -293,16 +282,17 @@ export function taskFixes(doctor: RunDoctor, t: DoctorTask): TaskFixes { ready.map((c) => openedFor(doctor, c.taskId)).find((u) => u !== null) ?? t.doctorPr; const ext = existingPrFix(doctor, t); - const other = ext && ext.pr.url !== ownPr ? ext : null; const base = { ownPr, ready, prepared: ownPr ? [] : ready, - proposed: other?.verified ? other : null, - maybe: other && !other.verified ? other : null, + existingPr: ext && ext.pr.url !== ownPr ? ext : null, others: cands.filter((c) => !isReady(c)), }; - const keys = [...(ownPr ? [ownPr] : []), ...(base.proposed ? [base.proposed.pr.url] : [])]; + const keys = [ + ...(ownPr ? [ownPr] : []), + ...(base.existingPr?.verified ? [base.existingPr.pr.url] : []), + ]; return { ...base, keys, tier: fixTier(base) }; } @@ -459,11 +449,23 @@ function Gates({ cands }: { cands: DoctorFix[] }) { return {g.join("; ")}; } +// "patch +a/-r", linked to the download only for an id in patchIds. function Patch({ c, ctx }: { c: DoctorFix; ctx: FixCtx }) { if (!c.patchLines) return null; + const text = `+${c.patchLines.added}/-${c.patchLines.removed}`; return ( - patch + patch{" "} + {ctx.dir && ctx.linked.has(c.id) ? ( + + {text} + + ) : ( + {text} + )} ); } @@ -496,14 +498,9 @@ function FixRow({ function PrLink({ pr }: { pr: DoctorPr }) { return ( - + {prLabel(pr.url)} {pr.title} - + ); } @@ -530,20 +527,16 @@ function FixRows({ fx, ctx }: { fx: TaskFixes; ctx: FixCtx }) { if (fx.ownPr) { rows.push( - + PR with fix: {prLabel(fx.ownPr)} ↗ - + , ); } - if (fx.proposed) { - const { pr, evidence, headSha } = fx.proposed; + const ext = fx.existingPr; + if (ext?.verified) { + const { pr, evidence, headSha } = ext; rows.push( Proposed PR @@ -562,8 +555,8 @@ function FixRows({ fx, ctx }: { fx: TaskFixes; ctx: FixCtx }) { , ); } - if (fx.maybe) { - const { pr, basis, evidence, headSha, needsWindows } = fx.maybe; + if (ext && !ext.verified) { + const { pr, basis, evidence, headSha, needsWindows } = ext; rows.push( Open PR may fix this (not verified) diff --git a/evalboard/lib/__tests__/doctor.test.ts b/evalboard/lib/__tests__/doctor.test.ts index 7a5b6227..e1e8e26f 100644 --- a/evalboard/lib/__tests__/doctor.test.ts +++ b/evalboard/lib/__tests__/doctor.test.ts @@ -171,6 +171,14 @@ describe("parseTriage / parseFixes", () => { expect(t.tasks[0].envGroup).toBe("connection example-conn"); }); + test("rows_executed must be a non-negative integer", () => { + const doc = JSON.parse(TRIAGE); + for (const [v, want] of [[10, 10], [0, 0], [-1, null], [2.5, null], ["10", null], [null, null]] as const) { + doc.run.rows_executed = v; + expect(parseTriage(JSON.stringify(doc))!.rowsRun).toBe(want); + } + }); + test("schema_version 2 or a broken triage is absent", () => { expect(parseTriage(JSON.stringify({ ...JSON.parse(TRIAGE), schema_version: 2 }))).toBeNull(); expect(parseTriage("{")).toBeNull(); @@ -216,7 +224,7 @@ describe("parseTriage / parseFixes", () => { }; const c = parseFixes(JSON.stringify(doc))!.prChecks; expect(Object.keys(c).sort()).toEqual(["a", "b", "c", "d"]); - expect([c.a.headSha, c.b.headSha, c.c.headSha, c.d.headSha]).toEqual([sha, null, null, null]); + expect([c.a!.headSha, c.b!.headSha, c.c!.headSha, c.d!.headSha]).toEqual([sha, null, null, null]); }); test("pr_checks: well-formed entries by task; malformed ones skipped", () => { @@ -234,11 +242,18 @@ describe("parseTriage / parseFixes", () => { // A "__proto__" task id, as JSON text (an object literal would set the prototype). const raw = JSON.stringify(doc).replace('"pr_checks":{', `"pr_checks":{"__proto__":${JSON.stringify(ok)},`); const checks = parseFixes(raw)!.prChecks; - expect(Object.keys(checks).sort()).toEqual(["__proto__", "good", "odd_reason"].sort()); - expect(checks.good.reason).toBe("needs-windows-run"); + // Malformed entries stay as null, so their task shows no PR row. + expect(Object.keys(checks).sort()).toEqual( + ["__proto__", "good", "odd_reason", "bad_host", "wrong_number", "bad_verdict", "no_number", "not_obj"].sort(), + ); + for (const k of ["bad_host", "wrong_number", "bad_verdict", "no_number", "not_obj"]) { + expect(checks[k]).toBeNull(); + } + expect(checks.__proto__).not.toBeNull(); + expect(checks.good!.reason).toBe("needs-windows-run"); // head_sha is optional: no key, null, or a malformed value all parse to null. - expect(checks.good.headSha).toBeNull(); - expect(checks.odd_reason.reason).toBeNull(); + expect(checks.good!.headSha).toBeNull(); + expect(checks.odd_reason!.reason).toBeNull(); expect(Object.getPrototypeOf(checks)).toBe(Object.prototype); doc.pr_checks = "nope"; expect(parseFixes(JSON.stringify(doc))!.prChecks).toEqual({}); diff --git a/evalboard/lib/doctor.ts b/evalboard/lib/doctor.ts index 57e32d75..2b431250 100644 --- a/evalboard/lib/doctor.ts +++ b/evalboard/lib/doctor.ts @@ -2,12 +2,13 @@ // `/skill-doctor/`: // - status.json: pointer + lock, changes while the doctor runs; // - triage-.json: the per-task failure analysis, immutable; -// - /fixes.json: proposed fixes, optional, create-only. +// - /fixes.json: fix candidates and pr_checks, optional, create-only; +// - /opened.json: the PRs the doctor's open step opened, optional. // // Pure data + string helpers, no Node builtins: the parsed shapes are passed to // the run page's components, and the tests import this module directly. The -// types are written by hand against the doctor's status_v1 / triage_v1 schemas -// and carry only the fields the run page renders. A file that does not parse, +// types are written by hand against the doctor's status_v1 / triage_v1 / +// fixes_v1 schemas and carry only the fields the run page renders. A file that does not parse, // or whose schema_version is not 1, is treated as absent: the page must never // fail because of a doctor file. @@ -82,10 +83,11 @@ export interface DoctorFixedByPr { basis: string; } -// The Fix stage's replay of a row's fixedByPr (fixes.json pr_checks). +// The Fix stage's replay of a row's fixedByPr (fixes.json pr_checks). Its +// url is validated (allowlisted, ends in /pull/) but not kept: the +// board links the PR from the row's existing PRs. export interface DoctorPrCheck { number: number; - url: string; verdict: "verified" | "not_verified" | "untested"; evidence: string; reason: "needs-windows-run" | null; @@ -126,8 +128,9 @@ export interface DoctorFix { export interface DoctorFixes { fixMode: string | null; - // By task id; built through a Map so task ids are always own keys. - prChecks: Record; + // By task id (own keys; read with own()). null: an entry that did not + // validate. + prChecks: Record; candidates: DoctorFix[]; } @@ -211,7 +214,6 @@ function parsePrCheck(v: unknown): DoctorPrCheck | null { if (verdict !== "verified" && verdict !== "not_verified" && verdict !== "untested") return null; return { number: n, - url: v.url, verdict, evidence: (str(v.evidence) ?? "").slice(0, 600), reason: v.reason === "needs-windows-run" ? "needs-windows-run" : null, @@ -219,17 +221,21 @@ function parsePrCheck(v: unknown): DoctorPrCheck | null { }; } -function parsePrChecks(v: unknown): Record { - const out = new Map(); +// A malformed entry is kept as null, so its task shows no PR row rather than +// falling back to "no check" (which would read as "may fix"). +function parsePrChecks(v: unknown): Record { + const out = new Map(); if (isObj(v)) { - for (const [k, c] of Object.entries(v)) { - const p = parsePrCheck(c); - if (p) out.set(k, p); - } + for (const [k, c] of Object.entries(v)) out.set(k, parsePrCheck(c)); } return Object.fromEntries(out); } +/** A report-keyed lookup that never reads the prototype ("constructor", ...). */ +export function own(r: Record | null | undefined, key: string): T | undefined { + return r && Object.hasOwn(r, key) ? r[key] : undefined; +} + function prList(v: unknown): DoctorPr[] { return Array.isArray(v) ? v.map(parsePr).filter((p): p is DoctorPr => p !== null) : []; } @@ -264,7 +270,12 @@ export function parseTriage(raw: string | null | undefined): DoctorTriage | null const s = j.summary; const run = isObj(j.run) ? j.run : {}; return { - rowsRun: typeof run.rows_executed === "number" ? int(run.rows_executed) : null, + rowsRun: + typeof run.rows_executed === "number" && + Number.isInteger(run.rows_executed) && + run.rows_executed >= 0 + ? run.rows_executed + : null, rowsFailed: int(s.rows_failed), rowsDiagnosed: int(s.rows_diagnosed), tasks: j.tasks.map(parseTask).filter((t): t is DoctorTask => t !== null), From 1cdb443151bc50046ba2ec0c8a7a716bfd144c23 Mon Sep 17 00:00:00 2001 From: CarlesUIPath Date: Fri, 20 Nov 2026 10:49:28 +0000 Subject: [PATCH 13/13] fix(evalboard): owner review round for the Task failure analysis - isValidId / isValidTaskId reject dot-only ids, so /api/file?run=.. can no longer read above the runs dir - resolveSafePath dispatches on the normalized path: ./skill-doctor/... and activation/../skill-doctor/... no longer skip the skill-doctor gate - /api/file: a storage error on a doctor patch propagates (5xx) instead of reading as 403/404; a patch link fetches only that patch; the page still never throws and says "could not be fetched" - only a done report fetches or serves fixes (a retracted one keeps fixes_dir but serves nothing) - a triage or fixes.json that names another run is ignored - a triage with a missing or malformed rows_failed / rows_diagnosed does not parse (no false "0 errors"); mutant counts must be integers - copies of the doctor's v1 schemas under the test fixtures, with a parity test for every field the parsers read and the closed value sets (states, layers, verdicts, the open-pr action kind) Not in this round: comparing run_json_etag with run.json's etag (a run.json re-graded in place after the doctor ran); fixes_v1 does not pin the candidate fields covers / patch_lines / title / gates yet. Co-Authored-By: Claude Opus 5.5 --- .../__tests__/task-failure-analysis.test.tsx | 7 + .../app/runs/[id]/task-failure-analysis.tsx | 11 +- evalboard/lib/__tests__/doctor-blob.test.ts | 103 ++++++++++++- evalboard/lib/__tests__/doctor-schema.test.ts | 134 ++++++++++++++++ evalboard/lib/__tests__/doctor.test.ts | 26 ++++ .../fixtures/skill-doctor/schemas/README.md | 11 ++ .../skill-doctor/schemas/fixes_v1.json | 57 +++++++ .../skill-doctor/schemas/status_v1.json | 48 ++++++ .../skill-doctor/schemas/triage_v1.json | 145 ++++++++++++++++++ evalboard/lib/doctor.ts | 37 +++-- evalboard/lib/ids.ts | 9 +- evalboard/lib/runs.ts | 61 ++++++-- 12 files changed, 606 insertions(+), 43 deletions(-) create mode 100644 evalboard/lib/__tests__/doctor-schema.test.ts create mode 100644 evalboard/lib/__tests__/fixtures/skill-doctor/schemas/README.md create mode 100644 evalboard/lib/__tests__/fixtures/skill-doctor/schemas/fixes_v1.json create mode 100644 evalboard/lib/__tests__/fixtures/skill-doctor/schemas/status_v1.json create mode 100644 evalboard/lib/__tests__/fixtures/skill-doctor/schemas/triage_v1.json diff --git a/evalboard/app/runs/[id]/__tests__/task-failure-analysis.test.tsx b/evalboard/app/runs/[id]/__tests__/task-failure-analysis.test.tsx index 4b072ae4..2fe6e57f 100644 --- a/evalboard/app/runs/[id]/__tests__/task-failure-analysis.test.tsx +++ b/evalboard/app/runs/[id]/__tests__/task-failure-analysis.test.tsx @@ -61,6 +61,13 @@ describe("digest", () => { expect(digest(doctor())).toBe("5 errors · 1 PR with fix"); expect(digest(withOpened())).toBe("5 errors · 2 PRs with fix"); }); + test("done without a triage: a fetch error is not blamed on the contract", () => { + const d = { ...doctor(), triage: null }; + expect(digest(d)).toBe("report done, but its triage file is missing or did not parse"); + expect(digest({ ...d, fetchFailed: true })).toBe( + "report done, but its triage file could not be fetched", + ); + }); test("no linked PR: the errors only", () => { const d = doctor(); d.triage!.tasks[2].fixedByPr = null; diff --git a/evalboard/app/runs/[id]/task-failure-analysis.tsx b/evalboard/app/runs/[id]/task-failure-analysis.tsx index df25371a..ae8082a4 100644 --- a/evalboard/app/runs/[id]/task-failure-analysis.tsx +++ b/evalboard/app/runs/[id]/task-failure-analysis.tsx @@ -37,12 +37,12 @@ const LAYERS = new Map([ ["flaky", { label: "Flaky", color: "#7d8794" }], ["undiagnosed", { label: "Not diagnosed", color: "#9aa5b3" }], ]); -const ORDER = [...LAYERS.keys()]; +export const LAYER_ORDER = [...LAYERS.keys()]; const OTHER_COLOR = "#9aa5b3"; const layerOf = (k: string) => LAYERS.get(k) ?? { label: k, color: OTHER_COLOR }; const rank = (k: string) => { - const i = ORDER.indexOf(k); - return i === -1 ? ORDER.length : i; + const i = LAYER_ORDER.indexOf(k); + return i === -1 ? LAYER_ORDER.length : i; }; const SOLVABLE = new Map([ @@ -140,7 +140,10 @@ export function digest(doctor: RunDoctor): string { case "retracted": return "the report for this run was retracted"; case "done": { - if (!triage) return "report done, but its triage file is missing or did not parse"; + if (!triage) + return doctor.fetchFailed + ? "report done, but its triage file could not be fetched" + : "report done, but its triage file is missing or did not parse"; const n = countFixPrs(doctor, triage.tasks); const errors = `${triage.rowsFailed} error${triage.rowsFailed === 1 ? "" : "s"}`; return n ? `${errors} · ${prsWithFix(n)}` : errors; diff --git a/evalboard/lib/__tests__/doctor-blob.test.ts b/evalboard/lib/__tests__/doctor-blob.test.ts index 7c9f0330..dd5b16d6 100644 --- a/evalboard/lib/__tests__/doctor-blob.test.ts +++ b/evalboard/lib/__tests__/doctor-blob.test.ts @@ -133,6 +133,61 @@ describe("readRunDoctor: what it fetches", () => { expect(state.downloads).toEqual([`${D}/status.json`]); }); + test.each(["queued", "running", "failed", "skipped", "retracted"])( + "%s with fixes_dir kept: only status.json is fetched, no patch is served", + async (st) => { + seedBlobs(statusWith({ state: st })); + const doctor = await read(); + expect(doctor?.status.state).toBe(st); + expect(doctor?.triage ?? null).toBeNull(); + expect(doctor?.fixes ?? null).toBeNull(); + const { resolveSafePath } = await import("../runs"); + expect( + await resolveSafePath(RUN, "skill-doctor/fixes-1000001/example-grader-fix.patch"), + ).toBeNull(); + expect([...new Set(state.downloads)]).toEqual([`${D}/status.json`]); + }, + ); + + test("a storage error on the triage: the page still loads, flagged fetchFailed", async () => { + state.failWith = { statusCode: 500 }; + state.failOn = `${D}/triage-1000001.json`; + const err = vi.spyOn(console, "error").mockImplementation(() => {}); + const doctor = await read(); + expect(doctor?.triage).toBeNull(); + expect(doctor?.fetchFailed).toBe(true); + expect(err).toHaveBeenCalled(); + err.mockRestore(); + }); + + test("a triage that names another run is ignored", async () => { + const t = JSON.parse(fixture("triage-1000001.json")); + t.run.run_id = "2099-01-02_04-23-30"; + state.blobs[`${D}/triage-1000001.json`] = JSON.stringify(t); + const warn = vi.spyOn(console, "warn").mockImplementation(() => {}); + const doctor = await read(); + expect(doctor?.triage).toBeNull(); + expect(doctor?.fetchFailed).toBe(false); + expect(warn).toHaveBeenCalledWith(expect.stringMatching(/names run/)); + warn.mockRestore(); + }); + + test("a fixes.json that names another run lists no patch", async () => { + const f = JSON.parse(fixture("fixes-1000001/fixes.json")); + f.run_id = "2099-01-02_04-23-30"; + state.blobs[`${D}/fixes-1000001/fixes.json`] = JSON.stringify(f); + const warn = vi.spyOn(console, "warn").mockImplementation(() => {}); + const doctor = await read(); + expect(doctor?.fixes).toBeNull(); + expect(doctor?.triage).not.toBeNull(); + const { resolveSafePath } = await import("../runs"); + expect( + await resolveSafePath(RUN, "skill-doctor/fixes-1000001/example-grader-fix.patch"), + ).toBeNull(); + expect(state.downloads.some((n) => n.endsWith(".patch"))).toBe(false); + warn.mockRestore(); + }); + test("a bad triage_file name is never fetched", async () => { seedBlobs(statusWith({ triage_file: "../run.json", fixes_dir: null })); expect((await read())?.triage).toBeNull(); @@ -302,20 +357,52 @@ describe("readRunDoctor", () => { expect(state.downloads).toEqual([]); }); - test("a storage error on the patch: no throw, nothing on disk", async () => { + test.each([ + ["the patch", `${D}/fixes-1000001/example-grader-fix.patch`], + ["fixes.json", `${D}/fixes-1000001/fixes.json`], + ["status.json", `${D}/status.json`], + ])("a storage error on %s propagates (the route answers 5xx, not 403/404)", async (_, name) => { state.failWith = { statusCode: 500 }; - state.failOn = `${D}/fixes-1000001/example-grader-fix.patch`; - const err = vi.spyOn(console, "error").mockImplementation(() => {}); + state.failOn = name; + const { resolveSafePath } = await import("../runs"); + await expect( + resolveSafePath(RUN, "skill-doctor/fixes-1000001/example-grader-fix.patch"), + ).rejects.toThrow(/blob failed/); + }); + + test("a patch link fetches only that patch: a storage error elsewhere does not break it", async () => { + state.failWith = { statusCode: 500 }; + state.failOn = `${D}/fixes-1000001/opened.json`; const { resolveSafePath } = await import("../runs"); const abs = await resolveSafePath( RUN, "skill-doctor/fixes-1000001/example-grader-fix.patch", ); - // /api/file then answers 404 for the missing file. - expect(abs).not.toBeNull(); - await expect(fs.access(abs!)).rejects.toThrow(); - expect(err).toHaveBeenCalled(); - err.mockRestore(); + expect(abs).toMatch(/example-grader-fix\.patch$/); + await expect(fs.access(abs!)).resolves.toBeUndefined(); + expect(state.downloads).not.toContain(`${D}/fixes-1000001/opened.json`); + }); + + test("normalized before the dispatch: ./ and ../ cannot skip the gate", async () => { + vi.stubEnv("EVALBOARD_EDITION", ""); + const { resolveSafePath } = await import("../runs"); + for (const bad of [ + "./skill-doctor/fixes-1000001/example-grader-fix.patch", + "activation/../skill-doctor/fixes-1000001/example-grader-fix.patch", + "./skill-doctor/status.json", + "../x", + "..", + ]) { + expect(await resolveSafePath(RUN, bad)).toBeNull(); + } + expect(state.downloads).toEqual([]); + }); + + test.each(["..", "."])("a run id of %j is refused", async (id) => { + const { resolveSafePath } = await import("../runs"); + expect(await resolveSafePath(id, ".env")).toBeNull(); + expect(await resolveSafePath(id, "run.json")).toBeNull(); + expect(state.downloads).toEqual([]); }); test("only patch paths are served from skill-doctor/", async () => { diff --git a/evalboard/lib/__tests__/doctor-schema.test.ts b/evalboard/lib/__tests__/doctor-schema.test.ts new file mode 100644 index 00000000..353e58be --- /dev/null +++ b/evalboard/lib/__tests__/doctor-schema.test.ts @@ -0,0 +1,134 @@ +import { readFileSync } from "node:fs"; +import path from "node:path"; +import { describe, expect, test } from "vitest"; +import { LAYER_ORDER } from "@/app/runs/[id]/task-failure-analysis"; +import { DOCTOR_STATES } from "../doctor"; + +// The board's parsers (lib/doctor.ts) are written by hand against the doctor's +// contract v1. These tests pin them to copies of the producer's schemas +// (fixtures/skill-doctor/schemas/, see its README): a field the parsers read +// must be declared there, and the hand-written fixtures may use only declared +// keys. A rename on the producer side then fails here once the copies are +// refreshed, instead of silently dropping a column on the board. + +type Schema = Record; + +const FIX = path.join(__dirname, "fixtures", "skill-doctor"); +const load = (p: string) => JSON.parse(readFileSync(path.join(FIX, p), "utf-8")); +const SCHEMAS: Record = { + status: load("schemas/status_v1.json"), + triage: load("schemas/triage_v1.json"), + fixes: load("schemas/fixes_v1.json"), +}; + +function deref(root: Schema, node: unknown): Schema | null { + let n = node; + while (n && typeof n === "object" && "$ref" in n) { + const ref = (n as Schema).$ref as string; + n = (root.$defs as Schema)[ref.replace("#/$defs/", "")]; + } + return n && typeof n === "object" ? (n as Schema) : null; +} + +// "a.[].b": `[]` steps into array items, `*` into additionalProperties. +function nodeAt(root: Schema, p: string): Schema | null { + let node: Schema | null = root; + for (const seg of p.split(".")) { + node = deref(root, node); + if (!node) return null; + if (seg === "[]") node = deref(root, node.items); + else if (seg === "*") node = deref(root, node.additionalProperties); + else node = deref(root, (node.properties as Schema | undefined)?.[seg]); + } + return node; +} + +// Every field the parsers in lib/doctor.ts read. +const READS: Record = { + status: [ + "schema_version", "state", "reason", "started_at", "build_url", + "triage_file", "fixes_dir", "trigger", + ], + triage: [ + "schema_version", "run.run_id", "run.rows_executed", + "summary.rows_failed", "summary.rows_diagnosed", + ...[ + "task_id", "skill", "layer", "solvable", "run_status", "confidence", + "why", "fix", "criterion", "evidence", "passed_before", "env_group", + "doctor_pr", "existing_prs.[].url", "existing_prs.[].title", + "existing_prs.[].state", "fixed_by_pr.number", "fixed_by_pr.verdict", + "fixed_by_pr.basis", + ].map((f) => `tasks.[].${f}`), + "actions.[].kind", "actions.[].count", "actions.[].text", "actions.[].task_ids", + ], + fixes: [ + "schema_version", "fix_mode", "candidates.[].outcome", + ...["number", "verdict", "url", "evidence", "reason", "head_sha"].map( + (f) => `pr_checks.*.${f}`, + ), + ], +}; + +describe("doctor contract: the parsers read only declared fields", () => { + test.each(Object.entries(READS).flatMap(([s, ps]) => ps.map((p) => [s, p])))( + "%s_v1 declares %s", + (s, p) => { + expect(nodeAt(SCHEMAS[s], p)).not.toBeNull(); + }, + ); + + test("fixes_v1 requires the candidate keys the board needs", () => { + const items = deref(SCHEMAS.fixes, nodeAt(SCHEMAS.fixes, "candidates")?.items); + expect(items?.required).toEqual(expect.arrayContaining(["id", "task_id", "outcome"])); + }); +}); + +const enumAt = (s: string, p: string) => nodeAt(SCHEMAS[s], p)?.enum as string[]; + +describe("doctor contract: closed value sets match the producer", () => { + test("status states", () => { + expect([...DOCTOR_STATES].sort()).toEqual([...enumAt("status", "state")].sort()); + }); + + test("every layer has a label, and only those", () => { + expect([...LAYER_ORDER].sort()).toEqual([...enumAt("triage", "tasks.[].layer")].sort()); + }); + + test("the action kind the board leaves out still exists", () => { + expect(enumAt("triage", "actions.[].kind")).toContain("open-pr"); + }); + + test("PR verdicts", () => { + expect([...enumAt("triage", "tasks.[].fixed_by_pr.verdict")].sort()).toEqual( + ["fixes", "may_help", "unrelated"], + ); + expect([...enumAt("fixes", "pr_checks.*.verdict")].sort()).toEqual( + ["not_verified", "untested", "verified"], + ); + }); +}); + +// Keys a fixture uses that the schema at that point does not declare (where +// it declares properties and closes the rest). +function undeclared(root: Schema, node: unknown, value: unknown, at = ""): string[] { + const s = deref(root, node); + if (!s || value === null || typeof value !== "object") return []; + if (Array.isArray(value)) + return value.flatMap((v, i) => undeclared(root, s.items, v, `${at}[${i}]`)); + const props = (s.properties ?? {}) as Schema; + return Object.entries(value).flatMap(([k, v]) => { + if (k in props) return undeclared(root, props[k], v, `${at}.${k}`); + if (s.additionalProperties === false) return [`${at}.${k}`]; + return undeclared(root, s.additionalProperties, v, `${at}.${k}`); + }); +} + +describe("doctor contract: the hand-written fixtures use declared keys", () => { + test.each([ + ["status", "status.json"], + ["triage", "triage-1000001.json"], + ["fixes", "fixes-1000001/fixes.json"], + ])("%s fixture %s", (s, file) => { + expect(undeclared(SCHEMAS[s], SCHEMAS[s], load(file))).toEqual([]); + }); +}); diff --git a/evalboard/lib/__tests__/doctor.test.ts b/evalboard/lib/__tests__/doctor.test.ts index e1e8e26f..41b2892d 100644 --- a/evalboard/lib/__tests__/doctor.test.ts +++ b/evalboard/lib/__tests__/doctor.test.ts @@ -133,6 +133,32 @@ describe("parseDoctorStatus", () => { }); describe("parseTriage / parseFixes", () => { + test.each([ + ["no counts", {}], + ["a string count", { rows_failed: "5", rows_diagnosed: 3 }], + ["a negative count", { rows_failed: -1, rows_diagnosed: 3 }], + ["a fractional count", { rows_failed: 5, rows_diagnosed: 2.5 }], + ])("a triage with %s does not parse (never a false 0)", (_, summary) => { + const t = JSON.parse(TRIAGE); + t.summary = { ...t.summary, rows_failed: undefined, rows_diagnosed: undefined, ...summary }; + expect(parseTriage(JSON.stringify(t))).toBeNull(); + }); + + test("the run ids are kept, for readRunDoctor's check", () => { + expect(parseTriage(TRIAGE)!.runId).toBe("2099-01-03_04-23-30"); + expect(parseFixes(FIXES)!.runId).toBe("2099-01-03_04-23-30"); + }); + + test.each([ + [{ n: 3.5, killed: 1 }], + [{ n: 3, killed: -1 }], + [{ n: "3", killed: 1 }], + ])("mutants %j are not shown as counts", (mutants) => { + const doc = JSON.parse(FIXES); + doc.candidates[0].gates = { ...doc.candidates[0].gates, mutants }; + expect(parseFixes(JSON.stringify(doc))!.candidates[0].gates.mutants).toBeNull(); + }); + test("reads the fixture: the card fields", () => { const t = parseTriage(TRIAGE)!; expect(t.tasks.map((x) => x.taskId)).toEqual([ diff --git a/evalboard/lib/__tests__/fixtures/skill-doctor/schemas/README.md b/evalboard/lib/__tests__/fixtures/skill-doctor/schemas/README.md new file mode 100644 index 00000000..017986cd --- /dev/null +++ b/evalboard/lib/__tests__/fixtures/skill-doctor/schemas/README.md @@ -0,0 +1,11 @@ +# skill-doctor contract v1 schemas + +Copies of the producer's JSON Schemas (`src/skill_doctor/schemas/` in the +skill-doctor repo), taken 2026-11-20. `doctor-schema.test.ts` checks that every +field `lib/doctor.ts` reads is declared here and that the hand-written fixtures +use only declared keys, so a producer-side rename fails a test once these are +refreshed. Refresh them whenever the doctor's contract changes. + +fixes_v1 pins only `id`, `task_id` and `outcome` of a candidate; the other +candidate fields the board reads (`covers`, `patch_lines`, `title`, `gates`) +are not pinned by the producer yet. diff --git a/evalboard/lib/__tests__/fixtures/skill-doctor/schemas/fixes_v1.json b/evalboard/lib/__tests__/fixtures/skill-doctor/schemas/fixes_v1.json new file mode 100644 index 00000000..2d1a5322 --- /dev/null +++ b/evalboard/lib/__tests__/fixtures/skill-doctor/schemas/fixes_v1.json @@ -0,0 +1,57 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "skill-doctor/fixes_v1.json", + "title": "runs//skill-doctor/fixes-/fixes.json (nightly-fix emit; contract v1, partial)", + "description": "The Fix stage's outcome per candidate, the skipped rows, and pr_checks: the replay check of the open PR each row's diagnosis names as its fix (fixed_by_pr). Only the keys below are pinned; candidates carry more fields (gates, patch lines).", + "type": "object", + "required": ["schema_version", "run_id", "fix_mode", "candidates", "skipped", "cost_usd"], + "properties": { + "schema_version": {"const": 1}, + "run_id": {"type": "string"}, + "fix_mode": {"enum": ["off", "dryrun", "open"]}, + "base_sha": {"type": ["string", "null"]}, + "cost_usd": {"type": "number", "minimum": 0}, + "max_prs": {"type": ["integer", "null"], "description": "the cap on PRs OPENED by this run (rank order); not a selection cut"}, + "history_runs": {"type": "array", "items": {"type": "string"}, "description": "the earlier nights the memory read (newest first)"}, + "candidates": {"type": "array", "items": {"type": "object", "required": ["id", "task_id", "outcome"], + "properties": {"outcome": {"type": "string", "description": "pr-ready, needs-live-run, needs-windows-run (a deciding criterion runs Windows-only tooling on a windows-tagged task), held-overlap, body-lint-failed, no-budget (not tried: the run's spend cap was used up; it rolls to the next night), ..."}, + "rank": {"type": "integer", "minimum": 1, "description": "select's order: rows covered, confidence, pr-ready on an earlier night, stable diagnosis, task id"}, + "reused_from": {"$ref": "#/$defs/record", "description": "present when the patch of an earlier night was reused (rebased, model-free gates passed; no session, $0)"}, + "memory_note": {"type": "string", "description": "what the earlier nights said about this candidate"}, + "inputs": {"type": "object", "additionalProperties": {"type": ["string", "null"]}, "description": "git blob id per input file at the base (task yaml, its check files, the skill files the diagnosis names)"}, + "inputs_hash": {"type": "string"}, + "contested": {"type": "array", "items": {"type": "string"}, "description": "the workflow's collision list (\"#N open (author)\", \" merged (author)\", \"contested by #N\")"}, + "session_cap_usd": {"type": "number", "description": "the cap the candidate's session started with"}, + "diagnosis_flip": {"type": ["object", "null"], "description": "{night, from, to}: the layer/solvable changed since the previous night"}}}}, + "skipped": {"type": "array", "items": {"type": "object", "additionalProperties": false, "required": ["task_id", "reason"], + "properties": {"task_id": {"type": "string"}, "reason": {"type": "string"}, + "pr_url": {"type": "string", "pattern": "^https://github\\.com/UiPath/skills/pull/\\d+$", "description": "pr-open: the open doctor PR"}, + "skip_reason": {"enum": ["pr-open", "rejected-unchanged"], "description": "a memory skip ($0): a doctor PR with the fix is still open, or an earlier night rejected the fix and its inputs are unchanged"}, + "reused_from": {"$ref": "#/$defs/record"}}}}, + "pr_checks": {"type": "object", "additionalProperties": {"$ref": "#/$defs/pr_check"}} + }, + "$defs": { + "record": { + "type": "object", + "additionalProperties": false, + "required": ["run_id", "fixes_dir", "id", "outcome"], + "description": "the earlier night's fixes.json candidate a memory decision used: runs//skill-doctor//", + "properties": {"run_id": {"type": "string"}, "fixes_dir": {"type": "string", "pattern": "^fixes(-[0-9]+)?$"}, + "id": {"type": ["string", "null"]}, "outcome": {"type": ["string", "null"]}} + }, + "pr_check": { + "type": "object", + "additionalProperties": false, + "required": ["number", "url", "verdict", "evidence"], + "properties": { + "number": {"type": "integer", "minimum": 1}, + "url": {"type": "string", "pattern": "^https://github\\.com/UiPath/skills/pull/\\d+$"}, + "head_sha": {"type": ["string", "null"], "pattern": "^[0-9a-f]{40}$", "description": "the PR head the check applied (refs/pull//head as fetched)"}, + "verdict": {"enum": ["verified", "not_verified", "untested"]}, + "evidence": {"type": "string", "maxLength": 600}, + "claim": {"enum": ["fixes", "may_help"]}, + "reason": {"enum": ["needs-windows-run"], "description": "why an untested check could not be replayed, when the board should say so"} + } + } + } +} diff --git a/evalboard/lib/__tests__/fixtures/skill-doctor/schemas/status_v1.json b/evalboard/lib/__tests__/fixtures/skill-doctor/schemas/status_v1.json new file mode 100644 index 00000000..201e64be --- /dev/null +++ b/evalboard/lib/__tests__/fixtures/skill-doctor/schemas/status_v1.json @@ -0,0 +1,48 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "skill-doctor/status_v1.json", + "title": "runs//skill-doctor/status.json (contract v1)", + "description": "Pointer and lock for one nightly run's doctor output. Written first as a create-if-absent lock (state running), last on success. triage_file names the immutable triage-.json next to it.", + "type": "object", + "additionalProperties": false, + "required": ["schema_version", "state", "reason", "run_id", "harness", "model", "history_base_run", "run_json_etag", + "started_at", "finished_at", "doctor_version", "build_id", "build_url", "cost_usd", "rows_failed", + "rows_diagnosed", "triage_file"], + "properties": { + "schema_version": {"const": 1}, + "state": {"enum": ["queued", "running", "done", "failed", "skipped", "retracted"]}, + "reason": {"type": "string", "maxLength": 500}, + "run_id": {"type": "string", "pattern": "^[A-Za-z0-9][A-Za-z0-9._-]{0,99}$"}, + "harness": {"type": ["string", "null"]}, + "model": {"type": ["string", "null"]}, + "adhoc": {"type": "boolean"}, + "triage_only": {"type": "boolean"}, + "history_base_run": {"type": ["string", "null"]}, + "run_json_etag": {"type": ["string", "null"]}, + "started_at": {"type": "string"}, + "finished_at": {"type": ["string", "null"]}, + "updated_at": {"type": "string"}, + "doctor_version": {"type": "string"}, + "build_id": {"type": ["string", "null"]}, + "build_url": {"type": ["string", "null"]}, + "cost_usd": {"type": "number", "minimum": 0}, + "cost_complete": {"type": "boolean", "description": "false when a session's cost is unknown (timed out / killed): cost_usd counts it as 0, reason says which group"}, + "rows_failed": {"type": ["integer", "null"], "minimum": 0}, + "rows_diagnosed": {"type": "integer", "minimum": 0}, + "triage_file": {"type": ["string", "null"], "pattern": "^triage-[A-Za-z0-9._-]+\\.json$"}, + "trigger": {"type": ["string", "null"], "pattern": "^https://dev\\.azure\\.com/[A-Za-z0-9-]+/[A-Za-z0-9._%-]+/_build/results\\?buildId=[0-9]+$", + "description": "the nightly build that queued this run (the sourceBuild parameter); null when queued by hand"}, + "fixes_dir": {"type": ["string", "null"], "pattern": "^fixes(-[0-9]+)?$", + "description": "the Fix stage's create-only dir next to this file (fixes.json, .patch, .md); set by FixEmit"}, + "fixes_dirs": {"type": "array", "items": {"type": "string", "pattern": "^fixes(-[0-9]+)?$"}, + "description": "every Fix stage's dir of this run, oldest first (fixes_dir is the newest): the memory of the next nights reads them all"}, + "fixes": {"type": "object", "additionalProperties": false, "required": ["fix_mode", "cost_usd", "counts"], + "description": "FixEmit's digest of fixes.json: candidates per outcome (pr-ready, needs-live-run, held-overlap, ...)", + "properties": { + "fix_mode": {"enum": ["dryrun", "open"]}, + "cost_usd": {"type": "number", "minimum": 0}, + "total_cap_usd": {"type": ["number", "null"], "minimum": 0}, + "counts": {"type": "object", "additionalProperties": {"type": "integer", "minimum": 0}} + }} + } +} diff --git a/evalboard/lib/__tests__/fixtures/skill-doctor/schemas/triage_v1.json b/evalboard/lib/__tests__/fixtures/skill-doctor/schemas/triage_v1.json new file mode 100644 index 00000000..7cffd7ee --- /dev/null +++ b/evalboard/lib/__tests__/fixtures/skill-doctor/schemas/triage_v1.json @@ -0,0 +1,145 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "skill-doctor/triage_v1.json", + "title": "runs//skill-doctor/triage->.json (contract v1)", + "description": "One nightly run's failed rows: the $0 triage (class, owners, recurrence), the optional model diagnosis (layer ... passed_before), fix scouting (existing_prs, doctor_pr) and cross-task actions. All text is scrubbed (emails, tenant ids, token shapes). Immutable once written.", + "type": "object", + "additionalProperties": false, + "required": ["schema_version", "run", "summary", "tasks", "actions", "disclaimer"], + "properties": { + "schema_version": {"const": 1}, + "disclaimer": {"type": "string"}, + "run": { + "type": "object", + "additionalProperties": false, + "required": ["run_id", "harness", "model", "skills_git_commit", "cli_version", "tasks_run", "rows_executed", "adhoc", + "history_base_run", "run_json_etag", "build_id", "doctor_version", "generated_at", "mode", "mode_reason"], + "properties": { + "run_id": {"type": "string"}, + "harness": {"type": ["string", "null"]}, + "model": {"type": ["string", "null"]}, + "skills_git_commit": {"type": ["string", "null"]}, + "cli_version": {"type": ["string", "null"]}, + "tasks_run": {"type": ["integer", "null"]}, + "rows_executed": {"type": ["integer", "null"]}, + "adhoc": {"type": "boolean"}, + "history_base_run": {"type": ["string", "null"]}, + "run_json_etag": {"type": ["string", "null"]}, + "build_id": {"type": ["string", "null"]}, + "build_url": {"type": ["string", "null"]}, + "doctor_version": {"type": "string"}, + "generated_at": {"type": "string"}, + "mode": {"enum": ["triage-only", "diagnosed"]}, + "mode_reason": {"type": "string"}, + "cost_usd": {"type": "number", "minimum": 0}, + "pr_data": {"type": "string"} + } + }, + "summary": { + "type": "object", + "additionalProperties": false, + "required": ["rows_failed", "rows_diagnosed", "by_class", "by_layer", "vs_history", "env_groups"], + "properties": { + "rows_failed": {"type": "integer", "minimum": 0}, + "rows_diagnosed": {"type": "integer", "minimum": 0}, + "rows_diagnosis_rejected": {"type": "integer", "minimum": 0}, + "by_class": {"type": "object", "additionalProperties": {"type": "integer"}}, + "by_layer": {"type": "object", "additionalProperties": {"type": "integer"}}, + "vs_history": { + "type": "object", + "additionalProperties": false, + "required": ["new", "persisting", "fixed", "base_run"], + "properties": { + "new": {"type": "integer", "minimum": 0}, + "persisting": {"type": "integer", "minimum": 0}, + "fixed": {"type": "integer", "minimum": 0}, + "fixed_tasks": {"type": "array", "items": {"type": "string"}}, + "base_run": {"type": ["string", "null"]} + } + }, + "env_groups": { + "type": "array", + "items": { + "type": "object", + "additionalProperties": false, + "required": ["cause", "count", "task_ids"], + "properties": { + "cause": {"type": "string"}, + "count": {"type": "integer", "minimum": 1}, + "task_ids": {"type": "array", "items": {"type": "string"}}, + "detail": {"type": "string"}, + "fix": {"type": "string"} + } + } + } + } + }, + "tasks": {"type": "array", "items": {"$ref": "#/$defs/task"}}, + "actions": { + "type": "array", + "items": { + "type": "object", + "additionalProperties": false, + "required": ["kind", "count", "text", "task_ids"], + "properties": { + "kind": {"enum": ["env", "open-pr", "upstream", "layer"]}, + "count": {"type": "integer", "minimum": 1}, + "text": {"type": "string"}, + "task_ids": {"type": "array", "items": {"type": "string"}} + } + } + } + }, + "$defs": { + "fixed_by_pr": {"type": ["object", "null"], "additionalProperties": false, "required": ["number", "verdict", "basis"], "description": "The diagnosis's judgement of the open PR that best fixes this row (an opinion, from its diff); the Fix stage's replay check of it is fixes.json pr_checks[task_id]", "properties": {"number": {"type": "integer", "minimum": 1}, "verdict": {"enum": ["fixes", "may_help", "unrelated"]}, "basis": {"type": "string", "maxLength": 300}}}, + "pr": { + "type": "object", + "additionalProperties": false, + "required": ["url", "title", "state"], + "properties": { + "url": {"type": "string", "pattern": "^https://github\\.com/UiPath/skills/pull/\\d+$"}, + "title": {"type": "string", "maxLength": 300}, + "state": {"enum": ["open", "merged", "merged-after-run", "unknown", "closed"]} + } + }, + "task": { + "type": "object", + "additionalProperties": false, + "required": ["task_id", "skill", "class", "family", "state", "nights_seen", "owners", "run_status", "criterion", "error", + "diagnosed", "layer", "confidence", "why", "evidence", "solvable", "fix", "related", "passed_before", + "existing_prs", "doctor_pr"], + "properties": { + "task_id": {"type": "string"}, + "skill": {"type": "string"}, + "class": {"enum": ["env", "harness-error", "turn-cap", "timeout", "failure", "known-upstream", "contested"]}, + "family": {"type": "string"}, + "state": {"enum": ["new", "changed", "known"]}, + "nights_seen": {"type": "integer", "minimum": 1}, + "owners": {"type": "array", "items": {"type": "string"}}, + "run_status": {"type": ["string", "null"]}, + "reps_failed": {"type": "integer", "minimum": 0}, + "criterion": {"type": ["string", "null"]}, + "error": {"type": ["string", "null"]}, + "contested": {"type": ["array", "null"], "items": {"type": "string"}}, + "known_upstream": {"type": ["string", "null"]}, + "recurrence": {"type": ["object", "null"]}, + "previous_nights": {"type": ["object", "null"]}, + "env_group": {"type": ["string", "null"]}, + "diagnosed": {"type": "boolean"}, + "diagnosis_skipped": {"type": ["string", "null"]}, + "layer": {"enum": ["agent", "skill", "grader", "task", "env", "upstream", "flaky", "undiagnosed"]}, + "confidence": {"type": ["string", "null"]}, + "why": {"type": ["string", "null"]}, + "evidence": {"type": ["string", "null"]}, + "solvable": {"type": ["string", "null"]}, + "fix": {"type": ["string", "null"]}, + "related": {"type": ["string", "null"]}, + "passed_before": {"type": ["string", "null"]}, + "existing_prs": {"type": ["array", "null"], "items": {"$ref": "#/$defs/pr"}}, + "related_skill_prs": {"type": ["array", "null"], "items": {"$ref": "#/$defs/pr"}}, + "doctor_pr": {"type": ["string", "null"], "pattern": "^https://github\\.com/UiPath/skills/pull/\\d+$"}, + "fixed_by_pr": {"$ref": "#/$defs/fixed_by_pr"} + } + } + } +} diff --git a/evalboard/lib/doctor.ts b/evalboard/lib/doctor.ts index 2b431250..b8a5cbfa 100644 --- a/evalboard/lib/doctor.ts +++ b/evalboard/lib/doctor.ts @@ -23,7 +23,7 @@ const PATCH_ID_RE = /^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$/; // At most this many patches per run are fetched, linked and served. const MAX_PATCHES = 10; -const DOCTOR_STATES = [ +export const DOCTOR_STATES = [ "queued", "running", "done", @@ -102,6 +102,9 @@ export interface DoctorAction { } export interface DoctorTriage { + // The run the triage describes (run.run_id); readRunDoctor drops a triage + // that names another run. + runId: string | null; // Rows the nightly executed (null when the run did not say). rowsRun: number | null; rowsFailed: number; @@ -127,6 +130,9 @@ export interface DoctorFix { } export interface DoctorFixes { + // The run fixes.json describes (run_id); readRunDoctor drops one that + // names another run. + runId: string | null; fixMode: string | null; // By task id (own keys; read with own()). null: an entry that did not // validate. @@ -142,6 +148,9 @@ export interface RunDoctor { // PRs the doctor opened (`/opened.json`): task id → PR URL, // allowlisted URLs only. opened?: Record; + // A doctor file could not be fetched (a storage error, not a 404): the + // parts that are missing may exist. + fetchFailed?: boolean; } // ---------- coercion helpers ---------- @@ -154,8 +163,11 @@ function isObj(v: unknown): v is Obj { function str(v: unknown): string | null { return typeof v === "string" ? v : null; } +function isCount(v: unknown): v is number { + return typeof v === "number" && Number.isInteger(v) && v >= 0; +} function int(v: unknown, dflt = 0): number { - return typeof v === "number" && Number.isInteger(v) && v >= 0 ? v : dflt; + return isCount(v) ? v : dflt; } function strList(v: unknown): string[] { return Array.isArray(v) ? v.filter((x): x is string => typeof x === "string") : []; @@ -268,16 +280,16 @@ export function parseTriage(raw: string | null | undefined): DoctorTriage | null if (!isObj(j) || j.schema_version !== 1) return null; if (!isObj(j.summary) || !Array.isArray(j.tasks)) return null; const s = j.summary; + // The header shows these counts: a missing or malformed one is contract + // drift, so the triage does not parse (and readRunDoctor logs it) rather + // than reading as a real 0. + if (!isCount(s.rows_failed) || !isCount(s.rows_diagnosed)) return null; const run = isObj(j.run) ? j.run : {}; return { - rowsRun: - typeof run.rows_executed === "number" && - Number.isInteger(run.rows_executed) && - run.rows_executed >= 0 - ? run.rows_executed - : null, - rowsFailed: int(s.rows_failed), - rowsDiagnosed: int(s.rows_diagnosed), + runId: str(run.run_id), + rowsRun: isCount(run.rows_executed) ? run.rows_executed : null, + rowsFailed: s.rows_failed, + rowsDiagnosed: s.rows_diagnosed, tasks: j.tasks.map(parseTask).filter((t): t is DoctorTask => t !== null), actions: Array.isArray(j.actions) ? j.actions.flatMap((a): DoctorAction[] => @@ -296,9 +308,7 @@ function parseGates(v: unknown): DoctorGates { const replay = isObj(v.replay) ? str(v.replay.status) : null; const m = isObj(v.mutants) ? v.mutants : null; const mutants = - m && typeof m.n === "number" && typeof m.killed === "number" - ? { n: int(m.n), killed: int(m.killed) } - : null; + m && isCount(m.n) && isCount(m.killed) ? { n: m.n, killed: m.killed } : null; return { replay, mutants, review: str(v.review) }; } @@ -306,6 +316,7 @@ export function parseFixes(raw: string | null | undefined): DoctorFixes | null { const j = parseJson(raw); if (!isObj(j) || j.schema_version !== 1 || !Array.isArray(j.candidates)) return null; return { + runId: str(j.run_id), fixMode: str(j.fix_mode), prChecks: parsePrChecks(j.pr_checks), candidates: j.candidates.flatMap((c): DoctorFix[] => { diff --git a/evalboard/lib/ids.ts b/evalboard/lib/ids.ts index c8380c9e..5272ee38 100644 --- a/evalboard/lib/ids.ts +++ b/evalboard/lib/ids.ts @@ -3,8 +3,10 @@ // re-exports them for its existing importers. // Run / task IDs get reflected into filesystem paths and blob prefixes, so -// reject anything outside a narrow whitelist before any side effect. +// reject anything outside a narrow whitelist before any side effect. A run id +// of only dots ("." / "..") would name the runs dir or its parent. const ID_RE = /^[\w.-]+$/; +const DOTS_RE = /^\.+$/; // Task IDs from dataset-expanded tasks look like "sentiment-classification/r3" // (suite/row). Each segment must still pass the narrow ID_RE — only the slash // separator between segments is additionally allowed. @@ -15,7 +17,8 @@ export function isValidId(id: unknown): id is string { typeof id === "string" && id.length > 0 && id.length < 128 && - ID_RE.test(id) + ID_RE.test(id) && + !DOTS_RE.test(id) ); } @@ -25,6 +28,6 @@ export function isValidTaskId(id: unknown): id is string { id.length > 0 && id.length < 256 && TASK_ID_RE.test(id) && - !id.split("/").some((s) => s === "." || s === "..") + !id.split("/").some((s) => DOTS_RE.test(s)) ); } diff --git a/evalboard/lib/runs.ts b/evalboard/lib/runs.ts index 45d372f3..360edfe3 100644 --- a/evalboard/lib/runs.ts +++ b/evalboard/lib/runs.ts @@ -2701,20 +2701,30 @@ const DOCTOR_STATUS_REFRESH_MS = 60_000; // The run's skill-doctor report, or null when there is none, it does not parse, // or its schema_version is not 1. This is the one place that decides which -// files a status implies: status.json → (when done) the triage file it names, -// and /fixes.json → the patches fixes.json lists. Never throws: a +// files a status implies: a done status.json → the triage file it names, and +// /fixes.json → the patches fixes.json lists (any other state, +// retracted included, implies no other file). By default it never throws: a // doctor file problem must not take the run page down, so a fetch error -// degrades to "no report" (or to the parts that did load). +// degrades to "no report" (or to the parts that did load) and sets +// fetchFailed. `strict` (for /api/file) lets a storage error other than a 404 +// propagate instead, so a click on a patch link can tell "storage down" from +// "not listed"; it then fetches only the one patch `strict` names (no +// opened.json, no other patch). export async function readRunDoctor( runId: string, source: Source = DEFAULT_SOURCE, + { strict }: { strict?: { patch: string } } = {}, ): Promise { const dir = runsDirFor(RUNS_DIR, source); const base = path.join(dir, runId, DOCTOR_DIR); + let fetchFailed = false; + // ensureRunDoctorFile already swallows a 404 (an absent file). const get = async (rel: string, refreshAfterMs?: number) => { try { await ensureRunDoctorFile(source.container, runId, rel, dir, refreshAfterMs); } catch (err) { + if (strict) throw err; + fetchFailed = true; console.error(`skill-doctor: fetching ${rel} failed for run ${runId}:`, err); } }; @@ -2735,31 +2745,45 @@ export async function readRunDoctor( await get("status.json", DOCTOR_STATUS_REFRESH_MS); const status = await load("status.json", parseDoctorStatus); if (!status) return null; - const triageRel = - status.state === "done" && status.triageFile ? status.triageFile : null; - const fixesRel = status.fixesDir ? `${status.fixesDir}/fixes.json` : null; + const done = status.state === "done"; + const fdir = done ? status.fixesDir : null; + const triageRel = done && status.triageFile ? status.triageFile : null; + const fixesRel = fdir ? `${fdir}/fixes.json` : null; await Promise.all([ triageRel && get(triageRel), fixesRel && get(fixesRel), ]); - const [triage, fixes] = await Promise.all([ + const [parsedTriage, parsedFixes] = await Promise.all([ triageRel ? load(triageRel, parseTriage) : null, fixesRel ? load(fixesRel, parseFixes) : null, ]); + // A triage or fixes.json that names another run (a copy under the + // wrong prefix) must not replace this run's Findings or list patches. + const ofThisRun = (v: T | null, rel: string | null) => { + if (!v || v.runId === runId) return v; + console.warn( + `skill-doctor: ${rel} names run ${JSON.stringify(v.runId)}, not ${runId}; ignored`, + ); + return null; + }; + const triage = ofThisRun(parsedTriage, triageRel); + const fixes = ofThisRun(parsedFixes, fixesRel); // Cached patches are not fetched again (downloadBlob skips them). // opened.json is written by the doctor's open step after the fixes, so // it may appear or change later: it is refetched like status.json. let opened: Record = {}; - if (status.fixesDir) { - const fdir = status.fixesDir; + if (fdir && strict) { + if (patchIds(fixes).has(strict.patch)) await get(`${fdir}/${strict.patch}.patch`); + } else if (fdir) { await Promise.all([ ...[...patchIds(fixes)].map((id) => get(`${fdir}/${id}.patch`)), get(`${fdir}/opened.json`, DOCTOR_STATUS_REFRESH_MS), ]); opened = (await load(`${fdir}/opened.json`, parseOpened)) ?? {}; } - return { status, triage, fixes, opened }; + return { status, triage, fixes, opened, fetchFailed }; } catch (err) { + if (strict) throw err; console.error(`readRunDoctor failed for run ${runId}:`, err); return null; } @@ -3029,6 +3053,11 @@ export async function resolveSafePath( source: Source = DEFAULT_SOURCE, ): Promise { if (!isValidId(runId)) return null; + // Dispatch on the normalized path, so "./skill-doctor/…" or + // "activation/../skill-doctor/…" cannot skip the skill-doctor gate below. + relPath = path.posix.normalize(relPath); + if (relPath === ".." || relPath.startsWith("../") || path.posix.isAbsolute(relPath)) + return null; const dir = runsDirFor(RUNS_DIR, source); // Artifact URLs embed the variant + task subdirs in relPath // (`//artifacts/...`) — extract them so the narrow @@ -3045,6 +3074,8 @@ export async function resolveSafePath( // one fetch path: it loads status and fixes.json when they are not // cached and fetches exactly those patches, so a cold cache or a direct // link works and a made-up name costs no blob call. Off with the panel. + // Strict: a storage error propagates (the route answers 5xx), so it + // does not read as "forbidden" or "not found". if ( !taskAnalysisEnabled( isInternal, @@ -3055,7 +3086,7 @@ export async function resolveSafePath( return null; const patch = patchIdFromPath(parts.slice(1).join("/")); if (!patch) return null; - const doctor = await readRunDoctor(runId, source); + const doctor = await readRunDoctor(runId, source, { strict: { patch: patch.id } }); if ( !doctor || doctor.status.fixesDir !== patch.fixesDir || @@ -3094,10 +3125,10 @@ export async function resolveSafePath( // Delete a run's locally-cached blob copy under `root` so the next view // re-downloads it from storage. `force: true` makes a never-cached run a -// harmless no-op. Returns false (deleting nothing) for an unsafe id — note -// isValidId still admits "." and ".." (dots are word-ish), so require the -// resolved target to be a strict child of `root` before rm can run, or a "." -// id would nuke the cache root and ".." its parent. +// harmless no-op. Returns false (deleting nothing) for an unsafe id. isValidId +// rejects "." and "..", and the resolved target must still be a strict child +// of `root` before rm can run (a "." id would nuke the cache root, ".." its +// parent), as a second guard. export async function clearRunCacheDir( root: string, id: string,