diff --git a/evalboard/README.md b/evalboard/README.md index 7e1f1b85..5f027d4c 100644 --- a/evalboard/README.md +++ b/evalboard/README.md @@ -207,5 +207,15 @@ belongs to it. and then every 4 minutes (`lib/warm.ts`, started from `instrumentation.ts`), so the per-run cache is rebuilt by those requests rather than by a visitor. `EVALBOARD_WARM_INTERVAL_SECONDS` sets the interval; `0` turns it off. +- Task Failure Analysis (internal edition, skills source): the run page reads + the skill-doctor report under `/skill-doctor/` (`lib/doctor.ts`, + `readRunDoctor`). `status.json` is refetched after 60 s; the triage file it + names, `/fixes.json` and up to 10 patches are cached for good. + `/api/file` serves only the patches fixes.json lists, loading them through + the same reader on a cold cache. The report is a collapsed block + under the AI summary, laid out like the doctor's own page (stats, failures by + layer, Fix first, per-skill failure cards). When the report is done and + `analysis.md` has a `## Findings` line, the AI summary stops there; other + states keep the AI summary and show the doctor's state in the block header. `EVALBOARD_TASK_ANALYSIS=off` turns it off. - Pass rows render green (`bg-green-50 text-green-700`), failures render red (`bg-red-50 text-red-700`), on a white background. diff --git a/evalboard/app/runs/[id]/__tests__/task-failure-analysis.test.tsx b/evalboard/app/runs/[id]/__tests__/task-failure-analysis.test.tsx new file mode 100644 index 00000000..2fe6e57f --- /dev/null +++ b/evalboard/app/runs/[id]/__tests__/task-failure-analysis.test.tsx @@ -0,0 +1,675 @@ +import { readFileSync } from "node:fs"; +import path from "node:path"; +import { describe, expect, test } from "vitest"; +import { fireEvent, render, screen, within } from "@testing-library/react"; +import { + countFixPrs, + digest, + groupBySkill, + existingPrFix, + TaskFailureAnalysis, +} from "../task-failure-analysis"; +import { + parseDoctorStatus, + parseFixes, + parseTriage, + type RunDoctor, +} from "@/lib/doctor"; + +const FIX = path.join(__dirname, "..", "..", "..", "..", "lib", "__tests__", "fixtures", "skill-doctor"); +const read = (p: string) => readFileSync(path.join(FIX, p), "utf-8"); +const RUN = "2099-01-03_04-23-30"; +const OPENED = "https://github.com/UiPath/skills/pull/300"; + +function status(patch: Record = {}) { + return parseDoctorStatus(JSON.stringify({ ...JSON.parse(read("status.json")), ...patch }))!; +} + +// The fixture: example-t1 has a pr-ready fix with no PR opened yet; example-t2 +// has a needs-live-run candidate, and the diagnosis says open PR #126 fixes it +// (fixed_by_pr "fixes"), which the Fix stage verified (pr_checks). +function doctor(patch: Record = {}): RunDoctor { + const s = status(patch); + return { + status: s, + triage: s.state === "done" ? parseTriage(read("triage-1000001.json")) : null, + fixes: parseFixes(read("fixes-1000001/fixes.json")), + opened: {}, + }; +} + +function withOpened(d: RunDoctor = doctor()): RunDoctor { + return { ...d, opened: { "example-t1": OPENED } }; +} + +function renderBlock(d: RunDoctor = doctor(), sourceId = "skills") { + return render(); +} + +const block = () => screen.getByRole("region", { name: "Task failure analysis" }); +const header = () => within(block()).getAllByRole("button")[0]; +const openBlock = () => fireEvent.click(header()); +const skillButton = (skill: string) => + screen.getByRole("button", { name: new RegExp(`^▸?▾? ?${skill} `) }); +const openSkill = (skill: string) => fireEvent.click(skillButton(skill)); +const card = (taskId: string) => + screen.getAllByTestId("task-card").find((c) => c.textContent!.includes(taskId))!; + +describe("digest", () => { + test("done: errors and linked PRs with fix; a prepared fix does not count", () => { + // Only #126 (proposed) is linked; example-t1's fix is not opened yet. + expect(digest(doctor())).toBe("5 errors · 1 PR with fix"); + expect(digest(withOpened())).toBe("5 errors · 2 PRs with fix"); + }); + test("done without a triage: a fetch error is not blamed on the contract", () => { + const d = { ...doctor(), triage: null }; + expect(digest(d)).toBe("report done, but its triage file is missing or did not parse"); + expect(digest({ ...d, fetchFailed: true })).toBe( + "report done, but its triage file could not be fetched", + ); + }); + test("no linked PR: the errors only", () => { + const d = doctor(); + d.triage!.tasks[2].fixedByPr = null; + expect(digest(d)).toBe("5 errors"); + }); + test("not done: the state", () => { + expect(digest(doctor({ state: "running" }))).toMatch(/^running since /); + expect(digest(doctor({ state: "queued", started_at: null }))).toBe("queued"); + expect(digest(doctor({ state: "failed", reason: "diagnose timed out" }))).toBe( + "failed: diagnose timed out", + ); + expect(digest(doctor({ state: "skipped", reason: "ad-hoc run" }))).toBe("skipped: ad-hoc run"); + expect(digest(doctor({ state: "retracted" }))).toBe("the report for this run was retracted"); + expect(digest({ ...doctor(), triage: null })).toMatch(/triage file is missing/); + }); +}); + +describe("TaskFailureAnalysis block", () => { + test("collapsed by default, like the AI summary: header and digest only", () => { + renderBlock(); + expect(header()).toHaveAttribute("aria-expanded", "false"); + expect(header()).toHaveTextContent("Task failure analysis"); + expect(header()).toHaveTextContent("5 errors · 1 PR with fix"); + expect(screen.queryByText("Fix first")).toBeNull(); + expect(screen.queryByText("Skill failures")).toBeNull(); + }); + + test("not done: no body, the state and the build link", () => { + renderBlock(doctor({ state: "failed", reason: "boom" })); + expect(within(block()).queryByRole("button")).toBeNull(); + expect(block()).toHaveTextContent("failed: boom"); + expect(screen.getByRole("link", { name: "build" })).toBeInTheDocument(); + }); + + test("body order: stats, layer bar, Fix first, Skill failures", () => { + renderBlock(); + openBlock(); + const text = block().textContent!; + const at = (s: string) => text.indexOf(s); + expect(at("rows run")).toBeGreaterThan(-1); + expect(at("rows run")).toBeLessThan(at("Failures by layer")); + expect(at("Failures by layer")).toBeLessThan(at("Fix first")); + expect(at("Fix first")).toBeLessThan(at("Skill failures")); + expect(screen.queryByRole("table")).toBeNull(); + }); + + test("no costs anywhere in the panel", () => { + renderBlock(withOpened()); + openBlock(); + fireEvent.click(screen.getByRole("button", { name: "Expand all" })); + for (const b of within(block()).getAllByTestId("env-group")) { + fireEvent.click(within(b).getByRole("button")); + } + expect(block().textContent).not.toContain("$"); + renderBlock(doctor({ state: "running" })); + expect(document.body.textContent).not.toContain("$"); + }); + + test("stats as on the doctor page", () => { + renderBlock(); + openBlock(); + const stats = screen.getByTestId("doctor-stats"); + expect(stats).toHaveTextContent("10rows run"); + expect(stats).toHaveTextContent("5failed rows"); + expect(stats).toHaveTextContent("3environment"); + expect(stats).toHaveTextContent("2fixable in the repo"); + expect(stats).toHaveTextContent("0agent mistakes"); + expect(screen.getByRole("img", { name: "Failures by layer" }).children).toHaveLength(3); + }); + + test("Fix first: env and layer actions only, allowlisted links, a PRs-with-fix badge", () => { + const d = withOpened(); + d.triage!.actions[1].text += + " See https://github.com/UiPath/skills/pull/123, javascript:alert(1) and https://github.com.example.invalid/x."; + renderBlock(d); + openBlock(); + const items = within(screen.getByRole("region", { name: "Fix first" })).getAllByRole("listitem"); + // The fixture's open-pr action is left out. + expect(items).toHaveLength(2); + expect(block()).not.toHaveTextContent("touches the files"); + expect(items[0]).toHaveTextContent(/^3 · Environment/); + expect(within(items[1]).getAllByRole("link").map((a) => a.getAttribute("href"))).toEqual([ + "https://github.com/UiPath/skills/pull/123", + ]); + expect(items[1]).toHaveTextContent("javascript:alert(1)"); + expect(within(items[1]).getByTestId("action-ready")).toHaveTextContent("1 PR with fix"); + expect(within(items[0]).queryByTestId("action-ready")).toBeNull(); + }); + + test("a prepared fix gives no badge", () => { + renderBlock(); + openBlock(); + expect(screen.queryByTestId("action-ready")).toBeNull(); + }); + + test("no env or layer actions: no Fix first section", () => { + const d = doctor(); + d.triage!.actions = []; + renderBlock(d); + openBlock(); + expect(screen.queryByRole("region", { name: "Fix first" })).toBeNull(); + }); + + test("skills: grouped, all collapsed, with linked-PR counts and layer dots", () => { + renderBlock(); + openBlock(); + const a = skillButton("uipath-example"); + const b = skillButton("uipath-example-two"); + expect(a).toHaveAttribute("aria-expanded", "false"); + expect(b).toHaveAttribute("aria-expanded", "false"); + // example-t1's fix is only prepared: it does not count. + expect(a).toHaveTextContent("2 failed"); + expect(a).not.toHaveTextContent("with fix"); + expect(a).toHaveTextContent("1 environment"); + expect(a).toHaveTextContent("1 grader"); + expect(b).toHaveTextContent("3 failed · 1 PR with fix"); + expect(screen.queryAllByTestId("task-card")).toHaveLength(0); + }); + + test("env tasks collapse into one block per pre-run check", () => { + renderBlock(); + openBlock(); + openSkill("uipath-example-two"); + const groups = screen.getAllByTestId("env-group"); + expect(groups).toHaveLength(1); + const btn = within(groups[0]).getByRole("button"); + expect(btn).toHaveTextContent("2 tasks stopped at the pre-run check: connection example-conn"); + fireEvent.click(btn); + expect(within(groups[0]).getByText(/Reauthorize the connection/)).toBeInTheDocument(); + expect(within(groups[0]).getAllByRole("link").map((l) => l.textContent)).toEqual([ + "example-e2", + "example-e3", + ]); + expect(screen.getAllByTestId("task-card")).toHaveLength(1); + }); + + test("a card: task link, pills, diagnosis, dl fields; no file-touching PRs, no owner", () => { + renderBlock(); + openBlock(); + openSkill("uipath-example"); + const c = card("example-t1"); + expect(within(c).getByRole("link", { name: "example-t1" })).toHaveAttribute( + "href", + `/runs/${RUN}/example-t1`, + ); + for (const pill of ["Grader", "Fixable: grader", "failure", "high confidence"]) { + expect(within(c).getByText(pill)).toBeInTheDocument(); + } + expect(c).toHaveTextContent("The grader expects a literal"); + expect(within(c).getAllByRole("term").map((d) => d.textContent)).toEqual([ + "Fix", + "Failing check", + "Evidence", + "Passed before", + ]); + expect(within(c).getByText("literal").tagName).toBe("CODE"); + // #123 (open on these files) and #125 (related skill PR) are not shown. + expect(c).not.toHaveTextContent("#123"); + expect(c).not.toHaveTextContent("#125"); + expect(c).not.toHaveTextContent("Example Owner"); + expect(within(c).queryByTestId("proposed-pr")).toBeNull(); + }); + + test("a pr-ready fix with no PR opened: plain text, patch and gates, no link", () => { + renderBlock(); + openBlock(); + openSkill("uipath-example"); + const c = card("example-t1"); + expect(within(c).queryByTestId("pr-with-fix")).toBeNull(); + const row = within(c).getByTestId("fix-prepared"); + expect(row).toHaveTextContent("Fix prepared, PR not opened yet"); + expect(row).toHaveTextContent("replay pass · mutants 9/10 · review SEND"); + expect(within(row).getAllByRole("link")).toHaveLength(1); + const patch = within(row).getByRole("link", { name: "+3/-1" }); + const href = new URL(patch.getAttribute("href")!, "http://x"); + expect(href.pathname).toBe("/api/file"); + expect(href.searchParams.get("path")).toBe( + "skill-doctor/fixes-1000001/example-grader-fix.patch", + ); + }); + + test("an opened PR (opened.json): 'PR with fix: #N ↗' with the gates", () => { + renderBlock(withOpened()); + openBlock(); + openSkill("uipath-example"); + const c = card("example-t1"); + const row = within(c).getByTestId("pr-with-fix"); + expect(within(row).getByRole("link", { name: "PR with fix: #300 ↗" })).toHaveAttribute( + "href", + OPENED, + ); + expect(row).toHaveTextContent("replay pass · mutants 9/10 · review SEND"); + expect(within(c).queryByTestId("fix-prepared")).toBeNull(); + }); + + test("a doctor PR from the triage counts the same way", () => { + const d = doctor(); + d.triage!.tasks[1].doctorPr = "https://github.com/UiPath/skills/pull/200"; + expect(digest(d)).toBe("5 errors · 2 PRs with fix"); + renderBlock(d); + openBlock(); + openSkill("uipath-example"); + expect( + within(card("example-t1")).getByRole("link", { name: "PR with fix: #200 ↗" }), + ).toBeInTheDocument(); + }); + + test("a verified third-party PR: a Proposed PR row with the evidence", () => { + renderBlock(); + openBlock(); + openSkill("uipath-example-two"); + const c = card("example-t2"); + const rows = within(c).getAllByTestId("proposed-pr"); + expect(rows).toHaveLength(1); + expect(rows[0]).toHaveTextContent("Proposed PR#126 docs(example-two): document the current flagopen"); + expect(rows[0]).toHaveTextContent("run_command flips to pass on both"); + expect(within(c).queryByTestId("maybe-pr")).toBeNull(); + expect(within(rows[0]).getByRole("link")).toHaveAttribute( + "href", + "https://github.com/UiPath/skills/pull/126", + ); + expect(within(c).getByTestId("fix-candidate")).toHaveTextContent("Fix candidate: needs-live-run"); + }); + + test.each([ + ["untested", undefined], + ["untested", "needs-windows-run"], + ["none", undefined], + ])("check %s (%s): 'Open PR may fix this (not verified)', not counted", (verdict, reason) => { + const d = doctor(); + if (verdict === "none") d.fixes!.prChecks = {}; + else + d.fixes!.prChecks["example-t2"] = { + ...d.fixes!.prChecks["example-t2"]!, + verdict: "untested", + evidence: "no stored run could replay it", + reason: reason ? "needs-windows-run" : null, + }; + expect(digest(d)).toBe("5 errors"); + renderBlock(d); + openBlock(); + openSkill("uipath-example-two"); + const c = card("example-t2"); + expect(within(c).queryByTestId("proposed-pr")).toBeNull(); + const row = within(c).getByTestId("maybe-pr"); + expect(row).toHaveTextContent("Open PR may fix this (not verified)"); + expect(within(row).getByRole("link", { name: /#126/ })).toHaveAttribute( + "href", + "https://github.com/UiPath/skills/pull/126", + ); + expect(row).toHaveTextContent("documents the current flag"); + if (verdict === "none") expect(row).not.toHaveTextContent("replay"); + else expect(row).toHaveTextContent("no stored run could replay it"); + if (reason) expect(row).toHaveTextContent("needs a Windows run"); + else expect(row).not.toHaveTextContent("Windows"); + }); + + test.each([ + ["not_verified check", (d: RunDoctor) => { d.fixes!.prChecks["example-t2"]!.verdict = "not_verified"; }], + ["may_help", (d: RunDoctor) => { d.triage!.tasks[2].fixedByPr!.verdict = "may_help"; }], + ["unrelated", (d: RunDoctor) => { d.triage!.tasks[2].fixedByPr!.verdict = "unrelated"; }], + ["merged PR", (d: RunDoctor) => { d.triage!.tasks[2].fixedByPr!.number = 127; }], + ["PR not in existing_prs", (d: RunDoctor) => { d.triage!.tasks[2].fixedByPr!.number = 999; }], + ["no fixed_by_pr", (d: RunDoctor) => { d.triage!.tasks[2].fixedByPr = null; }], + ])("%s: no PR row", (_name, mutate) => { + const d = doctor(); + mutate(d); + expect(digest(d)).toBe("5 errors"); + renderBlock(d); + openBlock(); + openSkill("uipath-example-two"); + const c = card("example-t2"); + expect(within(c).queryByTestId("proposed-pr")).toBeNull(); + expect(within(c).queryByTestId("maybe-pr")).toBeNull(); + }); + + test("a PR only cited in the fix text shows nothing (no prose parsing)", () => { + const d = doctor(); + // #123 is open on t1's files and named in its fix text, but there is no fixed_by_pr. + d.triage!.tasks[1].fix = "Open PR #123 (https://github.com/UiPath/skills/pull/123) already fixes this."; + renderBlock(d); + openBlock(); + openSkill("uipath-example"); + const c = card("example-t1"); + expect(within(c).queryByTestId("proposed-pr")).toBeNull(); + expect(within(c).queryByTestId("maybe-pr")).toBeNull(); + }); + + test.each([ + "looks like a relaxation, not a fix: the PR loosens the check", + "the PR drops the failing criterion run_command: needs review", + "stored failure on a criterion the yaml no longer has", + ])("untested evidence %j renders as plain text under 'may fix'", (evidence) => { + const d = doctor(); + d.fixes!.prChecks["example-t2"] = { + ...d.fixes!.prChecks["example-t2"]!, + verdict: "untested", + evidence, + headSha: "0123456789abcdef0123456789abcdef01234567", + }; + renderBlock(d); + openBlock(); + openSkill("uipath-example-two"); + const row = within(card("example-t2")).getByTestId("maybe-pr"); + expect(row).toHaveTextContent(evidence); + expect(row).toHaveTextContent("(at 0123456)"); + expect(within(row).getAllByRole("link")).toHaveLength(1); + }); + + test("a malformed check for the task (even a not_verified one) shows nothing", () => { + const d = doctor(); + // What parseFixes keeps for an entry that did not validate. + d.fixes!.prChecks["example-t2"] = null; + expect(existingPrFix(d, d.triage!.tasks[2])).toBeNull(); + const raw = JSON.parse(read("fixes-1000001/fixes.json")); + raw.pr_checks["example-t2"] = { number: 126, url: "https://evil.example/pull/126", verdict: "not_verified", evidence: "" }; + const d2 = { ...doctor(), fixes: parseFixes(JSON.stringify(raw)) }; + renderBlock(d2); + openBlock(); + openSkill("uipath-example-two"); + const c = card("example-t2"); + expect(within(c).queryByTestId("maybe-pr")).toBeNull(); + expect(within(c).queryByTestId("proposed-pr")).toBeNull(); + }); + + test("not-run candidates are hidden and do not lift the sort tier", () => { + const d = doctor(); + d.fixes!.candidates = d.fixes!.candidates.map((c) => ({ + ...c, + outcome: "not-run", + patchLines: null, + gates: { replay: null, mutants: null, review: null }, + })); + d.triage!.tasks[2].fixedByPr = null; + renderBlock(d); + openBlock(); + fireEvent.click(screen.getByRole("button", { name: "Expand all" })); + expect(block()).not.toHaveTextContent("not-run"); + expect(screen.queryAllByTestId("fix-candidate")).toHaveLength(0); + expect(screen.queryAllByTestId("fix-prepared")).toHaveLength(0); + const g = groupBySkill(d, null, ""); + // Nothing lifts either skill: more failed tasks first. + expect(g.map((x) => x.skill)).toEqual(["uipath-example-two", "uipath-example"]); + expect(g.every((x) => x.prepared === 0)).toBe(true); + }); + + test("a check for another PR number does not verify this one", () => { + const d = doctor(); + d.fixes!.prChecks["example-t2"]!.number = 4; + expect(existingPrFix(d, d.triage!.tasks[2])).toMatchObject({ verified: false, evidence: null }); + }); + + test("an unsafe patch id shows its size without a link", () => { + const d = doctor(); + d.fixes!.candidates[0].id = "../bad"; + renderBlock(d); + openBlock(); + openSkill("uipath-example"); + expect(within(card("example-t1")).getByText("+3/-1").closest("a")).toBeNull(); + }); + + test("a task id the task page would reject is plain text", () => { + const d = doctor(); + d.triage!.tasks[1].taskId = "../escape"; + renderBlock(d); + openBlock(); + openSkill("uipath-example"); + expect(screen.getByText("../escape").closest("a")).toBeNull(); + }); + + test("Expand all / Collapse all", () => { + renderBlock(); + openBlock(); + fireEvent.click(screen.getByRole("button", { name: "Expand all" })); + expect(skillButton("uipath-example")).toHaveAttribute("aria-expanded", "true"); + expect(skillButton("uipath-example-two")).toHaveAttribute("aria-expanded", "true"); + fireEvent.click(screen.getByRole("button", { name: "Collapse all" })); + expect(skillButton("uipath-example")).toHaveAttribute("aria-expanded", "false"); + }); + + test("layer chips filter, and the last one off shows all again", () => { + renderBlock(); + openBlock(); + const chips = within(screen.getByRole("toolbar", { name: "Filter failures" })); + fireEvent.click(chips.getByRole("button", { name: "Skill" })); + expect(chips.getByRole("button", { name: "Skill" })).toHaveAttribute("aria-pressed", "true"); + expect(chips.getByRole("button", { name: "Grader" })).toHaveAttribute("aria-pressed", "false"); + expect(screen.queryByRole("button", { name: /^▸ uipath-example 2/ })).toBeNull(); + expect(skillButton("uipath-example-two")).toHaveTextContent("1 failed"); + fireEvent.click(chips.getByRole("button", { name: "Skill" })); + expect(chips.getByRole("button", { name: "Grader" })).toHaveAttribute("aria-pressed", "true"); + expect(skillButton("uipath-example")).toHaveTextContent("2 failed"); + }); + + test("search filters and opens the matches", () => { + renderBlock(); + openBlock(); + fireEvent.change(screen.getByRole("searchbox", { name: "Search failures" }), { + target: { value: "removed flag" }, + }); + expect(screen.queryByRole("button", { name: /uipath-example 2 failed/ })).toBeNull(); + expect(skillButton("uipath-example-two")).toHaveAttribute("aria-expanded", "true"); + expect(screen.getAllByTestId("task-card")).toHaveLength(1); + fireEvent.change(screen.getByRole("searchbox"), { target: { value: "zzz-nothing" } }); + expect(screen.getByText("No failures match this filter.")).toBeInTheDocument(); + }); + + test("untrusted build URLs are plain text", () => { + renderBlock(doctor({ state: "running", build_url: "javascript:alert(1)" })); + expect(screen.queryByRole("link", { name: "build" })).toBeNull(); + expect(screen.getByText("build").tagName).toBe("SPAN"); + }); + + test("the source travels in task links", () => { + renderBlock(doctor(), "gha"); + openBlock(); + openSkill("uipath-example"); + expect(screen.getByRole("link", { name: "example-t1" })).toHaveAttribute( + "href", + `/runs/${RUN}/example-t1?src=gha`, + ); + }); +}); + +describe("untrusted keys", () => { + test.each(["constructor", "__proto__", "toString", "hasOwnProperty"])( + "layer / solvable / opened task id %j do not crash and render as text", + (k) => { + const d = doctor(); + d.triage!.tasks[1] = { ...d.triage!.tasks[1], layer: k, solvable: k }; + d.opened = JSON.parse(JSON.stringify({ [k]: OPENED })); + renderBlock(d); + openBlock(); + fireEvent.click(screen.getByRole("button", { name: "Expand all" })); + const c = card("example-t1"); + expect(within(c).getAllByText(k).length).toBeGreaterThan(0); + expect(screen.getByRole("toolbar", { name: "Filter failures" })).toHaveTextContent(k); + }, + ); +}); + +describe("prototype-named task ids", () => { + test.each(["constructor", "__proto__", "toString"])( + "task id %j: opened.json and pr_checks entries resolve to their own rows", + (k) => { + const d = doctor(); + const t1 = { ...d.triage!.tasks[1], taskId: k }; + const t2 = { ...d.triage!.tasks[2], taskId: `${k}-2` }; + d.triage!.tasks = [t1, t2]; + d.fixes!.candidates = [ + { ...d.fixes!.candidates[0], taskId: k, covers: [k] }, + ]; + // As parsed from JSON text, so "__proto__" is an own key. + d.opened = JSON.parse(JSON.stringify({}).replace("{}", `{"${k}": "${OPENED}"}`)); + d.fixes!.prChecks = JSON.parse( + `{"${k}-2": {"number": 126, "verdict": "verified", "evidence": "ok", "reason": null, "headSha": null}}`, + ); + // t1 claims #126 too, but has no check of its own: "may fix". + t1.fixedByPr = { number: 126, verdict: "fixes", basis: "b" }; + t1.existingPrs = t2.existingPrs; + renderBlock(d); + openBlock(); + fireEvent.click(screen.getByRole("button", { name: "Expand all" })); + const c1 = screen.getAllByTestId("task-card").find((c) => within(c).queryByText(k))!; + expect(within(c1).getByRole("link", { name: "PR with fix: #300 ↗" })).toBeInTheDocument(); + expect(within(c1).getByTestId("maybe-pr")).toHaveTextContent("#126"); + const c2 = card(`${k}-2`); + expect(within(c2).getByTestId("proposed-pr")).toHaveTextContent("#126"); + expect(within(c2).queryByTestId("pr-with-fix")).toBeNull(); + // #300 (own) + #126 (verified on the second task). + expect(digest(d)).toBe("5 errors · 2 PRs with fix"); + }, + ); +}); + +describe("doctor PR resolution", () => { + test("a task covered by an opened candidate gets that PR, not 'Fix prepared'", () => { + const d = doctor(); + // example-grader-fix's primary task is example-t1; it also covers t2. + d.fixes!.candidates[0].covers = ["example-t1", "example-t2"]; + d.opened = { "example-t1": OPENED }; + renderBlock(d); + openBlock(); + fireEvent.click(screen.getByRole("button", { name: "Expand all" })); + for (const id of ["example-t1", "example-t2"]) { + const c = card(id); + expect(within(c).getByRole("link", { name: "PR with fix: #300 ↗" })).toBeInTheDocument(); + expect(within(c).queryByTestId("fix-prepared")).toBeNull(); + } + // #300 counted once, plus #126 on t2. + expect(digest(d)).toBe("5 errors · 2 PRs with fix"); + }); + + test("the doctor's own PR is not also a Proposed PR, and counts once", () => { + const d = doctor(); + const pr126 = "https://github.com/UiPath/skills/pull/126"; + d.triage!.tasks[2].doctorPr = pr126; + renderBlock(d); + openBlock(); + openSkill("uipath-example-two"); + const c = card("example-t2"); + expect(within(c).getByRole("link", { name: "PR with fix: #126 ↗" })).toBeInTheDocument(); + expect(within(c).queryByTestId("proposed-pr")).toBeNull(); + expect(digest(d)).toBe("5 errors · 1 PR with fix"); + }); + + test("no placeholder row for a fixable task without any PR or candidate", () => { + const d = doctor(); + d.fixes = null; + d.triage!.tasks[2].fixedByPr = null; + renderBlock(d); + openBlock(); + fireEvent.click(screen.getByRole("button", { name: "Expand all" })); + expect(block()).not.toHaveTextContent("none yet"); + }); +}); + +describe("groupBySkill", () => { + test("skills: linked PRs first, then prepared fixes, then failed tasks, then name", () => { + // -two: 1 linked (#126) of 3; uipath-example: 0 linked, 1 prepared, of 2. + expect(groupBySkill(doctor(), null, "").map((g) => g.skill)).toEqual([ + "uipath-example-two", + "uipath-example", + ]); + // Without #126, the prepared fix beats more failed tasks. + const d = doctor(); + d.triage!.tasks[2].fixedByPr = null; + expect(groupBySkill(d, null, "").map((g) => g.skill)).toEqual([ + "uipath-example", + "uipath-example-two", + ]); + // Nothing at all: more failed tasks first. + const none = { ...d, fixes: null }; + expect(groupBySkill(none, null, "").map((g) => g.skill)).toEqual([ + "uipath-example-two", + "uipath-example", + ]); + // Equal on everything: by name. + const tie = { ...none, triage: { ...none.triage!, tasks: [ + { ...none.triage!.tasks[1], skill: "b-skill", taskId: "x1" }, + { ...none.triage!.tasks[1], skill: "a-skill", taskId: "x2" }, + ] } }; + expect(groupBySkill(tie, null, "").map((g) => g.skill)).toEqual(["a-skill", "b-skill"]); + }); + + test("cards: own PR, verified PR, prepared fix, unverified PR, other candidates, the rest", () => { + const d = doctor(); + const base = { ...d.triage!.tasks[1], doctorPr: null, fix: "plain" }; + d.triage!.tasks = [ + { ...base, taskId: "z-plain-skill", layer: "skill" }, + { ...base, taskId: "a-plain-grader", layer: "grader" }, + { ...base, taskId: "m-live", layer: "grader" }, + { ...base, taskId: "y-prepared", layer: "grader" }, + { ...base, taskId: "w-opened", layer: "grader" }, + { ...base, taskId: "x-doctor-pr", layer: "grader", doctorPr: "https://github.com/UiPath/skills/pull/7" }, + { ...base, taskId: "p-proposed", layer: "grader", fixedByPr: { number: 123, verdict: "fixes", basis: "" } }, + { ...base, taskId: "q-maybe", layer: "grader", fixedByPr: { number: 123, verdict: "fixes", basis: "" } }, + { ...d.triage!.tasks[0], skill: base.skill }, + ]; + d.fixes!.candidates = [ + { ...d.fixes!.candidates[0], id: "f1", taskId: "y-prepared", covers: ["y-prepared"] }, + { ...d.fixes!.candidates[0], id: "f3", taskId: "w-opened", covers: ["w-opened"] }, + { ...d.fixes!.candidates[1], id: "f2", taskId: "m-live", covers: ["m-live"] }, + ]; + d.opened = { "w-opened": "https://github.com/UiPath/skills/pull/8" }; + d.fixes!.prChecks = { + "p-proposed": { number: 123, verdict: "verified", evidence: "", reason: null, headSha: null }, + }; + const [g] = groupBySkill(d, null, ""); + expect(g.cards.map((t) => t.taskId)).toEqual([ + "w-opened", + "x-doctor-pr", + "p-proposed", + "y-prepared", + "q-maybe", + "m-live", + // The rest: skill ranks after grader on the doctor page, so first. + "z-plain-skill", + "a-plain-grader", + ]); + expect(g.envGroups.map((e) => e.tasks.map((t) => t.taskId))).toEqual([["example-e1"]]); + // Linked: #8, #7, #123 (verified). The prepared f1 and the unverified q-maybe are not. + expect(g.fixPrs).toBe(3); + }); + + test("the rendered accordions follow the same order", () => { + const d = doctor(); + d.triage!.tasks[2].fixedByPr = null; + renderBlock(d); + openBlock(); + const order = Array.from(document.querySelectorAll("[data-skill]")).map((e) => + e.getAttribute("data-skill"), + ); + expect(order).toEqual(["uipath-example", "uipath-example-two"]); + }); + + test("countFixPrs counts each linked PR once, and never a prepared fix", () => { + const d = doctor(); + d.triage!.tasks[1].fixedByPr = d.triage!.tasks[2].fixedByPr; + d.triage!.tasks[1].existingPrs = d.triage!.tasks[2].existingPrs; + d.fixes!.prChecks["example-t1"] = d.fixes!.prChecks["example-t2"]; + // t1 and t2 both verified against #126: one PR. + expect(countFixPrs(d, d.triage!.tasks)).toBe(1); + }); +}); diff --git a/evalboard/app/runs/[id]/page.tsx b/evalboard/app/runs/[id]/page.tsx index 49401abf..9abb9b76 100644 --- a/evalboard/app/runs/[id]/page.tsx +++ b/evalboard/app/runs/[id]/page.tsx @@ -9,6 +9,7 @@ import { findMatureSourceRuns, readActivationScore, readRunAnalysis, + readRunDoctor, readRunMeta, readRunSummary, readRunTasks, @@ -24,6 +25,8 @@ import { sourceById } from "@/lib/sources"; import { scalarParam } from "@/app/_lib/source-param"; import { fmtRunTime } from "@/lib/format"; import { AnalysisPanel } from "./analysis-panel"; +import { TaskFailureAnalysis } from "./task-failure-analysis"; +import { planAnalysis, taskAnalysisEnabled } from "@/lib/doctor"; import { RefreshButton } from "./refresh-button"; import { RunView } from "./run-view"; import { RunIdentity } from "./run-identity"; @@ -43,7 +46,14 @@ export default async function RunPage({ // Which container this run lives in. Unknown/absent coerces to the skills // nightly, so every URL that predates the Scribe tab keeps resolving as-is. const source = sourceById(scalarParam((await searchParams).src)); - const [summary, tasks, activation, analysis, reviewIndex, meta] = + // The skill-doctor report: internal-only, skills nightly only, with an env + // kill switch (EVALBOARD_TASK_ANALYSIS=off). readRunDoctor never throws. + const doctorOn = taskAnalysisEnabled( + isInternal, + source.id, + process.env.EVALBOARD_TASK_ANALYSIS, + ); + const [summary, tasks, activation, analysis, reviewIndex, meta, doctor] = await Promise.all([ readRunSummary(id, source), readRunTasks(id, source), @@ -53,6 +63,7 @@ export default async function RunPage({ readRunAnalysis(id, source), readRunReviewIndex(id, source), readRunMeta(id, source), + doctorOn ? readRunDoctor(id, source) : Promise.resolve(null), ]); if (!summary || !tasks) notFound(); // Mature-skipped tasks weren't executed this run, so they have no detail page @@ -85,6 +96,9 @@ export default async function RunPage({ const tagCounts = reviewIndex ? tagCountsForRun(reviewIndex) : []; const reviewsByTask = reviewIndex ? indexByTask(reviewIndex) : undefined; const human = fmtRunTime(id); + // A done report replaces analysis.md's Findings and everything below it; + // otherwise the AI summary is unchanged. The doctor block sits under it. + const plan = planAnalysis(analysis, doctor); const series = isInternal ? seriesForRun(source.id, { series: meta?.series ?? null, @@ -156,7 +170,14 @@ export default async function RunPage({ )} - {analysis && } + {plan.summary && } + {plan.doctor && ( + + )} {/* The header above streams as soon as run.json is parsed, but the grid waits on the review index and the mature-source scan. With diff --git a/evalboard/app/runs/[id]/task-failure-analysis.tsx b/evalboard/app/runs/[id]/task-failure-analysis.tsx new file mode 100644 index 00000000..ae8082a4 --- /dev/null +++ b/evalboard/app/runs/[id]/task-failure-analysis.tsx @@ -0,0 +1,984 @@ +"use client"; + +import Link from "next/link"; +import { useMemo, useState, type ReactNode } from "react"; +import { withSource } from "@/app/_lib/source-param"; +import { fmtTimestamp } from "@/lib/format"; +import { + DOCTOR_DIR, + isAdoUrl, + isLinkableTaskId, + isSkillsPrUrl, + own, + patchIds, + type DoctorFix, + type DoctorPr, + type DoctorStatus, + type DoctorTask, + type DoctorTriage, + type RunDoctor, +} from "@/lib/doctor"; + +// The skill-doctor report for one run, drawn natively from its JSON in the +// order of the doctor's own page (stats → failures by layer → Fix first → +// skill failures), never by embedding that page. Every string is plain text; +// the only links are this board's task pages and URLs of an exact known shape. + +// Layer order, labels and colors, as on the doctor's page. Maps, not object +// literals: layer and solvable come from the report, and a key such as +// "constructor" or "__proto__" must not reach the object prototype. +const LAYERS = new Map([ + ["env", { label: "Environment", color: "#56708f" }], + ["grader", { label: "Grader", color: "#b07410" }], + ["task", { label: "Task setup", color: "#7a4fb0" }], + ["agent", { label: "Agent mistake", color: "#c2413a" }], + ["skill", { label: "Skill", color: "#d0661c" }], + ["upstream", { label: "Upstream (CLI)", color: "#2f6fd0" }], + ["flaky", { label: "Flaky", color: "#7d8794" }], + ["undiagnosed", { label: "Not diagnosed", color: "#9aa5b3" }], +]); +export const LAYER_ORDER = [...LAYERS.keys()]; +const OTHER_COLOR = "#9aa5b3"; +const layerOf = (k: string) => LAYERS.get(k) ?? { label: k, color: OTHER_COLOR }; +const rank = (k: string) => { + const i = LAYER_ORDER.indexOf(k); + return i === -1 ? LAYER_ORDER.length : i; +}; + +const SOLVABLE = new Map([ + ["env-fix", "Fix the tenant"], + ["yes-grader", "Fixable: grader"], + ["yes-task", "Fixable: task"], + ["yes-skill", "Fixable: skill"], + ["upstream-report", "Upstream report"], + ["no", "Not a repo fix"], +]); +const isFixable = (t: DoctorTask) => /^yes/.test(t.solvable ?? ""); +const isReady = (c: DoctorFix) => c.outcome === "pr-ready"; + +/** Tasks per layer, in the doctor page's layer order. */ +export function layerCounts(tasks: DoctorTask[]): [string, number][] { + const m = new Map(); + for (const t of tasks) m.set(t.layer, (m.get(t.layer) ?? 0) + 1); + return [...m.entries()].sort((a, b) => rank(a[0]) - rank(b[0])); +} + +// ---------- text ---------- + +/** + * Plain text with `code` spans, and URLs turned into links only when they + * pass an allowlist (a UiPath/skills PR or a dev.azure.com build). + */ +export function RichText({ text }: { text: string }) { + const out: ReactNode[] = []; + text.split(/(`[^`]+`)/).forEach((part, i) => { + if (part.length > 2 && part.startsWith("`") && part.endsWith("`")) { + out.push( + + {part.slice(1, -1)} + , + ); + return; + } + part.split(/(https:\/\/[^\s)]+)/).forEach((seg, j) => { + if (!seg) return; + const url = seg.replace(/[.,;:]+$/, ""); + if (isSkillsPrUrl(url) || isAdoUrl(url)) { + out.push( + + {url} + , + seg.slice(url.length), + ); + } else out.push(seg); + }); + }); + return <>{out}; +} + +/** An external link, opened in a new tab. Callers pass only allowlisted URLs. */ +function ExtA({ + href, + className = "text-studio-blue hover:underline", + children, +}: { + href: string; + className?: string; + children: ReactNode; +}) { + return ( + + {children} + + ); +} + +/** A build link only for a dev.azure.com URL; anything else is the label as text. */ +function BuildLink({ href, label }: { href: string; label: string }) { + if (!isAdoUrl(href)) return {label}; + return {label}; +} + +function prLabel(url: string): string { + const m = /\/pull\/(\d+)$/.exec(url); + return m ? `#${m[1]}` : url; +} + +// ---------- digest + state ---------- + +export function digest(doctor: RunDoctor): string { + const { status, triage } = doctor; + switch (status.state) { + case "queued": + case "running": + return status.startedAt + ? `${status.state} since ${fmtTimestamp(status.startedAt)}` + : status.state; + case "failed": + case "skipped": + return status.reason ? `${status.state}: ${status.reason}` : status.state; + case "retracted": + return "the report for this run was retracted"; + case "done": { + if (!triage) + return doctor.fetchFailed + ? "report done, but its triage file could not be fetched" + : "report done, but its triage file is missing or did not parse"; + const n = countFixPrs(doctor, triage.tasks); + const errors = `${triage.rowsFailed} error${triage.rowsFailed === 1 ? "" : "s"}`; + return n ? `${errors} · ${prsWithFix(n)}` : errors; + } + } +} + +// ---------- fixes ---------- + +function patchHref(runId: string, sourceId: string, dir: string, id: string) { + return withSource( + `/api/file?run=${encodeURIComponent(runId)}&path=${encodeURIComponent(`${DOCTOR_DIR}/${dir}/${id}.patch`)}`, + sourceId, + ); +} + +function gateText(f: DoctorFix): string { + const g = f.gates; + const parts: string[] = []; + if (g.replay) parts.push(`replay ${g.replay}`); + if (g.mutants) parts.push(`mutants ${g.mutants.killed}/${g.mutants.n}`); + if (g.review) parts.push(`review ${g.review}`); + return parts.join(" · "); +} + +interface FixCtx { + runId: string; + sourceId: string; + dir: string | null; + // The same set readRunDoctor fetches and /api/file serves. + linked: Set; +} + +// ---------- grouping ---------- + +export interface SkillGroup { + skill: string; + cards: DoctorTask[]; + envGroups: { cause: string; tasks: DoctorTask[] }[]; + layerCounts: [string, number][]; + // Distinct PRs with fix among the skill's tasks (countFixPrs). + fixPrs: number; + // Tasks with a fix prepared but no PR yet: sorts right after linked PRs. + prepared: number; + total: number; +} + +// A candidate whose Fix stage never ran (outcome not-run) has no patch and no +// gates: the board leaves it out entirely. +export function fixesForTask(doctor: RunDoctor, taskId: string): DoctorFix[] { + return (doctor.fixes?.candidates ?? []).filter( + (c) => c.outcome !== "not-run" && (c.taskId === taskId || c.covers.includes(taskId)), + ); +} + +export interface ExistingPrFix { + pr: DoctorPr; + // The Fix stage replayed the PR and it flipped the row ("Proposed PR"). + verified: boolean; + // The diagnosis's reason for naming this PR. + basis: string; + // The replay's evidence, when there was a check, and the PR head it used. + evidence: string | null; + headSha: string | null; + // The check could not run: it needs a Windows run. + needsWindows: boolean; +} + +/** + * Someone else's open PR that fixes this row, from the doctor's structured + * fields only (no reading of prose): the diagnosis says `fixed_by_pr` with + * verdict "fixes", the PR is open in the row's existing PRs, and the Fix + * stage's pr_checks entry for the row (same number) is verified (counts as a + * PR with fix) or untested / absent (shown as "may fix this", not counted). + * not_verified, a malformed check, may_help and unrelated show nothing. + */ +export function existingPrFix(doctor: RunDoctor, t: DoctorTask): ExistingPrFix | null { + const f = t.fixedByPr; + if (!f || f.verdict !== "fixes") return null; + const pr = t.existingPrs.find((p) => p.state === "open" && p.url.endsWith(`/pull/${f.number}`)); + if (!pr) return null; + const c = own(doctor.fixes?.prChecks, t.taskId); + if (c === null) return null; // an entry that did not validate + const check = c && c.number === f.number ? c : null; + if (check?.verdict === "not_verified") return null; + return { + pr, + verified: check?.verdict === "verified", + basis: f.basis, + evidence: check?.evidence || null, + headSha: check?.headSha ?? null, + needsWindows: check?.reason === "needs-windows-run", + }; +} + +export interface TaskFixes { + // The doctor's own PR: opened for this task or for a pr-ready candidate + // that covers it (opened.json is keyed by the candidate's primary task), + // else the triage's doctor_pr. "PR with fix". + ownPr: string | null; + // The pr-ready candidates behind it, for their gate verdicts. + ready: DoctorFix[]; + // pr-ready candidates with no PR opened yet: "Fix prepared, PR not + // opened yet". Not a PR with fix. + prepared: DoctorFix[]; + // Someone else's open PR for this row: verified ("Proposed PR"; a PR with + // fix) or not ("Open PR may fix this", not counted). Never the doctor's + // own PR. + existingPr: ExistingPrFix | null; + // Other fix candidates (needs-live-run, ...). + others: DoctorFix[]; + // One key per distinct PR with fix (own or verified existing), for counting. + keys: string[]; + // Card sort tier, lower first (see fixTier). + tier: number; +} + +function openedFor(doctor: RunDoctor, taskId: string): string | null { + return own(doctor.opened, taskId) ?? null; +} + +// Own PR, then a verified proposed PR, then a prepared fix, then an +// unverified open PR, then other candidates, then the rest. +function fixTier(f: Omit): number { + if (f.ownPr) return 0; + if (f.existingPr?.verified) return 1; + if (f.prepared.length) return 2; + if (f.existingPr) return 3; + if (f.others.length) return 4; + return 5; +} + +export function taskFixes(doctor: RunDoctor, t: DoctorTask): TaskFixes { + const cands = fixesForTask(doctor, t.taskId); + const ready = cands.filter(isReady); + // URLs here are allowlisted by the parsers (parseOpened, parseTriage). + const ownPr = + openedFor(doctor, t.taskId) ?? + ready.map((c) => openedFor(doctor, c.taskId)).find((u) => u !== null) ?? + t.doctorPr; + const ext = existingPrFix(doctor, t); + const base = { + ownPr, + ready, + prepared: ownPr ? [] : ready, + existingPr: ext && ext.pr.url !== ownPr ? ext : null, + others: cands.filter((c) => !isReady(c)), + }; + const keys = [ + ...(ownPr ? [ownPr] : []), + ...(base.existingPr?.verified ? [base.existingPr.pr.url] : []), + ]; + return { ...base, keys, tier: fixTier(base) }; +} + +/** Tasks with a fix prepared but no PR opened (sort tie-break only). */ +export function countPrepared(doctor: RunDoctor, tasks: DoctorTask[]): number { + return tasks.filter((t) => taskFixes(doctor, t).prepared.length > 0).length; +} + +/** Distinct PRs with fix (the doctor's own or a verified proposed one) over these tasks. */ +export function countFixPrs(doctor: RunDoctor, tasks: DoctorTask[]): number { + return new Set(tasks.flatMap((t) => taskFixes(doctor, t).keys)).size; +} + +const prsWithFix = (n: number) => `${n} PR${n === 1 ? "" : "s"} with fix`; + +/** + * Visible failures by skill, as the doctor's page groups them. Skills with the + * most PRs with fix come first, then the most prepared fixes; ties go to more + * failed tasks, then name. + */ +export function groupBySkill( + doctor: RunDoctor, + active: Set | null, + query: string, +): SkillGroup[] { + const q = query.trim().toLowerCase(); + const vis = (doctor.triage?.tasks ?? []).filter( + (t) => + (!active || active.has(t.layer)) && + (!q || JSON.stringify(t).toLowerCase().includes(q)), + ); + const by = new Map(); + for (const t of vis) { + const k = t.skill || "(no skill)"; + by.set(k, [...(by.get(k) ?? []), t]); + } + return [...by.keys()] + .map((skill) => { + const items = by.get(skill)!; + const groups = new Map(); + const cards: DoctorTask[] = []; + for (const t of items) { + if (t.envGroup) groups.set(t.envGroup, [...(groups.get(t.envGroup) ?? []), t]); + else cards.push(t); + } + // By fix tier (fixTier), then layer rank and task id. + const tier = (t: DoctorTask) => taskFixes(doctor, t).tier; + cards.sort( + (a, b) => + tier(a) - tier(b) || + rank(b.layer) - rank(a.layer) || + a.taskId.localeCompare(b.taskId), + ); + return { + skill, + cards, + envGroups: [...groups.entries()].map(([cause, tasks]) => ({ cause, tasks })), + layerCounts: layerCounts(items), + fixPrs: countFixPrs(doctor, items), + prepared: countPrepared(doctor, items), + total: items.length, + }; + }) + .sort( + (a, b) => + b.fixPrs - a.fixPrs || + b.prepared - a.prepared || + b.total - a.total || + a.skill.localeCompare(b.skill), + ); +} + +// ---------- cards ---------- + +function Pill({ children, color }: { children: ReactNode; color?: string }) { + return ( + + {children} + + ); +} + +function TaskId({ id, ctx }: { id: string; ctx: FixCtx }) { + if (!isLinkableTaskId(id)) return {id}; + return ( + + {id} + + ); +} + +function TaskCard({ t, doctor, ctx }: { t: DoctorTask; doctor: RunDoctor; ctx: FixCtx }) { + const layer = layerOf(t.layer); + const fx = taskFixes(doctor, t); + const rows: [string, ReactNode, boolean?][] = []; + if (t.fix) rows.push(["Fix", ]); + if (t.criterion) rows.push(["Failing check", ]); + if (t.evidence) rows.push(["Evidence", , true]); + if (t.passedBefore) rows.push(["Passed before", ]); + return ( +
+
+ + {layer.label} + {t.solvable && ( + {SOLVABLE.get(t.solvable.split(" ")[0]) ?? t.solvable} + )} + {t.runStatus && ( + {t.runStatus.replace(/_/g, " ").toLowerCase()} + )} + {t.confidence && {t.confidence} confidence} +
+ {t.why && ( +

+ +

+ )} + + {rows.length > 0 && ( +
+ {rows.map(([k, v, mono]) => ( +
+
{k}
+
+ {v} +
+
+ ))} +
+ )} +
+ ); +} + +function Gates({ cands }: { cands: DoctorFix[] }) { + const g = cands.map(gateText).filter(Boolean); + if (!g.length) return null; + return {g.join("; ")}; +} + +// "patch +a/-r", linked to the download only for an id in patchIds. +function Patch({ c, ctx }: { c: DoctorFix; ctx: FixCtx }) { + if (!c.patchLines) return null; + const text = `+${c.patchLines.added}/-${c.patchLines.removed}`; + return ( + + patch{" "} + {ctx.dir && ctx.linked.has(c.id) ? ( + + {text} + + ) : ( + {text} + )} + + ); +} + +const ROW_BORDER = { + own: "border-studio-blue", + proposed: "border-studio-blue/60", + prepared: "border-dashed border-studio-blue/60", + candidate: "border-dashed border-gray-300", +} as const; + +function FixRow({ + kind, + testId, + children, +}: { + kind: keyof typeof ROW_BORDER; + testId: string; + children: ReactNode; +}) { + return ( +
+ {children} +
+ ); +} + +function PrLink({ pr }: { pr: DoctorPr }) { + return ( + + {prLabel(pr.url)} {pr.title} + + ); +} + +function Small({ children }: { children: ReactNode }) { + return {children}; +} + +// The replay's evidence as plain text, with the PR head it ran at. +function Evidence({ text, headSha }: { text: string; headSha: string | null }) { + return ( + + {text} + {headSha && (at {headSha.slice(0, 7)})} + + ); +} + +// The card's PR rows: "PR with fix" (the doctor's PR), "Proposed PR" (someone +// else's open PR, verified by the doctor's replay), "Fix prepared, PR not +// opened yet", "Open PR may fix this (not verified)", and "Fix candidate: +// " for candidates that are not ready. +function FixRows({ fx, ctx }: { fx: TaskFixes; ctx: FixCtx }) { + const rows: ReactNode[] = []; + if (fx.ownPr) { + rows.push( + + + PR with fix: {prLabel(fx.ownPr)} ↗ + + + , + ); + } + const ext = fx.existingPr; + if (ext?.verified) { + const { pr, evidence, headSha } = ext; + rows.push( + + Proposed PR + + {pr.state} + {evidence && } + , + ); + } + for (const c of fx.prepared) { + rows.push( + + Fix prepared, PR not opened yet + + + , + ); + } + if (ext && !ext.verified) { + const { pr, basis, evidence, headSha, needsWindows } = ext; + rows.push( + + Open PR may fix this (not verified) + + {needsWindows && needs a Windows run} + {basis && {basis}} + {evidence && } + , + ); + } + for (const c of fx.others) { + rows.push( + + Fix candidate: {c.outcome} + + + , + ); + } + if (rows.length === 0) return null; + return
{rows}
; +} + +function EnvGroup({ cause, tasks, ctx }: { cause: string; tasks: DoctorTask[]; ctx: FixCtx }) { + const [open, setOpen] = useState(false); + const first = tasks[0]; + return ( +
+ + {open && ( +
+ {first.why && ( +

+ +

+ )} + {first.fix && ( +

+ Fix: +

+ )} +
    + {tasks.map((t, i) => ( +
  • + +
  • + ))} +
+
+ )} +
+ ); +} + +function SkillAccordion({ + g, + open, + onToggle, + doctor, + ctx, +}: { + g: SkillGroup; + open: boolean; + onToggle: () => void; + doctor: RunDoctor; + ctx: FixCtx; +}) { + return ( +
+ + {open && ( +
+ {g.cards.map((t, i) => ( + + ))} + {g.envGroups.map((e) => ( + + ))} +
+ )} +
+ ); +} + +// ---------- body ---------- + +const chip = (pressed: boolean) => + `rounded-full border px-3 py-0.5 text-xs ${pressed ? "border-gray-900 bg-gray-900 text-white" : "border-gray-200 bg-white text-gray-800"}`; + +function Stats({ triage, counts }: { triage: DoctorTriage; counts: [string, number][] }) { + const count = (k: string) => counts.find(([l]) => l === k)?.[1] ?? 0; + const stats: [number | null, string][] = [ + [triage.rowsRun, "rows run"], + [triage.tasks.length, "failed rows"], + [count("env"), "environment"], + [triage.tasks.filter(isFixable).length, "fixable in the repo"], + [count("agent"), "agent mistakes"], + ]; + return ( +
+ {stats + .filter(([v]) => v !== null) + .map(([v, l]) => ( +
+ {v} + {l} +
+ ))} +
+ ); +} + +function LayerBar({ counts, n }: { counts: [string, number][]; n: number }) { + if (n === 0) return null; + return ( +
+
+ Failures by layer +
+
+ {counts.map(([k, c]) => ( + + ))} +
+
+ {counts.map(([k, c]) => ( + + + {layerOf(k).label} {c} + + ))} +
+
+ ); +} + +function FixFirst({ doctor, triage }: { doctor: RunDoctor; triage: DoctorTriage }) { + if (triage.actions.length === 0) return null; + return ( +
+

Fix first

+
    + {triage.actions.map((a, i) => { + const fixPrs = countFixPrs( + doctor, + triage.tasks.filter((t) => a.taskIds.includes(t.taskId)), + ); + return ( +
  1. + {a.count} + {" · "} + + {fixPrs > 0 && ( + + {prsWithFix(fixPrs)} + + )} +
  2. + ); + })} +
+
+ ); +} + +function SkillFailures({ + doctor, + present, + ctx, +}: { + doctor: RunDoctor; + present: string[]; + ctx: FixCtx; +}) { + // null: every layer shown (all chips pressed), as on the doctor's page. + const [active, setActive] = useState | null>(null); + const [query, setQuery] = useState(""); + const [openSkills, setOpenSkills] = useState>(new Set()); + const groups = groupBySkill(doctor, active, query); + const clickChip = (k: string) => { + setActive((cur) => { + if (!cur) return new Set([k]); + const next = new Set(cur); + if (next.has(k)) next.delete(k); + else next.add(k); + return next.size === 0 || present.every((p) => next.has(p)) ? null : next; + }); + }; + return ( +
+

Skill failures

+
+ + +
+
+ + Show + + {present.map((k) => { + const pressed = !active || active.has(k); + return ( + + ); + })} + setQuery(e.target.value)} + placeholder="Search task, cause, PR…" + aria-label="Search failures" + className="min-w-0 max-w-xs flex-1 rounded-md border border-gray-200 px-2.5 py-1 text-sm" + /> +
+
+ {groups.length === 0 ? ( +

No failures match this filter.

+ ) : ( + groups.map((g) => ( + + setOpenSkills((cur) => { + const next = new Set(cur); + if (next.has(g.skill)) next.delete(g.skill); + else next.add(g.skill); + return next; + }) + } + doctor={doctor} + ctx={ctx} + /> + )) + )} +
+
+ ); +} + +function Body({ doctor, ctx }: { doctor: RunDoctor; ctx: FixCtx }) { + const triage = doctor.triage!; + const counts = useMemo(() => layerCounts(triage.tasks), [triage.tasks]); + return ( +
+ + + + k)} ctx={ctx} /> +
+ ); +} + +// ---------- block ---------- + +function BuildLinks({ status }: { status: DoctorStatus }) { + return ( + <> + {status.buildUrl && ( + <> + {" · "} + + + )} + {status.trigger && ( + <> + {" · "} + + + )} + + ); +} + +/** + * The Task Failure Analysis block: shaped like the AI summary panel (bordered + * section, header button, collapsed by default), right under it. The header + * carries a one-line digest; only a done report with a triage has a body. + */ +export function TaskFailureAnalysis({ + doctor, + runId, + sourceId, +}: { + doctor: RunDoctor; + runId: string; + sourceId: string; +}) { + const [open, setOpen] = useState(false); + const hasBody = doctor.status.state === "done" && doctor.triage !== null; + const ctx: FixCtx = { + runId, + sourceId, + dir: doctor.status.fixesDir, + linked: patchIds(doctor.fixes), + }; + const label = ( + + Task failure analysis + + ); + const line = ( + + {digest(doctor)} + + ); + return ( +
+ {hasBody ? ( + + ) : ( +
+ {label} + {line} + + + +
+ )} + {hasBody && open && ( + <> +
+ skill-doctor + {doctor.fixes?.fixMode && ` · fix mode ${doctor.fixes.fixMode}`} + +
+ + + )} +
+ ); +} diff --git a/evalboard/lib/__tests__/doctor-blob.test.ts b/evalboard/lib/__tests__/doctor-blob.test.ts new file mode 100644 index 00000000..dd5b16d6 --- /dev/null +++ b/evalboard/lib/__tests__/doctor-blob.test.ts @@ -0,0 +1,426 @@ +// @vitest-environment node +import { promises as fs, readFileSync } from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { afterEach, beforeEach, describe, expect, test, vi } from "vitest"; + +const FIX = path.join(__dirname, "fixtures", "skill-doctor"); +const RUN = "2099-01-03_04-23-30"; + +// Blob name (relative to the container) -> body; a missing name is a 404. +const state = vi.hoisted(() => ({ + blobs: {} as Record, + downloads: [] as string[], + failWith: null as null | { statusCode: number }, + // Fail only this blob name (with failWith's status). + failOn: null as null | string, +})); + +vi.mock("@azure/storage-blob", () => ({ + BlobServiceClient: class { + getContainerClient() { + return { + getBlobClient: (name: string) => ({ + downloadToFile: async (dest: string) => { + state.downloads.push(name); + if (state.failWith && (state.failOn === null || state.failOn === name)) + throw Object.assign(new Error("blob failed"), state.failWith); + if (!(name in state.blobs)) + throw Object.assign(new Error("404"), { statusCode: 404 }); + await fs.writeFile(dest, state.blobs[name]); + }, + }), + }; + } + }, +})); +vi.mock("@azure/identity", () => ({ DefaultAzureCredential: class {} })); + +const fixture = (p: string) => readFileSync(path.join(FIX, p), "utf-8"); +const STATUS = fixture("status.json"); + +function seedBlobs(status: string = STATUS) { + const d = `${RUN}/skill-doctor`; + state.blobs = { + [`${d}/status.json`]: status, + [`${d}/triage-1000001.json`]: fixture("triage-1000001.json"), + [`${d}/fixes-1000001/fixes.json`]: fixture("fixes-1000001/fixes.json"), + [`${d}/fixes-1000001/example-grader-fix.patch`]: fixture( + "fixes-1000001/example-grader-fix.patch", + ), + [`${d}/fixes-1000001/opened.json`]: JSON.stringify({ opened: [] }), + }; +} + +function statusWith(patch: Record): string { + return JSON.stringify({ ...JSON.parse(STATUS), ...patch }); +} + +let root: string; + +beforeEach(async () => { + vi.resetModules(); + vi.stubEnv("EVALBOARD_LOCAL_RUNS_DIR", ""); + root = await fs.mkdtemp(path.join(os.tmpdir(), "doctor-blob-")); + vi.stubEnv("EVALBOARD_RUNS_DIR", root); + vi.stubEnv("EVALBOARD_EDITION", "internal"); + state.downloads = []; + state.failWith = null; + state.failOn = null; + seedBlobs(); +}); + +afterEach(async () => { + vi.unstubAllEnvs(); + await fs.rm(root, { recursive: true, force: true }); +}); + +async function read() { + const { readRunDoctor } = await import("../runs"); + return readRunDoctor(RUN); +} + +const D = `${RUN}/skill-doctor`; + +describe("ensureRunDoctorFile guard", () => { + test.each([ + ["../x"], + ["fixes-1/../status.json"], + ["a/b/c"], + [""], + ["/abs.json"], + ["a b.json"], + ])("rejects %j before any download", async (rel) => { + const { ensureRunDoctorFile } = await import("../blob"); + await expect(ensureRunDoctorFile("runs", RUN, rel, root)).rejects.toThrow( + /Invalid skill-doctor path/, + ); + expect(state.downloads).toEqual([]); + }); + + test("rejects a bad run id before any download", async () => { + const { ensureRunDoctorFile } = await import("../blob"); + await expect(ensureRunDoctorFile("runs", "../x", "status.json", root)).rejects.toThrow( + /Invalid runId/, + ); + expect(state.downloads).toEqual([]); + }); +}); + +describe("readRunDoctor: what it fetches", () => { + test("a done report: status, triage, fixes.json and the one patch", async () => { + await read(); + expect(state.downloads.sort()).toEqual( + [ + `${D}/status.json`, + `${D}/triage-1000001.json`, + `${D}/fixes-1000001/fixes.json`, + `${D}/fixes-1000001/example-grader-fix.patch`, + `${D}/fixes-1000001/opened.json`, + ].sort(), + ); + }); + + test("no report: one 404, swallowed", async () => { + state.blobs = {}; + await expect(read()).resolves.toBeNull(); + expect(state.downloads).toEqual([`${D}/status.json`]); + }); + + test("running: the triage is not fetched", async () => { + seedBlobs(statusWith({ state: "running", fixes_dir: null })); + expect((await read())?.status.state).toBe("running"); + expect(state.downloads).toEqual([`${D}/status.json`]); + }); + + test.each(["queued", "running", "failed", "skipped", "retracted"])( + "%s with fixes_dir kept: only status.json is fetched, no patch is served", + async (st) => { + seedBlobs(statusWith({ state: st })); + const doctor = await read(); + expect(doctor?.status.state).toBe(st); + expect(doctor?.triage ?? null).toBeNull(); + expect(doctor?.fixes ?? null).toBeNull(); + const { resolveSafePath } = await import("../runs"); + expect( + await resolveSafePath(RUN, "skill-doctor/fixes-1000001/example-grader-fix.patch"), + ).toBeNull(); + expect([...new Set(state.downloads)]).toEqual([`${D}/status.json`]); + }, + ); + + test("a storage error on the triage: the page still loads, flagged fetchFailed", async () => { + state.failWith = { statusCode: 500 }; + state.failOn = `${D}/triage-1000001.json`; + const err = vi.spyOn(console, "error").mockImplementation(() => {}); + const doctor = await read(); + expect(doctor?.triage).toBeNull(); + expect(doctor?.fetchFailed).toBe(true); + expect(err).toHaveBeenCalled(); + err.mockRestore(); + }); + + test("a triage that names another run is ignored", async () => { + const t = JSON.parse(fixture("triage-1000001.json")); + t.run.run_id = "2099-01-02_04-23-30"; + state.blobs[`${D}/triage-1000001.json`] = JSON.stringify(t); + const warn = vi.spyOn(console, "warn").mockImplementation(() => {}); + const doctor = await read(); + expect(doctor?.triage).toBeNull(); + expect(doctor?.fetchFailed).toBe(false); + expect(warn).toHaveBeenCalledWith(expect.stringMatching(/names run/)); + warn.mockRestore(); + }); + + test("a fixes.json that names another run lists no patch", async () => { + const f = JSON.parse(fixture("fixes-1000001/fixes.json")); + f.run_id = "2099-01-02_04-23-30"; + state.blobs[`${D}/fixes-1000001/fixes.json`] = JSON.stringify(f); + const warn = vi.spyOn(console, "warn").mockImplementation(() => {}); + const doctor = await read(); + expect(doctor?.fixes).toBeNull(); + expect(doctor?.triage).not.toBeNull(); + const { resolveSafePath } = await import("../runs"); + expect( + await resolveSafePath(RUN, "skill-doctor/fixes-1000001/example-grader-fix.patch"), + ).toBeNull(); + expect(state.downloads.some((n) => n.endsWith(".patch"))).toBe(false); + warn.mockRestore(); + }); + + test("a bad triage_file name is never fetched", async () => { + seedBlobs(statusWith({ triage_file: "../run.json", fixes_dir: null })); + expect((await read())?.triage).toBeNull(); + expect(state.downloads).toEqual([`${D}/status.json`]); + }); + + test("schema_version 2: nothing beyond status.json, and a warning", async () => { + seedBlobs(statusWith({ schema_version: 2 })); + const warn = vi.spyOn(console, "warn").mockImplementation(() => {}); + await expect(read()).resolves.toBeNull(); + expect(state.downloads).toEqual([`${D}/status.json`]); + expect(warn).toHaveBeenCalledTimes(1); + // Names the file, never its contents. + expect(String(warn.mock.calls[0][0])).toContain("status.json"); + expect(String(warn.mock.calls[0][0])).not.toContain("schema_version"); + warn.mockRestore(); + }); + + test("status.json and opened.json are refetched after a minute, not within it; the rest never", async () => { + await read(); + state.downloads = []; + await read(); + expect(state.downloads).toEqual([]); + const old = new Date(Date.now() - 120_000); + for (const f of ["status.json", "fixes-1000001/opened.json"]) { + await fs.utimes(path.join(root, RUN, "skill-doctor", f), old, old); + } + await read(); + expect(state.downloads.sort()).toEqual( + [`${D}/status.json`, `${D}/fixes-1000001/opened.json`].sort(), + ); + }); + + test("an opened.json that changes is picked up after the refresh age", async () => { + expect((await read())!.opened).toEqual({}); + state.blobs[`${D}/fixes-1000001/opened.json`] = JSON.stringify({ + opened: [{ task_id: "example-t1", pr_url: "https://github.com/UiPath/skills/pull/300" }], + }); + // Within the minute: the cached copy. + expect((await read())!.opened).toEqual({}); + const old = new Date(Date.now() - 120_000); + await fs.utimes(path.join(root, RUN, "skill-doctor", "fixes-1000001", "opened.json"), old, old); + expect((await read())!.opened).toEqual({ + "example-t1": "https://github.com/UiPath/skills/pull/300", + }); + }); + + test("patch fan-out is capped and ids are checked", async () => { + const doc = JSON.parse(fixture("fixes-1000001/fixes.json")); + const base = doc.candidates[0]; + doc.candidates = [ + { ...base, id: "../escape" }, + ...Array.from({ length: 15 }, (_, i) => ({ ...base, id: `fix-${i}` })), + ]; + state.blobs[`${D}/fixes-1000001/fixes.json`] = JSON.stringify(doc); + await read(); + const patches = state.downloads.filter((n) => n.endsWith(".patch")); + expect(patches).toHaveLength(10); + expect(patches.some((n) => n.includes(".."))).toBe(false); + }); +}); + +describe("readRunDoctor", () => { + test("parses the cached report", async () => { + const { readRunDoctor } = await import("../runs"); + const d = await readRunDoctor(RUN); + expect(d?.status.state).toBe("done"); + expect(d?.triage?.tasks).toHaveLength(5); + expect(d?.fixes?.candidates).toHaveLength(2); + expect(d!.opened).toEqual({}); + }); + + test("opened.json maps opened PRs by task; a bad host or a problem entry does not", async () => { + state.blobs[`${D}/fixes-1000001/opened.json`] = JSON.stringify({ + opened: [ + { task_id: "example-t1", pr_url: "https://github.com/UiPath/skills/pull/300" }, + { task_id: "example-t2", pr_url: "https://evil.example/UiPath/skills/pull/301" }, + { task_id: "example-e1", problem: "push refused" }, + ], + }); + const { readRunDoctor } = await import("../runs"); + const d = await readRunDoctor(RUN); + expect(d!.opened).toEqual({ "example-t1": "https://github.com/UiPath/skills/pull/300" }); + }); + + test("no or malformed opened.json: no opened PRs, no throw", async () => { + delete state.blobs[`${D}/fixes-1000001/opened.json`]; + const { readRunDoctor } = await import("../runs"); + expect((await readRunDoctor(RUN))!.opened).toEqual({}); + vi.resetModules(); + state.blobs[`${D}/fixes-1000001/opened.json`] = "{not json"; + const warn = vi.spyOn(console, "warn").mockImplementation(() => {}); + const { readRunDoctor: again } = await import("../runs"); + // Cached copy from before is fresh, so drop it to read the bad one. + await fs.rm(path.join(root, RUN, "skill-doctor", "fixes-1000001", "opened.json"), { force: true }); + expect((await again(RUN))!.opened).toEqual({}); + expect(warn).toHaveBeenCalled(); + warn.mockRestore(); + }); + + test("never throws: a storage error is no report", async () => { + state.failWith = { statusCode: 500 }; + const { readRunDoctor } = await import("../runs"); + const err = vi.spyOn(console, "error").mockImplementation(() => {}); + await expect(readRunDoctor(RUN)).resolves.toBeNull(); + err.mockRestore(); + }); + + test("an unparsable status is no report", async () => { + seedBlobs("{oops"); + const { readRunDoctor } = await import("../runs"); + await expect(readRunDoctor(RUN)).resolves.toBeNull(); + }); + + test("/api/file's path check serves a cached patch without a listing", async () => { + const { readRunDoctor, resolveSafePath } = await import("../runs"); + await readRunDoctor(RUN); + state.downloads = []; + const abs = await resolveSafePath( + RUN, + "skill-doctor/fixes-1000001/example-grader-fix.patch", + ); + expect(abs).toBe( + path.join( + await fs.realpath(root), + RUN, + "skill-doctor", + "fixes-1000001", + "example-grader-fix.patch", + ), + ); + expect(state.downloads).toEqual([]); + }); + + test("a cold cache: /api/file loads status + fixes.json, then the listed patch", async () => { + const { resolveSafePath } = await import("../runs"); + const abs = await resolveSafePath( + RUN, + "skill-doctor/fixes-1000001/example-grader-fix.patch", + ); + expect(state.downloads).toContain(`${D}/status.json`); + expect(state.downloads).toContain(`${D}/fixes-1000001/fixes.json`); + expect(state.downloads).toContain(`${D}/fixes-1000001/example-grader-fix.patch`); + expect(await fs.readFile(abs!, "utf-8")).toContain("diff --git"); + }); + + test("a patch fixes.json does not list, or another fixes dir, is refused", async () => { + const { resolveSafePath } = await import("../runs"); + expect(await resolveSafePath(RUN, "skill-doctor/fixes-1000001/made-up.patch")).toBeNull(); + expect( + await resolveSafePath(RUN, "skill-doctor/fixes-9/example-grader-fix.patch"), + ).toBeNull(); + expect(state.downloads.some((n) => n.endsWith("made-up.patch"))).toBe(false); + expect(state.downloads.some((n) => n.includes("fixes-9"))).toBe(false); + }); + + test("kill switch off or the OSS edition: nothing served, nothing fetched", async () => { + vi.stubEnv("EVALBOARD_TASK_ANALYSIS", "Off"); + let { resolveSafePath } = await import("../runs"); + const rel = "skill-doctor/fixes-1000001/example-grader-fix.patch"; + expect(await resolveSafePath(RUN, rel)).toBeNull(); + vi.resetModules(); + vi.stubEnv("EVALBOARD_TASK_ANALYSIS", ""); + vi.stubEnv("EVALBOARD_EDITION", ""); + ({ resolveSafePath } = await import("../runs")); + expect(await resolveSafePath(RUN, rel)).toBeNull(); + expect(state.downloads).toEqual([]); + }); + + test.each([ + ["the patch", `${D}/fixes-1000001/example-grader-fix.patch`], + ["fixes.json", `${D}/fixes-1000001/fixes.json`], + ["status.json", `${D}/status.json`], + ])("a storage error on %s propagates (the route answers 5xx, not 403/404)", async (_, name) => { + state.failWith = { statusCode: 500 }; + state.failOn = name; + const { resolveSafePath } = await import("../runs"); + await expect( + resolveSafePath(RUN, "skill-doctor/fixes-1000001/example-grader-fix.patch"), + ).rejects.toThrow(/blob failed/); + }); + + test("a patch link fetches only that patch: a storage error elsewhere does not break it", async () => { + state.failWith = { statusCode: 500 }; + state.failOn = `${D}/fixes-1000001/opened.json`; + const { resolveSafePath } = await import("../runs"); + const abs = await resolveSafePath( + RUN, + "skill-doctor/fixes-1000001/example-grader-fix.patch", + ); + expect(abs).toMatch(/example-grader-fix\.patch$/); + await expect(fs.access(abs!)).resolves.toBeUndefined(); + expect(state.downloads).not.toContain(`${D}/fixes-1000001/opened.json`); + }); + + test("normalized before the dispatch: ./ and ../ cannot skip the gate", async () => { + vi.stubEnv("EVALBOARD_EDITION", ""); + const { resolveSafePath } = await import("../runs"); + for (const bad of [ + "./skill-doctor/fixes-1000001/example-grader-fix.patch", + "activation/../skill-doctor/fixes-1000001/example-grader-fix.patch", + "./skill-doctor/status.json", + "../x", + "..", + ]) { + expect(await resolveSafePath(RUN, bad)).toBeNull(); + } + expect(state.downloads).toEqual([]); + }); + + test.each(["..", "."])("a run id of %j is refused", async (id) => { + const { resolveSafePath } = await import("../runs"); + expect(await resolveSafePath(id, ".env")).toBeNull(); + expect(await resolveSafePath(id, "run.json")).toBeNull(); + expect(state.downloads).toEqual([]); + }); + + test("only patch paths are served from skill-doctor/", async () => { + const { resolveSafePath } = await import("../runs"); + for (const bad of [ + "skill-doctor/status.json", + "skill-doctor/triage-1000001.json", + "skill-doctor/fixes-1000001/fixes.json", + // /api/file serves .patch files only. + "skill-doctor/fixes-1000001/example-grader-fix.md", + "skill-doctor/fixes-1000001/opened.json", + "skill-doctor/fixes-1000001/../status.json", + "skill-doctor/other/x.patch", + "skill-doctor/fixes-1000001/a/b.patch", + "skill-doctor/../../x", + ]) { + expect(await resolveSafePath(RUN, bad)).toBeNull(); + } + expect(state.downloads).toEqual([]); + }); +}); diff --git a/evalboard/lib/__tests__/doctor-schema.test.ts b/evalboard/lib/__tests__/doctor-schema.test.ts new file mode 100644 index 00000000..353e58be --- /dev/null +++ b/evalboard/lib/__tests__/doctor-schema.test.ts @@ -0,0 +1,134 @@ +import { readFileSync } from "node:fs"; +import path from "node:path"; +import { describe, expect, test } from "vitest"; +import { LAYER_ORDER } from "@/app/runs/[id]/task-failure-analysis"; +import { DOCTOR_STATES } from "../doctor"; + +// The board's parsers (lib/doctor.ts) are written by hand against the doctor's +// contract v1. These tests pin them to copies of the producer's schemas +// (fixtures/skill-doctor/schemas/, see its README): a field the parsers read +// must be declared there, and the hand-written fixtures may use only declared +// keys. A rename on the producer side then fails here once the copies are +// refreshed, instead of silently dropping a column on the board. + +type Schema = Record; + +const FIX = path.join(__dirname, "fixtures", "skill-doctor"); +const load = (p: string) => JSON.parse(readFileSync(path.join(FIX, p), "utf-8")); +const SCHEMAS: Record = { + status: load("schemas/status_v1.json"), + triage: load("schemas/triage_v1.json"), + fixes: load("schemas/fixes_v1.json"), +}; + +function deref(root: Schema, node: unknown): Schema | null { + let n = node; + while (n && typeof n === "object" && "$ref" in n) { + const ref = (n as Schema).$ref as string; + n = (root.$defs as Schema)[ref.replace("#/$defs/", "")]; + } + return n && typeof n === "object" ? (n as Schema) : null; +} + +// "a.[].b": `[]` steps into array items, `*` into additionalProperties. +function nodeAt(root: Schema, p: string): Schema | null { + let node: Schema | null = root; + for (const seg of p.split(".")) { + node = deref(root, node); + if (!node) return null; + if (seg === "[]") node = deref(root, node.items); + else if (seg === "*") node = deref(root, node.additionalProperties); + else node = deref(root, (node.properties as Schema | undefined)?.[seg]); + } + return node; +} + +// Every field the parsers in lib/doctor.ts read. +const READS: Record = { + status: [ + "schema_version", "state", "reason", "started_at", "build_url", + "triage_file", "fixes_dir", "trigger", + ], + triage: [ + "schema_version", "run.run_id", "run.rows_executed", + "summary.rows_failed", "summary.rows_diagnosed", + ...[ + "task_id", "skill", "layer", "solvable", "run_status", "confidence", + "why", "fix", "criterion", "evidence", "passed_before", "env_group", + "doctor_pr", "existing_prs.[].url", "existing_prs.[].title", + "existing_prs.[].state", "fixed_by_pr.number", "fixed_by_pr.verdict", + "fixed_by_pr.basis", + ].map((f) => `tasks.[].${f}`), + "actions.[].kind", "actions.[].count", "actions.[].text", "actions.[].task_ids", + ], + fixes: [ + "schema_version", "fix_mode", "candidates.[].outcome", + ...["number", "verdict", "url", "evidence", "reason", "head_sha"].map( + (f) => `pr_checks.*.${f}`, + ), + ], +}; + +describe("doctor contract: the parsers read only declared fields", () => { + test.each(Object.entries(READS).flatMap(([s, ps]) => ps.map((p) => [s, p])))( + "%s_v1 declares %s", + (s, p) => { + expect(nodeAt(SCHEMAS[s], p)).not.toBeNull(); + }, + ); + + test("fixes_v1 requires the candidate keys the board needs", () => { + const items = deref(SCHEMAS.fixes, nodeAt(SCHEMAS.fixes, "candidates")?.items); + expect(items?.required).toEqual(expect.arrayContaining(["id", "task_id", "outcome"])); + }); +}); + +const enumAt = (s: string, p: string) => nodeAt(SCHEMAS[s], p)?.enum as string[]; + +describe("doctor contract: closed value sets match the producer", () => { + test("status states", () => { + expect([...DOCTOR_STATES].sort()).toEqual([...enumAt("status", "state")].sort()); + }); + + test("every layer has a label, and only those", () => { + expect([...LAYER_ORDER].sort()).toEqual([...enumAt("triage", "tasks.[].layer")].sort()); + }); + + test("the action kind the board leaves out still exists", () => { + expect(enumAt("triage", "actions.[].kind")).toContain("open-pr"); + }); + + test("PR verdicts", () => { + expect([...enumAt("triage", "tasks.[].fixed_by_pr.verdict")].sort()).toEqual( + ["fixes", "may_help", "unrelated"], + ); + expect([...enumAt("fixes", "pr_checks.*.verdict")].sort()).toEqual( + ["not_verified", "untested", "verified"], + ); + }); +}); + +// Keys a fixture uses that the schema at that point does not declare (where +// it declares properties and closes the rest). +function undeclared(root: Schema, node: unknown, value: unknown, at = ""): string[] { + const s = deref(root, node); + if (!s || value === null || typeof value !== "object") return []; + if (Array.isArray(value)) + return value.flatMap((v, i) => undeclared(root, s.items, v, `${at}[${i}]`)); + const props = (s.properties ?? {}) as Schema; + return Object.entries(value).flatMap(([k, v]) => { + if (k in props) return undeclared(root, props[k], v, `${at}.${k}`); + if (s.additionalProperties === false) return [`${at}.${k}`]; + return undeclared(root, s.additionalProperties, v, `${at}.${k}`); + }); +} + +describe("doctor contract: the hand-written fixtures use declared keys", () => { + test.each([ + ["status", "status.json"], + ["triage", "triage-1000001.json"], + ["fixes", "fixes-1000001/fixes.json"], + ])("%s fixture %s", (s, file) => { + expect(undeclared(SCHEMAS[s], SCHEMAS[s], load(file))).toEqual([]); + }); +}); diff --git a/evalboard/lib/__tests__/doctor.test.ts b/evalboard/lib/__tests__/doctor.test.ts new file mode 100644 index 00000000..41b2892d --- /dev/null +++ b/evalboard/lib/__tests__/doctor.test.ts @@ -0,0 +1,466 @@ +import { readFileSync } from "node:fs"; +import path from "node:path"; +import { describe, expect, test } from "vitest"; +import { + isAdoUrl, + isLinkableTaskId, + patchIdFromPath, + isSkillsPrUrl, + parseDoctorStatus, + parseFixes, + parseOpened, + parseTriage, + patchIds, + planAnalysis, + splitAnalysis, + taskAnalysisEnabled, + type RunDoctor, +} from "../doctor"; + +const FIX = path.join(__dirname, "fixtures", "skill-doctor"); +const read = (p: string) => readFileSync(path.join(FIX, p), "utf-8"); +const STATUS = read("status.json"); +const TRIAGE = read("triage-1000001.json"); +const FIXES = read("fixes-1000001/fixes.json"); + +function statusWith(patch: Record): string { + return JSON.stringify({ ...JSON.parse(STATUS), ...patch }); +} + +function doctor(patch: Record = {}): RunDoctor { + const status = parseDoctorStatus(statusWith(patch))!; + return { + status, + triage: status.state === "done" ? parseTriage(TRIAGE) : null, + fixes: parseFixes(FIXES), + }; +} + +const MD = [ + "# Analysis", + "", + "## TL;DR", + "Top part.", + "", + "## Findings", + "Bottom part.", + "", + "## Quick Wins", + "More.", + "", +].join("\n"); + +describe("splitAnalysis", () => { + test("no heading: no split", () => { + expect(splitAnalysis("# A\n\n## TL;DR\ntext\n")).toBeNull(); + }); + + test("one heading: cuts at the Findings line", () => { + const s = splitAnalysis(MD)!; + expect(s.top).toBe("# Analysis\n\n## TL;DR\nTop part.\n\n"); + expect(s.findings.startsWith("## Findings\nBottom part.")).toBe(true); + expect(s.top + s.findings).toBe(MD); + }); + + test("cuts at the first of two", () => { + const s = splitAnalysis("a\n## Findings\nb\n## Findings\nc")!; + expect(s.top).toBe("a\n"); + expect(s.findings).toBe("## Findings\nb\n## Findings\nc"); + }); + + test("a Findings line inside a fenced code block is not a split point", () => { + const md = "x\n```md\n## Findings\n```\ny\n~~~\n## Findings\n~~~\nz\n"; + expect(splitAnalysis(md)).toBeNull(); + const s = splitAnalysis(`${md}## Findings\nreal\n`)!; + expect(s.findings).toBe("## Findings\nreal\n"); + }); + + test("### Findings, a suffix or leading spaces are not split points", () => { + expect(splitAnalysis("### Findings\n")).toBeNull(); + expect(splitAnalysis("## Findings (3)\n")).toBeNull(); + expect(splitAnalysis(" ## Findings\n")).toBeNull(); + expect(splitAnalysis("## Findings \n")).toBeNull(); + }); + + test("CRLF line endings still split", () => { + const s = splitAnalysis("a\r\n## Findings\r\nb")!; + expect(s.top).toBe("a\r\n"); + }); +}); + +describe("parseDoctorStatus", () => { + test("reads the fixture", () => { + const s = parseDoctorStatus(STATUS)!; + expect(s.state).toBe("done"); + expect(s.triageFile).toBe("triage-1000001.json"); + expect(s.fixesDir).toBe("fixes-1000001"); + }); + + test("schema_version 2 is absent", () => { + expect(parseDoctorStatus(statusWith({ schema_version: 2 }))).toBeNull(); + }); + + test("unparsable JSON, a non-object or an unknown state is absent", () => { + expect(parseDoctorStatus("{not json")).toBeNull(); + expect(parseDoctorStatus("[1]")).toBeNull(); + expect(parseDoctorStatus(null)).toBeNull(); + expect(parseDoctorStatus(statusWith({ state: "exploded" }))).toBeNull(); + }); + + test("a bad triage_file name is treated as absent", () => { + for (const bad of [ + "../run.json", + "triage-1/../../x.json", + "summary.json", + "triage-.json", + "triage-1.json.bak", + ]) { + expect(parseDoctorStatus(statusWith({ triage_file: bad }))!.triageFile).toBeNull(); + } + }); + + test("a bad fixes_dir is treated as absent; the old shape has none", () => { + expect(parseDoctorStatus(statusWith({ fixes_dir: "../x" }))!.fixesDir).toBeNull(); + expect(parseDoctorStatus(statusWith({ fixes_dir: "fixes" }))!.fixesDir).toBe("fixes"); + const old = JSON.parse(STATUS); + delete old.fixes_dir; + delete old.fixes; + delete old.trigger; + const s = parseDoctorStatus(JSON.stringify(old))!; + expect(s.fixesDir).toBeNull(); + expect(s.trigger).toBeNull(); + }); +}); + +describe("parseTriage / parseFixes", () => { + test.each([ + ["no counts", {}], + ["a string count", { rows_failed: "5", rows_diagnosed: 3 }], + ["a negative count", { rows_failed: -1, rows_diagnosed: 3 }], + ["a fractional count", { rows_failed: 5, rows_diagnosed: 2.5 }], + ])("a triage with %s does not parse (never a false 0)", (_, summary) => { + const t = JSON.parse(TRIAGE); + t.summary = { ...t.summary, rows_failed: undefined, rows_diagnosed: undefined, ...summary }; + expect(parseTriage(JSON.stringify(t))).toBeNull(); + }); + + test("the run ids are kept, for readRunDoctor's check", () => { + expect(parseTriage(TRIAGE)!.runId).toBe("2099-01-03_04-23-30"); + expect(parseFixes(FIXES)!.runId).toBe("2099-01-03_04-23-30"); + }); + + test.each([ + [{ n: 3.5, killed: 1 }], + [{ n: 3, killed: -1 }], + [{ n: "3", killed: 1 }], + ])("mutants %j are not shown as counts", (mutants) => { + const doc = JSON.parse(FIXES); + doc.candidates[0].gates = { ...doc.candidates[0].gates, mutants }; + expect(parseFixes(JSON.stringify(doc))!.candidates[0].gates.mutants).toBeNull(); + }); + + test("reads the fixture: the card fields", () => { + const t = parseTriage(TRIAGE)!; + expect(t.tasks.map((x) => x.taskId)).toEqual([ + "example-e1", + "example-t1", + "example-t2", + "example-e2", + "example-e3", + ]); + expect(t.rowsRun).toBe(10); + expect(t.rowsFailed).toBe(5); + expect(t.actions[0]).toEqual({ + count: 3, + text: expect.stringContaining("Reauthorize"), + taskIds: ["example-e1", "example-e2", "example-e3"], + }); + const t1 = t.tasks[1]; + expect(t1).toMatchObject({ + layer: "grader", + solvable: "yes-grader", + runStatus: "FAILURE", + confidence: "high", + criterion: "file_contains", + passedBefore: "2099-01-01_04-20-00 (claude-code)", + envGroup: null, + doctorPr: null, + }); + // Only allowlisted PR URLs survive the parser (javascript: and a + // look-alike host are dropped). + expect(t1.existingPrs.map((p) => p.url)).toEqual(["https://github.com/UiPath/skills/pull/123"]); + expect(t1).not.toHaveProperty("related"); + expect(t1).not.toHaveProperty("relatedSkillPrs"); + // open-pr actions (PRs that merely touch the files) are left out. + expect(t.actions.map((a) => a.text)).not.toContainEqual(expect.stringMatching(/^Open PR/)); + expect(t.actions).toHaveLength(2); + expect(t.tasks[0].envGroup).toBe("connection example-conn"); + }); + + test("rows_executed must be a non-negative integer", () => { + const doc = JSON.parse(TRIAGE); + for (const [v, want] of [[10, 10], [0, 0], [-1, null], [2.5, null], ["10", null], [null, null]] as const) { + doc.run.rows_executed = v; + expect(parseTriage(JSON.stringify(doc))!.rowsRun).toBe(want); + } + }); + + test("schema_version 2 or a broken triage is absent", () => { + expect(parseTriage(JSON.stringify({ ...JSON.parse(TRIAGE), schema_version: 2 }))).toBeNull(); + expect(parseTriage("{")).toBeNull(); + expect(parseTriage(JSON.stringify({ schema_version: 1 }))).toBeNull(); + }); + + test("fixed_by_pr: parsed when well-formed, else null", () => { + const doc = JSON.parse(TRIAGE); + expect(parseTriage(TRIAGE)!.tasks[2].fixedByPr).toEqual({ + number: 126, + verdict: "fixes", + basis: expect.stringContaining("documents the current flag"), + }); + for (const bad of [ + { number: "126", verdict: "fixes", basis: "" }, + { number: 0, verdict: "fixes", basis: "" }, + { number: 1.5, verdict: "fixes", basis: "" }, + { number: 126, verdict: "yes", basis: "" }, + { number: 126 }, + [126, "fixes"], + "126", + ]) { + doc.tasks[2].fixed_by_pr = bad; + expect(parseTriage(JSON.stringify(doc))!.tasks[2].fixedByPr).toBeNull(); + } + doc.tasks[2].fixed_by_pr = { number: 5, verdict: "may_help", basis: 7 }; + expect(parseTriage(JSON.stringify(doc))!.tasks[2].fixedByPr).toEqual({ + number: 5, + verdict: "may_help", + basis: "", + }); + }); + + test("pr_checks: head_sha is kept when it is a 40-hex sha", () => { + const doc = JSON.parse(FIXES); + const sha = "0123456789abcdef0123456789abcdef01234567"; + const base = doc.pr_checks["example-t2"]; + doc.pr_checks = { + a: { ...base, head_sha: sha }, + b: { ...base, head_sha: null }, + c: { ...base, head_sha: "not-a-sha" }, + d: { ...base }, + }; + const c = parseFixes(JSON.stringify(doc))!.prChecks; + expect(Object.keys(c).sort()).toEqual(["a", "b", "c", "d"]); + expect([c.a!.headSha, c.b!.headSha, c.c!.headSha, c.d!.headSha]).toEqual([sha, null, null, null]); + }); + + test("pr_checks: well-formed entries by task; malformed ones skipped", () => { + const doc = JSON.parse(FIXES); + const ok = { number: 5, url: "https://github.com/UiPath/skills/pull/5", verdict: "untested", evidence: "e" }; + doc.pr_checks = { + good: { ...ok, reason: "needs-windows-run" }, + odd_reason: { ...ok, reason: "something-else" }, + bad_host: { ...ok, url: "https://github.com.example.invalid/UiPath/skills/pull/5" }, + wrong_number: { ...ok, url: "https://github.com/UiPath/skills/pull/6" }, + bad_verdict: { ...ok, verdict: "fixed" }, + no_number: { ...ok, number: null }, + not_obj: "x", + }; + // A "__proto__" task id, as JSON text (an object literal would set the prototype). + const raw = JSON.stringify(doc).replace('"pr_checks":{', `"pr_checks":{"__proto__":${JSON.stringify(ok)},`); + const checks = parseFixes(raw)!.prChecks; + // Malformed entries stay as null, so their task shows no PR row. + expect(Object.keys(checks).sort()).toEqual( + ["__proto__", "good", "odd_reason", "bad_host", "wrong_number", "bad_verdict", "no_number", "not_obj"].sort(), + ); + for (const k of ["bad_host", "wrong_number", "bad_verdict", "no_number", "not_obj"]) { + expect(checks[k]).toBeNull(); + } + expect(checks.__proto__).not.toBeNull(); + expect(checks.good!.reason).toBe("needs-windows-run"); + // head_sha is optional: no key, null, or a malformed value all parse to null. + expect(checks.good!.headSha).toBeNull(); + expect(checks.odd_reason!.reason).toBeNull(); + expect(Object.getPrototypeOf(checks)).toBe(Object.prototype); + doc.pr_checks = "nope"; + expect(parseFixes(JSON.stringify(doc))!.prChecks).toEqual({}); + delete doc.pr_checks; + expect(parseFixes(JSON.stringify(doc))!.prChecks).toEqual({}); + }); + + test("doctor_pr is kept only when allowlisted", () => { + const doc = JSON.parse(TRIAGE); + doc.tasks[1].doctor_pr = "https://github.com/UiPath/skills/pull/9"; + doc.tasks[2].doctor_pr = "https://github.com.example.invalid/UiPath/skills/pull/9"; + const t = parseTriage(JSON.stringify(doc))!; + expect(t.tasks[1].doctorPr).toBe("https://github.com/UiPath/skills/pull/9"); + expect(t.tasks[2].doctorPr).toBeNull(); + }); + + test("parseOpened: prototype-named task ids are own keys", () => { + const o = parseOpened( + JSON.stringify({ + opened: [ + { task_id: "__proto__", pr_url: "https://github.com/UiPath/skills/pull/1" }, + { task_id: "constructor", pr_url: "https://github.com/UiPath/skills/pull/2" }, + ], + }), + )!; + expect(Object.hasOwn(o, "__proto__")).toBe(true); + expect(o.constructor).toBe("https://github.com/UiPath/skills/pull/2"); + expect(Object.getPrototypeOf(o)).toBe(Object.prototype); + }); + + test("parseOpened: allowlisted PR URLs map by task; the rest is skipped", () => { + expect( + parseOpened( + JSON.stringify({ + opened: [ + { task_id: "a", pr_url: "https://github.com/UiPath/skills/pull/300", branch: "x" }, + { task_id: "b", pr_url: "https://github.com.example.invalid/UiPath/skills/pull/301" }, + { task_id: "c", pr_url: "javascript:alert(1)" }, + { task_id: "d", problem: "push refused" }, + { pr_url: "https://github.com/UiPath/skills/pull/302" }, + "junk", + ], + error: "1 of 5 failed", + }), + ), + ).toEqual({ a: "https://github.com/UiPath/skills/pull/300" }); + expect(parseOpened("{oops")).toBeNull(); + expect(parseOpened(JSON.stringify({ opened: "x" }))).toBeNull(); + expect(parseOpened(JSON.stringify({ opened: [] }))).toEqual({}); + }); + + test("fixes.json with schema_version 2, or none, is absent", () => { + const doc = JSON.parse(FIXES); + expect(parseFixes(JSON.stringify({ ...doc, schema_version: 2 }))).toBeNull(); + delete doc.schema_version; + expect(parseFixes(JSON.stringify(doc))).toBeNull(); + }); + + test("fixes: candidates, gates and patch ids", () => { + const f = parseFixes(FIXES)!; + expect(f.candidates.map((c) => c.outcome)).toEqual(["pr-ready", "needs-live-run"]); + expect(f.candidates[0].gates).toEqual({ replay: "pass", mutants: { n: 10, killed: 9 }, review: "SEND" }); + expect([...patchIds(f)]).toEqual(["example-grader-fix"]); + }); + + test("a candidate id that is not a safe file name gets no patch", () => { + const doc = JSON.parse(FIXES); + doc.candidates[0].id = "../../run"; + expect(patchIds(parseFixes(JSON.stringify(doc))).size).toBe(0); + }); + + test("patchIdFromPath", () => { + expect(patchIdFromPath("fixes-1/a.patch")).toEqual({ fixesDir: "fixes-1", id: "a" }); + expect(patchIdFromPath("fixes/a.b.patch")).toEqual({ fixesDir: "fixes", id: "a.b" }); + for (const bad of ["fixes-1/fixes.json", "fixes-1/.patch", "x/a.patch", "fixes-1/../a.patch", "a.patch", "fixes-1/a/b.patch"]) { + expect(patchIdFromPath(bad)).toBeNull(); + } + }); +}); + +describe("link validation", () => { + test("skills PR links", () => { + expect(isSkillsPrUrl("https://github.com/UiPath/skills/pull/123")).toBe(true); + for (const bad of [ + "javascript:alert(1)", + "http://github.com/UiPath/skills/pull/123", + "https://github.com/UiPath/skills/pull/123/files", + "https://github.com/UiPath/skills/pull/123?x=1", + "https://github.com.example.invalid/UiPath/skills/pull/123", + "https://github.com/UiPath/skills-fork/pull/123", + "https://github.com/Other/skills/pull/123", + "https://evil.example/https://github.com/UiPath/skills/pull/1", + " https://github.com/UiPath/skills/pull/123", + ]) { + expect(isSkillsPrUrl(bad)).toBe(false); + } + }); + + test("build links", () => { + expect(isAdoUrl("https://dev.azure.com/example-org/example-project/_build/results?buildId=1")).toBe(true); + for (const bad of [ + "javascript:alert(1)", + "http://dev.azure.com/example-org/x", + "https://dev.azure.com.example.invalid/x", + "https://dev.azure.com@example.invalid/x", + "https://example.invalid/dev.azure.com/x", + "https://dev.azure.com/x\"onmouseover=\"y", + null, + ]) { + expect(isAdoUrl(bad)).toBe(false); + } + }); +}); + +describe("isLinkableTaskId", () => { + test("accepts the ids the task page accepts", () => { + expect(isLinkableTaskId("example-t1")).toBe(true); + expect(isLinkableTaskId("suite/row-3")).toBe(true); + }); + test("rejects path tricks, URLs and odd characters", () => { + for (const bad of ["", "../x", "a/../b", "a/./b", "/abs", "a//b", "https://x", "a b", "a?b", "a#b", "x".repeat(256)]) { + expect(isLinkableTaskId(bad)).toBe(false); + } + }); +}); + +describe("taskAnalysisEnabled (gating + kill switch)", () => { + test("internal + skills + switch not off", () => { + expect(taskAnalysisEnabled(true, "skills", undefined)).toBe(true); + expect(taskAnalysisEnabled(true, "skills", "")).toBe(true); + expect(taskAnalysisEnabled(true, "skills", "on")).toBe(true); + }); + test("off, OSS or another source", () => { + expect(taskAnalysisEnabled(true, "skills", "off")).toBe(false); + expect(taskAnalysisEnabled(true, "skills", "OFF")).toBe(false); + expect(taskAnalysisEnabled(true, "skills", " Off ")).toBe(false); + expect(taskAnalysisEnabled(false, "skills", undefined)).toBe(false); + expect(taskAnalysisEnabled(true, "scribe", undefined)).toBe(false); + expect(taskAnalysisEnabled(true, "gha", undefined)).toBe(false); + }); +}); + +describe("planAnalysis (states)", () => { + test("absent: today's view, no block", () => { + expect(planAnalysis(MD, null)).toEqual({ summary: MD, doctor: null }); + expect(planAnalysis(null, null)).toEqual({ summary: null, doctor: null }); + }); + + test("done with a split: summary keeps the top, the block replaces the rest", () => { + const d = doctor(); + const p = planAnalysis(MD, d); + expect(p.summary).toBe("# Analysis\n\n## TL;DR\nTop part.\n\n"); + expect(p.doctor).toBe(d); + }); + + test("done without a split: the full analysis, then the block", () => { + const md = "# A\n\n## TL;DR\nonly\n"; + const d = doctor(); + expect(planAnalysis(md, d)).toEqual({ summary: md, doctor: d }); + }); + + test("done without analysis.md: the block alone", () => { + const d = doctor(); + expect(planAnalysis(null, d)).toEqual({ summary: null, doctor: d }); + }); + + test("an analysis.md that starts with Findings: no summary", () => { + const d = doctor(); + expect(planAnalysis("## Findings\nx\n", d)).toEqual({ summary: null, doctor: d }); + }); + + test("done but the triage did not load: the full analysis", () => { + const d = { ...doctor(), triage: null }; + expect(planAnalysis(MD, d)).toEqual({ summary: MD, doctor: d }); + }); + + test.each(["queued", "running", "failed", "skipped", "retracted"])( + "%s: the full analysis (Findings included) and the block", + (state) => { + const d = doctor({ state, reason: "why" }); + expect(planAnalysis(MD, d)).toEqual({ summary: MD, doctor: d }); + }, + ); +}); diff --git a/evalboard/lib/__tests__/fixtures/skill-doctor/fixes-1000001/example-grader-fix.patch b/evalboard/lib/__tests__/fixtures/skill-doctor/fixes-1000001/example-grader-fix.patch new file mode 100644 index 00000000..730e99e9 --- /dev/null +++ b/evalboard/lib/__tests__/fixtures/skill-doctor/fixes-1000001/example-grader-fix.patch @@ -0,0 +1,9 @@ +diff --git a/tests/tasks/example/example.yaml b/tests/tasks/example/example.yaml +--- a/tests/tasks/example/example.yaml ++++ b/tests/tasks/example/example.yaml +@@ -1,2 +1,4 @@ +-expected: literal ++expected: documented ++# example ++# example + other: value diff --git a/evalboard/lib/__tests__/fixtures/skill-doctor/fixes-1000001/fixes.json b/evalboard/lib/__tests__/fixtures/skill-doctor/fixes-1000001/fixes.json new file mode 100644 index 00000000..8815d293 --- /dev/null +++ b/evalboard/lib/__tests__/fixtures/skill-doctor/fixes-1000001/fixes.json @@ -0,0 +1,93 @@ +{ + "schema_version": 1, + "run_id": "2099-01-03_04-23-30", + "fix_mode": "dryrun", + "base_sha": "0a1b2c3d4e", + "run_commit": "0a1b2c3d", + "pr_data": "gh", + "considered": 2, + "cost_usd": 2.5, + "total_cap_usd": 60, + "generated_at": "2099-01-03T09:20:00Z", + "skipped": [], + "candidates": [ + { + "id": "example-grader-fix", + "task_id": "example-t1", + "covers": [ + "example-t1", + "example-t3" + ], + "layer": "grader", + "outcome": "pr-ready", + "cost_usd": 1.5, + "patch_lines": { + "added": 3, + "removed": 1 + }, + "title": "fix(example): match the documented output", + "problem": "", + "files": [ + "tests/tasks/example/example.yaml" + ], + "overlap": [], + "open_prs_on_files": [], + "gates": { + "replay": { + "status": "pass", + "table": "", + "evidence": "" + }, + "mutants": { + "n": 10, + "killed": 9, + "survivors": 1, + "pre_existing": 0 + }, + "flip_judge": null, + "genuine": null, + "review": "SEND", + "holes": null, + "history": [] + } + }, + { + "id": "example-skill-fix", + "task_id": "example-t2", + "covers": [ + "example-t2" + ], + "layer": "skill", + "outcome": "needs-live-run", + "cost_usd": 1.0, + "patch_lines": null, + "title": "", + "problem": "blocked-env", + "files": [], + "overlap": [], + "open_prs_on_files": [], + "gates": { + "replay": { + "status": "blocked-env", + "table": "", + "evidence": "" + }, + "mutants": null, + "flip_judge": null, + "genuine": null, + "review": null, + "holes": null, + "history": [] + } + } + ], + "pr_checks": { + "example-t2": { + "number": 126, + "url": "https://github.com/UiPath/skills/pull/126", + "verdict": "verified", + "claim": "fixes", + "evidence": "replay of #126 on 2 stored runs: run_command flips to pass on both" + } + } +} diff --git a/evalboard/lib/__tests__/fixtures/skill-doctor/schemas/README.md b/evalboard/lib/__tests__/fixtures/skill-doctor/schemas/README.md new file mode 100644 index 00000000..017986cd --- /dev/null +++ b/evalboard/lib/__tests__/fixtures/skill-doctor/schemas/README.md @@ -0,0 +1,11 @@ +# skill-doctor contract v1 schemas + +Copies of the producer's JSON Schemas (`src/skill_doctor/schemas/` in the +skill-doctor repo), taken 2026-11-20. `doctor-schema.test.ts` checks that every +field `lib/doctor.ts` reads is declared here and that the hand-written fixtures +use only declared keys, so a producer-side rename fails a test once these are +refreshed. Refresh them whenever the doctor's contract changes. + +fixes_v1 pins only `id`, `task_id` and `outcome` of a candidate; the other +candidate fields the board reads (`covers`, `patch_lines`, `title`, `gates`) +are not pinned by the producer yet. diff --git a/evalboard/lib/__tests__/fixtures/skill-doctor/schemas/fixes_v1.json b/evalboard/lib/__tests__/fixtures/skill-doctor/schemas/fixes_v1.json new file mode 100644 index 00000000..2d1a5322 --- /dev/null +++ b/evalboard/lib/__tests__/fixtures/skill-doctor/schemas/fixes_v1.json @@ -0,0 +1,57 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "skill-doctor/fixes_v1.json", + "title": "runs//skill-doctor/fixes-/fixes.json (nightly-fix emit; contract v1, partial)", + "description": "The Fix stage's outcome per candidate, the skipped rows, and pr_checks: the replay check of the open PR each row's diagnosis names as its fix (fixed_by_pr). Only the keys below are pinned; candidates carry more fields (gates, patch lines).", + "type": "object", + "required": ["schema_version", "run_id", "fix_mode", "candidates", "skipped", "cost_usd"], + "properties": { + "schema_version": {"const": 1}, + "run_id": {"type": "string"}, + "fix_mode": {"enum": ["off", "dryrun", "open"]}, + "base_sha": {"type": ["string", "null"]}, + "cost_usd": {"type": "number", "minimum": 0}, + "max_prs": {"type": ["integer", "null"], "description": "the cap on PRs OPENED by this run (rank order); not a selection cut"}, + "history_runs": {"type": "array", "items": {"type": "string"}, "description": "the earlier nights the memory read (newest first)"}, + "candidates": {"type": "array", "items": {"type": "object", "required": ["id", "task_id", "outcome"], + "properties": {"outcome": {"type": "string", "description": "pr-ready, needs-live-run, needs-windows-run (a deciding criterion runs Windows-only tooling on a windows-tagged task), held-overlap, body-lint-failed, no-budget (not tried: the run's spend cap was used up; it rolls to the next night), ..."}, + "rank": {"type": "integer", "minimum": 1, "description": "select's order: rows covered, confidence, pr-ready on an earlier night, stable diagnosis, task id"}, + "reused_from": {"$ref": "#/$defs/record", "description": "present when the patch of an earlier night was reused (rebased, model-free gates passed; no session, $0)"}, + "memory_note": {"type": "string", "description": "what the earlier nights said about this candidate"}, + "inputs": {"type": "object", "additionalProperties": {"type": ["string", "null"]}, "description": "git blob id per input file at the base (task yaml, its check files, the skill files the diagnosis names)"}, + "inputs_hash": {"type": "string"}, + "contested": {"type": "array", "items": {"type": "string"}, "description": "the workflow's collision list (\"#N open (author)\", \" merged (author)\", \"contested by #N\")"}, + "session_cap_usd": {"type": "number", "description": "the cap the candidate's session started with"}, + "diagnosis_flip": {"type": ["object", "null"], "description": "{night, from, to}: the layer/solvable changed since the previous night"}}}}, + "skipped": {"type": "array", "items": {"type": "object", "additionalProperties": false, "required": ["task_id", "reason"], + "properties": {"task_id": {"type": "string"}, "reason": {"type": "string"}, + "pr_url": {"type": "string", "pattern": "^https://github\\.com/UiPath/skills/pull/\\d+$", "description": "pr-open: the open doctor PR"}, + "skip_reason": {"enum": ["pr-open", "rejected-unchanged"], "description": "a memory skip ($0): a doctor PR with the fix is still open, or an earlier night rejected the fix and its inputs are unchanged"}, + "reused_from": {"$ref": "#/$defs/record"}}}}, + "pr_checks": {"type": "object", "additionalProperties": {"$ref": "#/$defs/pr_check"}} + }, + "$defs": { + "record": { + "type": "object", + "additionalProperties": false, + "required": ["run_id", "fixes_dir", "id", "outcome"], + "description": "the earlier night's fixes.json candidate a memory decision used: runs//skill-doctor//", + "properties": {"run_id": {"type": "string"}, "fixes_dir": {"type": "string", "pattern": "^fixes(-[0-9]+)?$"}, + "id": {"type": ["string", "null"]}, "outcome": {"type": ["string", "null"]}} + }, + "pr_check": { + "type": "object", + "additionalProperties": false, + "required": ["number", "url", "verdict", "evidence"], + "properties": { + "number": {"type": "integer", "minimum": 1}, + "url": {"type": "string", "pattern": "^https://github\\.com/UiPath/skills/pull/\\d+$"}, + "head_sha": {"type": ["string", "null"], "pattern": "^[0-9a-f]{40}$", "description": "the PR head the check applied (refs/pull//head as fetched)"}, + "verdict": {"enum": ["verified", "not_verified", "untested"]}, + "evidence": {"type": "string", "maxLength": 600}, + "claim": {"enum": ["fixes", "may_help"]}, + "reason": {"enum": ["needs-windows-run"], "description": "why an untested check could not be replayed, when the board should say so"} + } + } + } +} diff --git a/evalboard/lib/__tests__/fixtures/skill-doctor/schemas/status_v1.json b/evalboard/lib/__tests__/fixtures/skill-doctor/schemas/status_v1.json new file mode 100644 index 00000000..201e64be --- /dev/null +++ b/evalboard/lib/__tests__/fixtures/skill-doctor/schemas/status_v1.json @@ -0,0 +1,48 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "skill-doctor/status_v1.json", + "title": "runs//skill-doctor/status.json (contract v1)", + "description": "Pointer and lock for one nightly run's doctor output. Written first as a create-if-absent lock (state running), last on success. triage_file names the immutable triage-.json next to it.", + "type": "object", + "additionalProperties": false, + "required": ["schema_version", "state", "reason", "run_id", "harness", "model", "history_base_run", "run_json_etag", + "started_at", "finished_at", "doctor_version", "build_id", "build_url", "cost_usd", "rows_failed", + "rows_diagnosed", "triage_file"], + "properties": { + "schema_version": {"const": 1}, + "state": {"enum": ["queued", "running", "done", "failed", "skipped", "retracted"]}, + "reason": {"type": "string", "maxLength": 500}, + "run_id": {"type": "string", "pattern": "^[A-Za-z0-9][A-Za-z0-9._-]{0,99}$"}, + "harness": {"type": ["string", "null"]}, + "model": {"type": ["string", "null"]}, + "adhoc": {"type": "boolean"}, + "triage_only": {"type": "boolean"}, + "history_base_run": {"type": ["string", "null"]}, + "run_json_etag": {"type": ["string", "null"]}, + "started_at": {"type": "string"}, + "finished_at": {"type": ["string", "null"]}, + "updated_at": {"type": "string"}, + "doctor_version": {"type": "string"}, + "build_id": {"type": ["string", "null"]}, + "build_url": {"type": ["string", "null"]}, + "cost_usd": {"type": "number", "minimum": 0}, + "cost_complete": {"type": "boolean", "description": "false when a session's cost is unknown (timed out / killed): cost_usd counts it as 0, reason says which group"}, + "rows_failed": {"type": ["integer", "null"], "minimum": 0}, + "rows_diagnosed": {"type": "integer", "minimum": 0}, + "triage_file": {"type": ["string", "null"], "pattern": "^triage-[A-Za-z0-9._-]+\\.json$"}, + "trigger": {"type": ["string", "null"], "pattern": "^https://dev\\.azure\\.com/[A-Za-z0-9-]+/[A-Za-z0-9._%-]+/_build/results\\?buildId=[0-9]+$", + "description": "the nightly build that queued this run (the sourceBuild parameter); null when queued by hand"}, + "fixes_dir": {"type": ["string", "null"], "pattern": "^fixes(-[0-9]+)?$", + "description": "the Fix stage's create-only dir next to this file (fixes.json, .patch, .md); set by FixEmit"}, + "fixes_dirs": {"type": "array", "items": {"type": "string", "pattern": "^fixes(-[0-9]+)?$"}, + "description": "every Fix stage's dir of this run, oldest first (fixes_dir is the newest): the memory of the next nights reads them all"}, + "fixes": {"type": "object", "additionalProperties": false, "required": ["fix_mode", "cost_usd", "counts"], + "description": "FixEmit's digest of fixes.json: candidates per outcome (pr-ready, needs-live-run, held-overlap, ...)", + "properties": { + "fix_mode": {"enum": ["dryrun", "open"]}, + "cost_usd": {"type": "number", "minimum": 0}, + "total_cap_usd": {"type": ["number", "null"], "minimum": 0}, + "counts": {"type": "object", "additionalProperties": {"type": "integer", "minimum": 0}} + }} + } +} diff --git a/evalboard/lib/__tests__/fixtures/skill-doctor/schemas/triage_v1.json b/evalboard/lib/__tests__/fixtures/skill-doctor/schemas/triage_v1.json new file mode 100644 index 00000000..7cffd7ee --- /dev/null +++ b/evalboard/lib/__tests__/fixtures/skill-doctor/schemas/triage_v1.json @@ -0,0 +1,145 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "skill-doctor/triage_v1.json", + "title": "runs//skill-doctor/triage->.json (contract v1)", + "description": "One nightly run's failed rows: the $0 triage (class, owners, recurrence), the optional model diagnosis (layer ... passed_before), fix scouting (existing_prs, doctor_pr) and cross-task actions. All text is scrubbed (emails, tenant ids, token shapes). Immutable once written.", + "type": "object", + "additionalProperties": false, + "required": ["schema_version", "run", "summary", "tasks", "actions", "disclaimer"], + "properties": { + "schema_version": {"const": 1}, + "disclaimer": {"type": "string"}, + "run": { + "type": "object", + "additionalProperties": false, + "required": ["run_id", "harness", "model", "skills_git_commit", "cli_version", "tasks_run", "rows_executed", "adhoc", + "history_base_run", "run_json_etag", "build_id", "doctor_version", "generated_at", "mode", "mode_reason"], + "properties": { + "run_id": {"type": "string"}, + "harness": {"type": ["string", "null"]}, + "model": {"type": ["string", "null"]}, + "skills_git_commit": {"type": ["string", "null"]}, + "cli_version": {"type": ["string", "null"]}, + "tasks_run": {"type": ["integer", "null"]}, + "rows_executed": {"type": ["integer", "null"]}, + "adhoc": {"type": "boolean"}, + "history_base_run": {"type": ["string", "null"]}, + "run_json_etag": {"type": ["string", "null"]}, + "build_id": {"type": ["string", "null"]}, + "build_url": {"type": ["string", "null"]}, + "doctor_version": {"type": "string"}, + "generated_at": {"type": "string"}, + "mode": {"enum": ["triage-only", "diagnosed"]}, + "mode_reason": {"type": "string"}, + "cost_usd": {"type": "number", "minimum": 0}, + "pr_data": {"type": "string"} + } + }, + "summary": { + "type": "object", + "additionalProperties": false, + "required": ["rows_failed", "rows_diagnosed", "by_class", "by_layer", "vs_history", "env_groups"], + "properties": { + "rows_failed": {"type": "integer", "minimum": 0}, + "rows_diagnosed": {"type": "integer", "minimum": 0}, + "rows_diagnosis_rejected": {"type": "integer", "minimum": 0}, + "by_class": {"type": "object", "additionalProperties": {"type": "integer"}}, + "by_layer": {"type": "object", "additionalProperties": {"type": "integer"}}, + "vs_history": { + "type": "object", + "additionalProperties": false, + "required": ["new", "persisting", "fixed", "base_run"], + "properties": { + "new": {"type": "integer", "minimum": 0}, + "persisting": {"type": "integer", "minimum": 0}, + "fixed": {"type": "integer", "minimum": 0}, + "fixed_tasks": {"type": "array", "items": {"type": "string"}}, + "base_run": {"type": ["string", "null"]} + } + }, + "env_groups": { + "type": "array", + "items": { + "type": "object", + "additionalProperties": false, + "required": ["cause", "count", "task_ids"], + "properties": { + "cause": {"type": "string"}, + "count": {"type": "integer", "minimum": 1}, + "task_ids": {"type": "array", "items": {"type": "string"}}, + "detail": {"type": "string"}, + "fix": {"type": "string"} + } + } + } + } + }, + "tasks": {"type": "array", "items": {"$ref": "#/$defs/task"}}, + "actions": { + "type": "array", + "items": { + "type": "object", + "additionalProperties": false, + "required": ["kind", "count", "text", "task_ids"], + "properties": { + "kind": {"enum": ["env", "open-pr", "upstream", "layer"]}, + "count": {"type": "integer", "minimum": 1}, + "text": {"type": "string"}, + "task_ids": {"type": "array", "items": {"type": "string"}} + } + } + } + }, + "$defs": { + "fixed_by_pr": {"type": ["object", "null"], "additionalProperties": false, "required": ["number", "verdict", "basis"], "description": "The diagnosis's judgement of the open PR that best fixes this row (an opinion, from its diff); the Fix stage's replay check of it is fixes.json pr_checks[task_id]", "properties": {"number": {"type": "integer", "minimum": 1}, "verdict": {"enum": ["fixes", "may_help", "unrelated"]}, "basis": {"type": "string", "maxLength": 300}}}, + "pr": { + "type": "object", + "additionalProperties": false, + "required": ["url", "title", "state"], + "properties": { + "url": {"type": "string", "pattern": "^https://github\\.com/UiPath/skills/pull/\\d+$"}, + "title": {"type": "string", "maxLength": 300}, + "state": {"enum": ["open", "merged", "merged-after-run", "unknown", "closed"]} + } + }, + "task": { + "type": "object", + "additionalProperties": false, + "required": ["task_id", "skill", "class", "family", "state", "nights_seen", "owners", "run_status", "criterion", "error", + "diagnosed", "layer", "confidence", "why", "evidence", "solvable", "fix", "related", "passed_before", + "existing_prs", "doctor_pr"], + "properties": { + "task_id": {"type": "string"}, + "skill": {"type": "string"}, + "class": {"enum": ["env", "harness-error", "turn-cap", "timeout", "failure", "known-upstream", "contested"]}, + "family": {"type": "string"}, + "state": {"enum": ["new", "changed", "known"]}, + "nights_seen": {"type": "integer", "minimum": 1}, + "owners": {"type": "array", "items": {"type": "string"}}, + "run_status": {"type": ["string", "null"]}, + "reps_failed": {"type": "integer", "minimum": 0}, + "criterion": {"type": ["string", "null"]}, + "error": {"type": ["string", "null"]}, + "contested": {"type": ["array", "null"], "items": {"type": "string"}}, + "known_upstream": {"type": ["string", "null"]}, + "recurrence": {"type": ["object", "null"]}, + "previous_nights": {"type": ["object", "null"]}, + "env_group": {"type": ["string", "null"]}, + "diagnosed": {"type": "boolean"}, + "diagnosis_skipped": {"type": ["string", "null"]}, + "layer": {"enum": ["agent", "skill", "grader", "task", "env", "upstream", "flaky", "undiagnosed"]}, + "confidence": {"type": ["string", "null"]}, + "why": {"type": ["string", "null"]}, + "evidence": {"type": ["string", "null"]}, + "solvable": {"type": ["string", "null"]}, + "fix": {"type": ["string", "null"]}, + "related": {"type": ["string", "null"]}, + "passed_before": {"type": ["string", "null"]}, + "existing_prs": {"type": ["array", "null"], "items": {"$ref": "#/$defs/pr"}}, + "related_skill_prs": {"type": ["array", "null"], "items": {"$ref": "#/$defs/pr"}}, + "doctor_pr": {"type": ["string", "null"], "pattern": "^https://github\\.com/UiPath/skills/pull/\\d+$"}, + "fixed_by_pr": {"$ref": "#/$defs/fixed_by_pr"} + } + } + } +} diff --git a/evalboard/lib/__tests__/fixtures/skill-doctor/status.json b/evalboard/lib/__tests__/fixtures/skill-doctor/status.json new file mode 100644 index 00000000..d7031bcf --- /dev/null +++ b/evalboard/lib/__tests__/fixtures/skill-doctor/status.json @@ -0,0 +1,26 @@ +{ + "schema_version": 1, + "state": "done", + "reason": "", + "run_id": "2099-01-03_04-23-30", + "harness": "claude-code", + "model": "example-model", + "adhoc": false, + "triage_only": false, + "history_base_run": "2099-01-02_04-25-52", + "run_json_etag": "\"0x8D000000000000A\"", + "started_at": "2099-01-03T09:00:00Z", + "finished_at": "2099-01-03T09:12:00Z", + "updated_at": "2099-01-03T09:20:00Z", + "doctor_version": "0.3.0a0", + "build_id": "1000001", + "build_url": "https://dev.azure.com/example-org/example-project/_build/results?buildId=1000001", + "cost_usd": 1.25, + "cost_complete": true, + "rows_failed": 5, + "rows_diagnosed": 2, + "triage_file": "triage-1000001.json", + "trigger": "https://dev.azure.com/example-org/example-project/_build/results?buildId=1000000", + "fixes_dir": "fixes-1000001", + "fixes": {"fix_mode": "dryrun", "cost_usd": 2.5, "total_cap_usd": 60, "counts": {"pr-ready": 1, "needs-live-run": 1}} +} diff --git a/evalboard/lib/__tests__/fixtures/skill-doctor/triage-1000001.json b/evalboard/lib/__tests__/fixtures/skill-doctor/triage-1000001.json new file mode 100644 index 00000000..03dfe861 --- /dev/null +++ b/evalboard/lib/__tests__/fixtures/skill-doctor/triage-1000001.json @@ -0,0 +1,261 @@ +{ + "schema_version": 1, + "disclaimer": "Probable causes, not verified fixes. Existing PRs are context; only a doctor PR is a proposed fix.", + "run": { + "run_id": "2099-01-03_04-23-30", + "harness": "claude-code", + "model": "example-model", + "skills_git_commit": "0a1b2c3d", + "cli_version": "1.0.0", + "tasks_run": 10, + "rows_executed": 10, + "adhoc": false, + "history_base_run": "2099-01-02_04-25-52", + "run_json_etag": "\"0x8D000000000000A\"", + "build_id": "1000001", + "build_url": "https://dev.azure.com/example-org/example-project/_build/results?buildId=1000001", + "doctor_version": "0.3.0a0", + "generated_at": "2099-01-03T09:12:00Z", + "mode": "diagnosed", + "mode_reason": "example", + "cost_usd": 1.25 + }, + "summary": { + "rows_failed": 5, + "rows_diagnosed": 2, + "by_class": { + "env": 3, + "failure": 2 + }, + "by_layer": { + "env": 3, + "grader": 1, + "skill": 1 + }, + "vs_history": { + "new": 4, + "persisting": 1, + "fixed": 0, + "fixed_tasks": [], + "base_run": "2099-01-02_04-25-52" + }, + "env_groups": [ + { + "cause": "connection example-conn", + "count": 3, + "task_ids": [ + "example-e1", + "example-e2", + "example-e3" + ] + } + ] + }, + "tasks": [ + { + "task_id": "example-e1", + "skill": "uipath-example", + "class": "env", + "family": "preflight", + "state": "new", + "nights_seen": 1, + "owners": [ + "Example Owner" + ], + "run_status": "ERROR", + "criterion": null, + "error": "pre-run check failed", + "contested": null, + "env_group": "connection example-conn", + "diagnosed": false, + "layer": "env", + "confidence": null, + "why": "example-conn: no enabled connection", + "evidence": null, + "solvable": "env-fix", + "fix": "Reauthorize the connection, then re-run.", + "related": null, + "passed_before": null, + "existing_prs": null, + "doctor_pr": null, + "fixed_by_pr": null + }, + { + "task_id": "example-t1", + "skill": "uipath-example", + "class": "failure", + "family": "grader", + "state": "known", + "nights_seen": 3, + "owners": [ + "Example Owner", + "Second Owner" + ], + "run_status": "FAILURE", + "criterion": "file_contains", + "error": null, + "contested": null, + "env_group": null, + "diagnosed": true, + "layer": "grader", + "confidence": "high", + "why": "The grader expects a literal the task prompt never asks for.", + "evidence": "grader log: pattern `literal` not found", + "solvable": "yes-grader", + "fix": "Match the documented output instead of the literal.", + "related": "example-t3 fails the same check", + "passed_before": "2099-01-01_04-20-00 (claude-code)", + "existing_prs": [ + { + "url": "https://github.com/UiPath/skills/pull/123", + "title": "fix: example grader", + "state": "open" + }, + { + "url": "https://github.com.example.invalid/UiPath/skills/pull/124", + "title": "look-alike host", + "state": "open" + }, + { + "url": "javascript:alert(1)", + "title": "script url", + "state": "open" + } + ], + "doctor_pr": null, + "related_skill_prs": [ + { + "url": "https://github.com/UiPath/skills/pull/125", + "title": "docs(example): document the output", + "state": "merged" + } + ], + "fixed_by_pr": null + }, + { + "task_id": "example-t2", + "skill": "uipath-example-two", + "class": "failure", + "family": "skill", + "state": "new", + "nights_seen": 1, + "owners": [], + "run_status": "FAILURE", + "criterion": "run_command", + "error": null, + "contested": null, + "env_group": null, + "diagnosed": true, + "layer": "skill", + "confidence": "medium", + "why": "The skill documents a removed flag.", + "evidence": null, + "solvable": "yes-skill", + "fix": "Document the current flag.", + "related": null, + "passed_before": null, + "existing_prs": [ + { + "url": "https://github.com/UiPath/skills/pull/126", + "title": "docs(example-two): document the current flag", + "state": "open" + }, + { + "url": "https://github.com/UiPath/skills/pull/127", + "title": "docs: older attempt", + "state": "merged" + } + ], + "doctor_pr": null, + "fixed_by_pr": { + "number": 126, + "verdict": "fixes", + "basis": "The PR documents the current flag in the skill's reference page." + } + }, + { + "task_id": "example-e2", + "skill": "uipath-example-two", + "class": "env", + "family": "preflight", + "state": "new", + "nights_seen": 1, + "owners": [ + "Example Owner" + ], + "run_status": "ERROR", + "criterion": null, + "error": "pre-run check failed", + "contested": null, + "env_group": "connection example-conn", + "diagnosed": false, + "layer": "env", + "confidence": null, + "why": "example-conn: no enabled connection", + "evidence": null, + "solvable": "env-fix", + "fix": "Reauthorize the connection, then re-run.", + "related": null, + "passed_before": null, + "existing_prs": null, + "doctor_pr": null, + "fixed_by_pr": null + }, + { + "task_id": "example-e3", + "skill": "uipath-example-two", + "class": "env", + "family": "preflight", + "state": "new", + "nights_seen": 1, + "owners": [ + "Example Owner" + ], + "run_status": "ERROR", + "criterion": null, + "error": "pre-run check failed", + "contested": null, + "env_group": "connection example-conn", + "diagnosed": false, + "layer": "env", + "confidence": null, + "why": "example-conn: no enabled connection", + "evidence": null, + "solvable": "env-fix", + "fix": "Reauthorize the connection, then re-run.", + "related": null, + "passed_before": null, + "existing_prs": null, + "doctor_pr": null, + "fixed_by_pr": null + } + ], + "actions": [ + { + "kind": "env", + "count": 3, + "text": "Environment: 1 task stopped at the pre-run check. Reauthorize the connection, then re-run.", + "task_ids": [ + "example-e1", + "example-e2", + "example-e3" + ] + }, + { + "kind": "open-pr", + "count": 1, + "text": "Open PR https://github.com/UiPath/skills/pull/123 (fix: example grader) touches the files of 1 failing task. See javascript:alert(1) and https://github.com.example.invalid/x.", + "task_ids": [ + "example-t1" + ] + }, + { + "kind": "layer", + "count": 1, + "text": "uipath-example: 1 task diagnosed as a grader problem.", + "task_ids": [ + "example-t1" + ] + } + ] +} diff --git a/evalboard/lib/blob.ts b/evalboard/lib/blob.ts index ba2a17d8..16b1897a 100644 --- a/evalboard/lib/blob.ts +++ b/evalboard/lib/blob.ts @@ -3,6 +3,8 @@ import path from "node:path"; import { randomBytes } from "node:crypto"; import type { BlobServiceClient, ContainerClient } from "@azure/storage-blob"; import { DEFAULT_VARIANT_ID, isValidVariantId } from "./variants"; +import { isValidId, isValidTaskId } from "./ids"; +import { DOCTOR_DIR } from "./doctor"; const ACCOUNT = "coderevaltests"; @@ -12,32 +14,7 @@ const ACCOUNT = "coderevaltests"; // directly. export const LOCAL_RUNS_DIR = process.env.EVALBOARD_LOCAL_RUNS_DIR || null; -// Run / task IDs get reflected into filesystem paths and blob prefixes, so -// reject anything outside a narrow whitelist before any side effect. -const ID_RE = /^[\w.-]+$/; -// Task IDs from dataset-expanded tasks look like "sentiment-classification/r3" -// (suite/row). Each segment must still pass the narrow ID_RE — only the slash -// separator between segments is additionally allowed. -const TASK_ID_RE = /^[\w.-]+(\/[\w.-]+)*$/; - -export function isValidId(id: unknown): id is string { - return ( - typeof id === "string" && - id.length > 0 && - id.length < 128 && - ID_RE.test(id) - ); -} - -export function isValidTaskId(id: unknown): id is string { - return ( - typeof id === "string" && - id.length > 0 && - id.length < 256 && - TASK_ID_RE.test(id) && - !id.split("/").some((s) => s === "." || s === "..") - ); -} +export { isValidId, isValidTaskId }; function assertValidId(id: string, label: string): void { if (!isValidId(id)) { @@ -342,6 +319,40 @@ export async function ensureRunReviewIndex( }); } +// skill-doctor report sidecars under `/skill-doctor/` (see lib/doctor.ts +// and readRunDoctor, which decides which files a status implies). Download +// only, like the siblings above: status.json is mutable and passes a refresh +// age; the triage file, fixes.json and the patches are immutable and are never +// fetched again once cached. `relPath` comes from a parsed, pattern-checked +// field, and is checked here again before it becomes a blob name. 404s are +// swallowed as for analysis.md. +export async function ensureRunDoctorFile( + container: string, + runId: string, + relPath: string, + destRoot: string, + refreshAfterMs?: number, +): Promise { + assertValidId(runId, "runId"); + // One file directly under skill-doctor/, or one level down (the fixes dir). + if (!isValidTaskId(relPath) || relPath.split("/").length > 2) { + throw new Error(`Invalid skill-doctor path: ${JSON.stringify(relPath)}`); + } + if (LOCAL_RUNS_DIR) return; + return dedupe(`doctor:${container}:${runId}/${relPath}`, async () => { + try { + await downloadBlob( + container, + `${runId}/${DOCTOR_DIR}/${relPath}`, + destRoot, + refreshAfterMs, + ); + } catch (err) { + if (!isNotFound(err)) throw err; + } + }); +} + // There is deliberately no whole-run fetch. `ensureRunDir` used to pull EVERY // blob under the run prefix with no concurrency cap (~10k blobs / ~400 MB for a // nightly). Per-task download, below, is the supported shape. diff --git a/evalboard/lib/doctor.ts b/evalboard/lib/doctor.ts new file mode 100644 index 00000000..b8a5cbfa --- /dev/null +++ b/evalboard/lib/doctor.ts @@ -0,0 +1,479 @@ +// The skill-doctor report of one nightly run (contract v1), read from +// `/skill-doctor/`: +// - status.json: pointer + lock, changes while the doctor runs; +// - triage-.json: the per-task failure analysis, immutable; +// - /fixes.json: fix candidates and pr_checks, optional, create-only; +// - /opened.json: the PRs the doctor's open step opened, optional. +// +// Pure data + string helpers, no Node builtins: the parsed shapes are passed to +// the run page's components, and the tests import this module directly. The +// types are written by hand against the doctor's status_v1 / triage_v1 / +// fixes_v1 schemas and carry only the fields the run page renders. A file that does not parse, +// or whose schema_version is not 1, is treated as absent: the page must never +// fail because of a doctor file. + +import { isValidTaskId } from "./ids"; +import { SKILLS_SOURCE } from "./sources"; + +export const DOCTOR_DIR = "skill-doctor"; +const TRIAGE_FILE_RE = /^triage-[A-Za-z0-9._-]+\.json$/; +const FIXES_DIR_RE = /^fixes(-[0-9]+)?$/; +// Candidate ids name `.patch` inside the fixes dir. +const PATCH_ID_RE = /^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$/; +// At most this many patches per run are fetched, linked and served. +const MAX_PATCHES = 10; + +export const DOCTOR_STATES = [ + "queued", + "running", + "done", + "failed", + "skipped", + "retracted", +] as const; +export type DoctorState = (typeof DOCTOR_STATES)[number]; + +function isDoctorState(v: unknown): v is DoctorState { + return typeof v === "string" && (DOCTOR_STATES as readonly string[]).includes(v); +} + +export interface DoctorStatus { + state: DoctorState; + reason: string; + startedAt: string | null; + buildUrl: string | null; + triageFile: string | null; + fixesDir: string | null; + trigger: string | null; +} + +// Every PR URL the parsers hand out has passed isSkillsPrUrl: views can link +// them without checking again. +export interface DoctorPr { + url: string; + title: string; + state: string; +} + +// One failed row, with the fields the skill-doctor page shows on its card. +export interface DoctorTask { + taskId: string; + skill: string; + layer: string; + solvable: string | null; + runStatus: string | null; + confidence: string | null; + why: string | null; + fix: string | null; + criterion: string | null; + evidence: string | null; + passedBefore: string | null; + // Open/merged PRs on the task's files; only used to resolve fixedByPr. + existingPrs: DoctorPr[]; + // The diagnosis's judgement of the open PR that best fixes this row. + fixedByPr: DoctorFixedByPr | null; + envGroup: string | null; + // The doctor's PR for this task, allowlisted (else null). + doctorPr: string | null; +} + +export interface DoctorFixedByPr { + number: number; + verdict: "fixes" | "may_help" | "unrelated"; + basis: string; +} + +// The Fix stage's replay of a row's fixedByPr (fixes.json pr_checks). Its +// url is validated (allowlisted, ends in /pull/) but not kept: the +// board links the PR from the row's existing PRs. +export interface DoctorPrCheck { + number: number; + verdict: "verified" | "not_verified" | "untested"; + evidence: string; + reason: "needs-windows-run" | null; + // The PR head the check applied (40 hex), when the doctor recorded it. + headSha: string | null; +} + +export interface DoctorAction { + count: number; + text: string; + taskIds: string[]; +} + +export interface DoctorTriage { + // The run the triage describes (run.run_id); readRunDoctor drops a triage + // that names another run. + runId: string | null; + // Rows the nightly executed (null when the run did not say). + rowsRun: number | null; + rowsFailed: number; + rowsDiagnosed: number; + tasks: DoctorTask[]; + actions: DoctorAction[]; +} + +export interface DoctorGates { + replay: string | null; + mutants: { n: number; killed: number } | null; + review: string | null; +} + +export interface DoctorFix { + id: string; + taskId: string; + covers: string[]; + outcome: string; + patchLines: { added: number; removed: number } | null; + title: string; + gates: DoctorGates; +} + +export interface DoctorFixes { + // The run fixes.json describes (run_id); readRunDoctor drops one that + // names another run. + runId: string | null; + fixMode: string | null; + // By task id (own keys; read with own()). null: an entry that did not + // validate. + prChecks: Record; + candidates: DoctorFix[]; +} + +export interface RunDoctor { + status: DoctorStatus; + // Set only when the status is `done` and its triage file parsed. + triage: DoctorTriage | null; + fixes: DoctorFixes | null; + // PRs the doctor opened (`/opened.json`): task id → PR URL, + // allowlisted URLs only. + opened?: Record; + // A doctor file could not be fetched (a storage error, not a 404): the + // parts that are missing may exist. + fetchFailed?: boolean; +} + +// ---------- coercion helpers ---------- + +type Obj = Record; + +function isObj(v: unknown): v is Obj { + return typeof v === "object" && v !== null && !Array.isArray(v); +} +function str(v: unknown): string | null { + return typeof v === "string" ? v : null; +} +function isCount(v: unknown): v is number { + return typeof v === "number" && Number.isInteger(v) && v >= 0; +} +function int(v: unknown, dflt = 0): number { + return isCount(v) ? v : dflt; +} +function strList(v: unknown): string[] { + return Array.isArray(v) ? v.filter((x): x is string => typeof x === "string") : []; +} + +function parseJson(raw: string | null | undefined): unknown { + if (raw == null) return null; + try { + return JSON.parse(raw); + } catch { + return null; + } +} + +// ---------- parsers ---------- + +export function parseDoctorStatus(raw: string | null | undefined): DoctorStatus | null { + const j = parseJson(raw); + if (!isObj(j) || j.schema_version !== 1 || !isDoctorState(j.state)) return null; + const triageFile = str(j.triage_file); + const fixesDir = str(j.fixes_dir); + return { + state: j.state, + reason: str(j.reason) ?? "", + startedAt: str(j.started_at), + buildUrl: str(j.build_url), + // A name outside the pattern is treated as absent: it lands in a path. + triageFile: triageFile && TRIAGE_FILE_RE.test(triageFile) ? triageFile : null, + fixesDir: fixesDir && FIXES_DIR_RE.test(fixesDir) ? fixesDir : null, + trigger: str(j.trigger), + }; +} + +function parsePr(v: unknown): DoctorPr | null { + if (!isObj(v)) return null; + const url = str(v.url); + if (!isSkillsPrUrl(url)) return null; + return { url, title: str(v.title) ?? "", state: str(v.state) ?? "unknown" }; +} + +function parseFixedByPr(v: unknown): DoctorFixedByPr | null { + if (!isObj(v)) return null; + const n = v.number; + const verdict = v.verdict; + if (typeof n !== "number" || !Number.isInteger(n) || n < 1) return null; + if (verdict !== "fixes" && verdict !== "may_help" && verdict !== "unrelated") return null; + return { number: n, verdict, basis: (str(v.basis) ?? "").slice(0, 300) }; +} + +function parsePrCheck(v: unknown): DoctorPrCheck | null { + if (!isObj(v)) return null; + const n = v.number; + const verdict = v.verdict; + if (typeof n !== "number" || !Number.isInteger(n) || n < 1) return null; + if (!isSkillsPrUrl(v.url) || !v.url.endsWith(`/pull/${n}`)) return null; + if (verdict !== "verified" && verdict !== "not_verified" && verdict !== "untested") return null; + return { + number: n, + verdict, + evidence: (str(v.evidence) ?? "").slice(0, 600), + reason: v.reason === "needs-windows-run" ? "needs-windows-run" : null, + headSha: typeof v.head_sha === "string" && /^[0-9a-f]{40}$/.test(v.head_sha) ? v.head_sha : null, + }; +} + +// A malformed entry is kept as null, so its task shows no PR row rather than +// falling back to "no check" (which would read as "may fix"). +function parsePrChecks(v: unknown): Record { + const out = new Map(); + if (isObj(v)) { + for (const [k, c] of Object.entries(v)) out.set(k, parsePrCheck(c)); + } + return Object.fromEntries(out); +} + +/** A report-keyed lookup that never reads the prototype ("constructor", ...). */ +export function own(r: Record | null | undefined, key: string): T | undefined { + return r && Object.hasOwn(r, key) ? r[key] : undefined; +} + +function prList(v: unknown): DoctorPr[] { + return Array.isArray(v) ? v.map(parsePr).filter((p): p is DoctorPr => p !== null) : []; +} + +function parseTask(v: unknown): DoctorTask | null { + if (!isObj(v)) return null; + const taskId = str(v.task_id); + if (!taskId) return null; + return { + taskId, + skill: str(v.skill) ?? "", + layer: str(v.layer) ?? "undiagnosed", + solvable: str(v.solvable), + runStatus: str(v.run_status), + confidence: str(v.confidence), + why: str(v.why), + fix: str(v.fix), + criterion: str(v.criterion), + evidence: str(v.evidence), + passedBefore: str(v.passed_before), + existingPrs: prList(v.existing_prs), + fixedByPr: parseFixedByPr(v.fixed_by_pr), + envGroup: str(v.env_group), + doctorPr: isSkillsPrUrl(v.doctor_pr) ? v.doctor_pr : null, + }; +} + +export function parseTriage(raw: string | null | undefined): DoctorTriage | null { + const j = parseJson(raw); + if (!isObj(j) || j.schema_version !== 1) return null; + if (!isObj(j.summary) || !Array.isArray(j.tasks)) return null; + const s = j.summary; + // The header shows these counts: a missing or malformed one is contract + // drift, so the triage does not parse (and readRunDoctor logs it) rather + // than reading as a real 0. + if (!isCount(s.rows_failed) || !isCount(s.rows_diagnosed)) return null; + const run = isObj(j.run) ? j.run : {}; + return { + runId: str(run.run_id), + rowsRun: isCount(run.rows_executed) ? run.rows_executed : null, + rowsFailed: s.rows_failed, + rowsDiagnosed: s.rows_diagnosed, + tasks: j.tasks.map(parseTask).filter((t): t is DoctorTask => t !== null), + actions: Array.isArray(j.actions) + ? j.actions.flatMap((a): DoctorAction[] => + // "open-pr" actions list PRs that merely touch the files, not + // fixes: the board leaves them out. + isObj(a) && typeof a.text === "string" && a.kind !== "open-pr" + ? [{ count: int(a.count), text: a.text, taskIds: strList(a.task_ids) }] + : [], + ) + : [], + }; +} + +function parseGates(v: unknown): DoctorGates { + if (!isObj(v)) return { replay: null, mutants: null, review: null }; + const replay = isObj(v.replay) ? str(v.replay.status) : null; + const m = isObj(v.mutants) ? v.mutants : null; + const mutants = + m && isCount(m.n) && isCount(m.killed) ? { n: m.n, killed: m.killed } : null; + return { replay, mutants, review: str(v.review) }; +} + +export function parseFixes(raw: string | null | undefined): DoctorFixes | null { + const j = parseJson(raw); + if (!isObj(j) || j.schema_version !== 1 || !Array.isArray(j.candidates)) return null; + return { + runId: str(j.run_id), + fixMode: str(j.fix_mode), + prChecks: parsePrChecks(j.pr_checks), + candidates: j.candidates.flatMap((c): DoctorFix[] => { + if (!isObj(c)) return []; + const id = str(c.id); + const taskId = str(c.task_id); + if (!id || !taskId) return []; + const pl = isObj(c.patch_lines) ? c.patch_lines : null; + return [ + { + id, + taskId, + covers: strList(c.covers), + outcome: str(c.outcome) ?? "unknown", + patchLines: pl + ? { added: int(pl.added), removed: int(pl.removed) } + : null, + title: str(c.title) ?? "", + gates: parseGates(c.gates), + }, + ]; + }), + }; +} + +/** + * The only patches the board touches. readRunDoctor fetches exactly these, + * the Fixes table links exactly these, and /api/file serves only these (via + * readRunDoctor, so there is one fetch path). A patch exists only when + * fixes.json reports patch lines; the id must be a safe file name; at most + * MAX_PATCHES per run. + */ +export function patchIds(fixes: DoctorFixes | null): Set { + const ids = new Set(); + for (const c of fixes?.candidates ?? []) { + if (ids.size >= MAX_PATCHES) break; + if (c.patchLines !== null && PATCH_ID_RE.test(c.id)) ids.add(c.id); + } + return ids; +} + +/** + * `/opened.json` from the doctor's open step: + * `{"opened": [{"task_id", "pr_url", ...} | {"task_id", "problem"}], "error"?}`. + * Untrusted: only entries with a task id and an allowlisted PR URL map; + * problem entries and anything else are skipped. Null when it does not parse. + */ +export function parseOpened(raw: string | null | undefined): Record | null { + const j = parseJson(raw); + if (!isObj(j) || !Array.isArray(j.opened)) return null; + // Built through a Map so a task id such as "__proto__" or "constructor" + // is an own key, never the prototype; the first entry per task wins. + const out = new Map(); + for (const e of j.opened) { + if (!isObj(e)) continue; + const id = str(e.task_id); + const url = str(e.pr_url); + if (id && isSkillsPrUrl(url) && !out.has(id)) out.set(id, url); + } + return Object.fromEntries(out); +} + +/** `/.patch` → the id, or null for any other doctor path. */ +export function patchIdFromPath(rel: string): { fixesDir: string; id: string } | null { + const parts = rel.split("/"); + if (parts.length !== 2 || !FIXES_DIR_RE.test(parts[0])) return null; + const m = /^(.+)\.patch$/.exec(parts[1]); + if (!m || !PATCH_ID_RE.test(m[1])) return null; + return { fixesDir: parts[0], id: m[1] }; +} + +// ---------- links ---------- + +// The report's text comes from a model and from the run's own logs, so a URL +// in it is shown as a link only when it has exactly the expected shape. +const SKILLS_PR_RE = /^https:\/\/github\.com\/UiPath\/skills\/pull\/\d+$/; +const ADO_RE = /^https:\/\/dev\.azure\.com\/[^\s"'<>]*$/; + +export function isSkillsPrUrl(u: unknown): u is string { + return typeof u === "string" && SKILLS_PR_RE.test(u); +} + +export function isAdoUrl(u: unknown): u is string { + return typeof u === "string" && ADO_RE.test(u); +} + +/** A task link is built only from an id the task page itself accepts. */ +export const isLinkableTaskId = isValidTaskId; + +// ---------- analysis.md split ---------- + +/** + * Cut analysis.md at its first line that is exactly `## Findings`, outside a + * fenced code block. Null when there is no such line. + */ +export function splitAnalysis( + md: string, +): { top: string; findings: string } | null { + const lines = md.split("\n"); + let fence: string | null = null; + let offset = 0; + for (const raw of lines) { + const line = raw.endsWith("\r") ? raw.slice(0, -1) : raw; + const f = /^ {0,3}(`{3,}|~{3,})/.exec(line); + if (f) { + const mark = f[1]; + if (fence === null) fence = mark; + else if (mark[0] === fence[0] && mark.length >= fence.length && line.trim() === mark) + fence = null; + } else if (fence === null && line === "## Findings") { + return { top: md.slice(0, offset), findings: md.slice(offset) }; + } + offset += raw.length + 1; + } + return null; +} + +// ---------- gating + page plan ---------- + +/** + * The panel is internal-only, for the skills nightly, with an env kill switch + * (EVALBOARD_TASK_ANALYSIS=off, in any case). + */ +export function taskAnalysisEnabled( + internal: boolean, + sourceId: string, + env: string | undefined, +): boolean { + return ( + internal && + sourceId === SKILLS_SOURCE.id && + (env ?? "").trim().toLowerCase() !== "off" + ); +} + +export interface AnalysisPlan { + // Markdown for the AI summary panel (null: no panel). + summary: string | null; + // The Task Failure Analysis block, right under the summary (null: none). + doctor: RunDoctor | null; +} + +/** + * What the run page shows. With a done report whose triage parsed, the + * summary stops at analysis.md's `## Findings` line (the block replaces the + * rest), or keeps the whole file when there is no such line. In every other + * state the summary is unchanged (Findings included) and the block shows only + * the doctor's state. + */ +export function planAnalysis( + analysis: string | null, + doctor: RunDoctor | null, +): AnalysisPlan { + if (doctor && doctor.status.state === "done" && doctor.triage) { + const split = analysis ? splitAnalysis(analysis) : null; + const summary = split ? split.top : analysis; + return { summary: summary && summary.trim() ? summary : null, doctor }; + } + return { summary: analysis, doctor }; +} diff --git a/evalboard/lib/ids.ts b/evalboard/lib/ids.ts new file mode 100644 index 00000000..5272ee38 --- /dev/null +++ b/evalboard/lib/ids.ts @@ -0,0 +1,33 @@ +// Run / task id validators. Pure (no Node builtins), so client components and +// lib/doctor.ts can share the exact check the data layer applies; blob.ts +// re-exports them for its existing importers. + +// Run / task IDs get reflected into filesystem paths and blob prefixes, so +// reject anything outside a narrow whitelist before any side effect. A run id +// of only dots ("." / "..") would name the runs dir or its parent. +const ID_RE = /^[\w.-]+$/; +const DOTS_RE = /^\.+$/; +// Task IDs from dataset-expanded tasks look like "sentiment-classification/r3" +// (suite/row). Each segment must still pass the narrow ID_RE — only the slash +// separator between segments is additionally allowed. +const TASK_ID_RE = /^[\w.-]+(\/[\w.-]+)*$/; + +export function isValidId(id: unknown): id is string { + return ( + typeof id === "string" && + id.length > 0 && + id.length < 128 && + ID_RE.test(id) && + !DOTS_RE.test(id) + ); +} + +export function isValidTaskId(id: unknown): id is string { + return ( + typeof id === "string" && + id.length > 0 && + id.length < 256 && + TASK_ID_RE.test(id) && + !id.split("/").some((s) => DOTS_RE.test(s)) + ); +} diff --git a/evalboard/lib/runs.ts b/evalboard/lib/runs.ts index feb7d7c8..360edfe3 100644 --- a/evalboard/lib/runs.ts +++ b/evalboard/lib/runs.ts @@ -6,6 +6,7 @@ import { LOCAL_RUNS_DIR, ensureActivationSummary, ensureRunAnalysis, + ensureRunDoctorFile, ensureRunExperiment, ensureRunMeta, ensureRunSummary, @@ -19,6 +20,18 @@ import { DEFAULT_VARIANT_ID, isValidVariantId } from "./variants"; import { DEFAULT_SOURCE, runsDirFor, type Source } from "./sources"; import { DELIVERABLE_KINDS, DELIVERABLE_NAMES } from "./artifact-kinds"; import { messageCostUsd, normalizeModel } from "./pricing"; +import { + DOCTOR_DIR, + parseDoctorStatus, + parseFixes, + parseOpened, + parseTriage, + patchIdFromPath, + patchIds, + taskAnalysisEnabled, + type RunDoctor, +} from "./doctor"; +import { isInternal } from "./edition"; // Resolution order: // 1. EVALBOARD_LOCAL_RUNS_DIR — local mode, points at a coder_eval runs dir @@ -2680,6 +2693,102 @@ export async function readRunAnalysis( .catch(() => null); } +// ---------- skill-doctor report (skill-doctor/ sidecars) ---------- + +// status.json changes while the doctor runs: a cached copy older than this is +// fetched again. Everything it points at is immutable. +const DOCTOR_STATUS_REFRESH_MS = 60_000; + +// The run's skill-doctor report, or null when there is none, it does not parse, +// or its schema_version is not 1. This is the one place that decides which +// files a status implies: a done status.json → the triage file it names, and +// /fixes.json → the patches fixes.json lists (any other state, +// retracted included, implies no other file). By default it never throws: a +// doctor file problem must not take the run page down, so a fetch error +// degrades to "no report" (or to the parts that did load) and sets +// fetchFailed. `strict` (for /api/file) lets a storage error other than a 404 +// propagate instead, so a click on a patch link can tell "storage down" from +// "not listed"; it then fetches only the one patch `strict` names (no +// opened.json, no other patch). +export async function readRunDoctor( + runId: string, + source: Source = DEFAULT_SOURCE, + { strict }: { strict?: { patch: string } } = {}, +): Promise { + const dir = runsDirFor(RUNS_DIR, source); + const base = path.join(dir, runId, DOCTOR_DIR); + let fetchFailed = false; + // ensureRunDoctorFile already swallows a 404 (an absent file). + const get = async (rel: string, refreshAfterMs?: number) => { + try { + await ensureRunDoctorFile(source.container, runId, rel, dir, refreshAfterMs); + } catch (err) { + if (strict) throw err; + fetchFailed = true; + console.error(`skill-doctor: fetching ${rel} failed for run ${runId}:`, err); + } + }; + // Read + parse once. A file that exists but does not parse is logged by + // name only (never its contents), so contract drift is visible. + const load = async (rel: string, parse: (raw: string) => T | null) => { + const raw = await fs.readFile(path.join(base, rel), "utf-8").catch(() => null); + if (raw === null) return null; + const parsed = parse(raw); + if (parsed === null) { + console.warn( + `skill-doctor: ${rel} for run ${runId} did not parse as contract v1; ignored`, + ); + } + return parsed; + }; + try { + await get("status.json", DOCTOR_STATUS_REFRESH_MS); + const status = await load("status.json", parseDoctorStatus); + if (!status) return null; + const done = status.state === "done"; + const fdir = done ? status.fixesDir : null; + const triageRel = done && status.triageFile ? status.triageFile : null; + const fixesRel = fdir ? `${fdir}/fixes.json` : null; + await Promise.all([ + triageRel && get(triageRel), + fixesRel && get(fixesRel), + ]); + const [parsedTriage, parsedFixes] = await Promise.all([ + triageRel ? load(triageRel, parseTriage) : null, + fixesRel ? load(fixesRel, parseFixes) : null, + ]); + // A triage or fixes.json that names another run (a copy under the + // wrong prefix) must not replace this run's Findings or list patches. + const ofThisRun = (v: T | null, rel: string | null) => { + if (!v || v.runId === runId) return v; + console.warn( + `skill-doctor: ${rel} names run ${JSON.stringify(v.runId)}, not ${runId}; ignored`, + ); + return null; + }; + const triage = ofThisRun(parsedTriage, triageRel); + const fixes = ofThisRun(parsedFixes, fixesRel); + // Cached patches are not fetched again (downloadBlob skips them). + // opened.json is written by the doctor's open step after the fixes, so + // it may appear or change later: it is refetched like status.json. + let opened: Record = {}; + if (fdir && strict) { + if (patchIds(fixes).has(strict.patch)) await get(`${fdir}/${strict.patch}.patch`); + } else if (fdir) { + await Promise.all([ + ...[...patchIds(fixes)].map((id) => get(`${fdir}/${id}.patch`)), + get(`${fdir}/opened.json`, DOCTOR_STATUS_REFRESH_MS), + ]); + opened = (await load(`${fdir}/opened.json`, parseOpened)) ?? {}; + } + return { status, triage, fixes, opened, fetchFailed }; + } catch (err) { + if (strict) throw err; + console.error(`readRunDoctor failed for run ${runId}:`, err); + return null; + } +} + // ---------- run metadata (meta.json sidecar) ---------- // Optional, user-supplied at upload time. Null on pipeline runs and any run @@ -2944,6 +3053,11 @@ export async function resolveSafePath( source: Source = DEFAULT_SOURCE, ): Promise { if (!isValidId(runId)) return null; + // Dispatch on the normalized path, so "./skill-doctor/…" or + // "activation/../skill-doctor/…" cannot skip the skill-doctor gate below. + relPath = path.posix.normalize(relPath); + if (relPath === ".." || relPath.startsWith("../") || path.posix.isAbsolute(relPath)) + return null; const dir = runsDirFor(RUNS_DIR, source); // Artifact URLs embed the variant + task subdirs in relPath // (`//artifacts/...`) — extract them so the narrow @@ -2952,7 +3066,34 @@ export async function resolveSafePath( // (`activation/default//…`), so parts[0..1] are not (variant, task) // there; that case falls through to the run-summary fetch as it always did. const parts = relPath.split("/"); - if (parts[0] !== "activation" && isValidVariantId(parts[0]) && parts[1]) { + if (parts[0] === DOCTOR_DIR) { + // skill-doctor/ belongs to the doctor contract, so no experiment arm can + // live there: a variant of that name would share the blob prefix with + // the report. Only a fix patch is served from it, and only one that + // this run's status + fixes.json list (patchIds). readRunDoctor is the + // one fetch path: it loads status and fixes.json when they are not + // cached and fetches exactly those patches, so a cold cache or a direct + // link works and a made-up name costs no blob call. Off with the panel. + // Strict: a storage error propagates (the route answers 5xx), so it + // does not read as "forbidden" or "not found". + if ( + !taskAnalysisEnabled( + isInternal, + source.id, + process.env.EVALBOARD_TASK_ANALYSIS, + ) + ) + return null; + const patch = patchIdFromPath(parts.slice(1).join("/")); + if (!patch) return null; + const doctor = await readRunDoctor(runId, source, { strict: { patch: patch.id } }); + if ( + !doctor || + doctor.status.fixesDir !== patch.fixesDir || + !patchIds(doctor.fixes).has(patch.id) + ) + return null; + } else if (parts[0] !== "activation" && isValidVariantId(parts[0]) && parts[1]) { if (!isValidId(parts[1])) return null; await ensureTaskDir(source.container, runId, parts[1], dir, parts[0]); } else { @@ -2984,10 +3125,10 @@ export async function resolveSafePath( // Delete a run's locally-cached blob copy under `root` so the next view // re-downloads it from storage. `force: true` makes a never-cached run a -// harmless no-op. Returns false (deleting nothing) for an unsafe id — note -// isValidId still admits "." and ".." (dots are word-ish), so require the -// resolved target to be a strict child of `root` before rm can run, or a "." -// id would nuke the cache root and ".." its parent. +// harmless no-op. Returns false (deleting nothing) for an unsafe id. isValidId +// rejects "." and "..", and the resolved target must still be a strict child +// of `root` before rm can run (a "." id would nuke the cache root, ".." its +// parent), as a second guard. export async function clearRunCacheDir( root: string, id: string,