Refine Scenario configuration and sizing

frontend/e2e/api.spec.ts

@@ -1,26 +1,31 @@
 import { test, expect } from "@playwright/test";
+import type { APIRequestContext } from "@playwright/test";
 
 // API tests go through the Vite dev server proxy (/api -> configured backend)
 // rather than hitting the backend directly, so they work as soon as
 // Playwright's webServer is ready.
 
-test.describe("API Health Check", () => {
-  // The backend may still be starting when Vite is already up.
-  // Poll the health endpoint through the proxy until the backend is ready.
-  test.beforeAll(async ({ request }) => {
-    const maxWait = 30_000;
-    const interval = 1_000;
-    const start = Date.now();
-    while (Date.now() - start < maxWait) {
-      try {
-        const resp = await request.get("/api/health", { timeout: 2_000 });
-        if (resp.ok()) return;
-      } catch {
-        // Backend not ready yet
+async function waitForBackend(request: APIRequestContext): Promise<void> {
+  const maxWait = 30_000;
+  const interval = 1_000;
+  const start = Date.now();
+  while (Date.now() - start < maxWait) {
+    try {
+      const response = await request.get("/api/health", { timeout: 2_000 });
+      if (response.ok()) {
+        return;
       }
-      await new Promise((r) => setTimeout(r, interval));
+    } catch {
+      // Backend not ready yet
     }
-    throw new Error("Backend did not become healthy within 30 seconds");
+    await new Promise((resolve) => setTimeout(resolve, interval));
+  }
+  throw new Error("Backend did not become healthy within 30 seconds");
+}
+
+test.describe("API Health Check", () => {
+  test.beforeAll(async ({ request }) => {
+    await waitForBackend(request);
   });
 
   test("should have healthy backend API @seeded", async ({ request }) => {
@@ -38,24 +43,12 @@ test.describe("API Health Check", () => {
     const data = await response.json();
     expect(data).toBeDefined();
   });
+
 });
 
 test.describe("Targets API", () => {
   test.beforeAll(async ({ request }) => {
-    // Wait for backend readiness
-    const maxWait = 30_000;
-    const interval = 1_000;
-    const start = Date.now();
-    while (Date.now() - start < maxWait) {
-      try {
-        const resp = await request.get("/api/health", { timeout: 2_000 });
-        if (resp.ok()) return;
-      } catch {
-        // Backend not ready yet
-      }
-      await new Promise((r) => setTimeout(r, interval));
-    }
-    throw new Error("Backend did not become healthy within 30 seconds");
+    await waitForBackend(request);
   });
 
   test("should list targets @seeded", async ({ request }) => {
@@ -103,19 +96,7 @@ test.describe("Targets API", () => {
 
 test.describe("Attacks API", () => {
   test.beforeAll(async ({ request }) => {
-    const maxWait = 30_000;
-    const interval = 1_000;
-    const start = Date.now();
-    while (Date.now() - start < maxWait) {
-      try {
-        const resp = await request.get("/api/health", { timeout: 2_000 });
-        if (resp.ok()) return;
-      } catch {
-        // Backend not ready yet
-      }
-      await new Promise((r) => setTimeout(r, interval));
-    }
-    throw new Error("Backend did not become healthy within 30 seconds");
+    await waitForBackend(request);
   });
 
   test("should list attacks @seeded", async ({ request }) => {
@@ -124,10 +105,54 @@ test.describe("Attacks API", () => {
   });
 });
 
+test.describe("Scenarios API", () => {
+  test.beforeAll(async ({ request }) => {
+    await waitForBackend(request);
+  });
+
+  test("should expose scenario catalog details and queue state @seeded", async ({ request }) => {
+    test.setTimeout(90_000);
+    const catalogResponse = await request.get("/api/scenarios/catalog?limit=200");
+    expect(catalogResponse.ok()).toBe(true);
+    const catalog = await catalogResponse.json();
+    expect(catalog.items.length).toBeGreaterThan(0);
+
+    const scenarioName = catalog.items[0].scenario_name as string;
+    const detailResponse = await request.get(`/api/scenarios/catalog/${encodeURIComponent(scenarioName)}`);
+    expect(detailResponse.ok()).toBe(true);
+    const detail = await detailResponse.json();
+    expect(detail.scenario_name).toBe(scenarioName);
+    expect(detail.dataset_size_limit).toEqual(expect.objectContaining({
+      default_scope: expect.any(String),
+      override_scope: expect.any(String),
+    }));
+
+    const queueResponse = await request.get("/api/scenarios/runs/queue");
+    expect(queueResponse.ok()).toBe(true);
+    await expect(queueResponse.json()).resolves.toEqual(expect.objectContaining({
+      revision: expect.any(Number),
+      queued: expect.any(Array),
+    }));
+  });
+});
+
 test.describe("Error Handling", () => {
   test("should display UI when backend is slow", async ({ page }) => {
-    // Intercept and delay API calls
+    // Auth configuration must resolve before the application can render.
     await page.route("**/api/**", async (route) => {
+      if (new URL(route.request().url()).pathname === "/api/auth/config") {
+        await route.fulfill({
+          status: 200,
+          contentType: "application/json",
+          body: JSON.stringify({
+            clientId: "",
+            tenantId: "",
+            allowedGroupIds: "",
+          }),
+        });
+        return;
+      }
+
       await new Promise((resolve) => setTimeout(resolve, 2000));
       await route.continue();
     });

frontend/e2e/scenario-history.spec.ts

@@ -33,21 +33,26 @@ const datasetSummary = {
 };
 
 const configuredEstimate = {
-  estimated_attack_count: 8,
-  minimum_attack_count: null,
-  maximum_attack_count: null,
+  version: 1,
+  status: "exact",
+  total_attack_count: 8,
+  minimum_attack_count: 8,
+  maximum_attack_count: 8,
+  condition: null,
   components: [{
     label: "Prompt sending",
     count: 8,
     is_baseline: false,
     note: null,
   }],
   datasets: [datasetSummary],
+  adaptive_details: null,
   effective_parameters: {
     num_jailbreaks: 2,
     num_jailbreak_attempts: 1,
   },
   note: "The backend total is authoritative.",
+  retries_included: false,
 };
 
 const catalogScenario = {
@@ -82,7 +87,11 @@ const catalogScenario = {
     },
   ],
   default_datasets: ["harmbench"],
-  default_dataset_summaries: [datasetSummary],
+  dataset_size_limit: {
+    default_scope: "per_dataset",
+    default_count: 4,
+    override_scope: "per_dataset",
+  },
   baseline_policy: "enabled",
   include_baseline_by_default: false,
   supported_parameters: [
@@ -115,21 +124,26 @@ const catalogScenario = {
     },
   ],
   default_run_size: {
-    estimated_attack_count: 16,
-    minimum_attack_count: null,
-    maximum_attack_count: null,
+    version: 1,
+    status: "exact",
+    total_attack_count: 16,
+    minimum_attack_count: 16,
+    maximum_attack_count: 16,
+    condition: null,
     components: [{
       label: "Default attacks",
       count: 16,
       is_baseline: false,
       note: null,
     }],
     datasets: [datasetSummary],
+    adaptive_details: null,
     effective_parameters: {
       num_jailbreaks: 2,
       num_jailbreak_attempts: 1,
     },
     note: "Retries and internal turns are excluded.",
+    retries_included: false,
   },
 };
 
@@ -214,6 +228,8 @@ const progressAttempt = {
   timestamp: "2026-08-07T00:00:30Z",
   total_retries: 1,
   retries: [],
+  result_kind: "attack",
+  technique_name: "prompt_sending",
 };
 
 interface ScenarioMocks {
@@ -277,6 +293,14 @@ async function mockScenarioAPIs(page: Page): Promise<ScenarioMocks> {
     });
   });
 
+  await page.route(/\/api\/datasets(?:\?|$)/, async (route) => {
+    await route.fulfill({
+      status: 200,
+      contentType: "application/json",
+      body: JSON.stringify({ items: [{ name: "harmbench" }] }),
+    });
+  });
+
   await page.route(new RegExp(`/api/scenarios/catalog/${SCENARIO_NAME.replace(".", "\\.")}/estimate$`), async (route) => {
     const request = route.request().postDataJSON() as Record<string, unknown>;
     estimateRequests.push(request);
@@ -496,16 +520,20 @@ async function mockScenarioAPIs(page: Page): Promise<ScenarioMocks> {
 
 async function configurePromptSendingRun(page: Page): Promise<void> {
   await expect(page.getByTestId("scenario-target-select")).toHaveValue("test-target");
-  await page.getByTestId("technique-prompt_sending").check();
-  await page.getByTestId("technique-jailbreak_system_prompt").uncheck();
+  await page.getByTestId("technique-mode-custom").click();
+  await expect(page.getByTestId("technique-prompt_sending")).toBeChecked();
+  await expect(page.getByTestId("technique-jailbreak_system_prompt")).toBeChecked();
+  await page.getByTestId("technique-jailbreak_system_prompt").click();
   await page.getByTestId("scenario-param-num_jailbreaks").fill("2");
   await page.getByTestId("scenario-param-num_jailbreak_attempts").fill("1");
   await expect(page.getByTestId("baseline-checkbox")).not.toBeChecked();
-  await expect(page.getByTestId("run-estimate").getByText("8", { exact: true })).toBeVisible();
+  await expect(page.getByRole("group", {
+    name: "8 planned attacks.",
+  })).toBeVisible();
 }
 
 test.describe("Scenario catalog, history, and live run routing", () => {
-  test("renders the semantic catalog, full metadata, safe MyST, and both sidebar destinations", async ({ page }) => {
+  test("opens the Configure page from the semantic launch index with complete safe metadata", async ({ page }) => {
     await mockScenarioAPIs(page);
     await page.goto("/scanner");
 
@@ -524,17 +552,52 @@ test.describe("Scenario catalog, history, and live run routing", () => {
     ]);
     await expect(page.getByTitle("Scanner")).toHaveAttribute("aria-current", "page");
     await expect(page.getByRole("table", { name: "Registered scenarios" })).toBeVisible();
-    await expect(page.getByRole("columnheader", { name: "Default run size" })).toBeVisible();
-
+    await expect(page.getByRole("columnheader")).toHaveText([
+      "Scenario / purpose",
+      "Configure",
+      "Default datasets",
+      "Default techniques",
+      "Default run size",
+    ]);
     const row = page.getByTestId(`scenario-card-${SCENARIO_NAME}`);
-    await row.getByRole("link", { name: SCENARIO_NAME }).click();
+    const cells = row.getByRole("cell");
+    await expect(cells).toHaveCount(5);
+    const configureButton = cells.nth(1).getByRole("button", { name: "Configure run" });
+    await expect(configureButton).toBeVisible();
+    const [scenarioCellBox, configureCellBox, datasetCellBox] = await Promise.all([
+      cells.nth(0).boundingBox(),
+      cells.nth(1).boundingBox(),
+      cells.nth(2).boundingBox(),
+    ]);
+    expect(scenarioCellBox).not.toBeNull();
+    expect(configureCellBox).not.toBeNull();
+    expect(datasetCellBox).not.toBeNull();
+    expect(configureCellBox!.x).toBeGreaterThan(scenarioCellBox!.x);
+    expect(configureCellBox!.x).toBeLessThan(datasetCellBox!.x);
+    await expect(page.getByRole("button", { name: /show details|hide details/i })).toHaveCount(0);
+
+    await configureButton.click();
     await expect(page).toHaveURL(`/scanner/${SCENARIO_NAME}`);
     await expect(page.getByRole("heading", { name: SCENARIO_NAME, level: 1 })).toBeVisible();
+    await expect(page.getByText("Jailbreak · v4")).toBeVisible();
     const description = page.getByTestId("scenario-detail-description");
     await expect(description.getByText("dataset")).toHaveCSS("font-weight", /^(600|700)$/);
     await expect(description.locator("code").filter({ hasText: "num_jailbreaks" })).toBeVisible();
     await expect(description.locator("img")).toHaveCount(0);
     await expect(description).toContainText(RAW_IMAGE_HTML);
+    await expect(page.getByRole("radio", { name: /Recommended \(default\).*2 techniques/ })).toBeChecked();
+    await expect(page.getByRole("radio", { name: /Easy.*1 technique/ })).toBeVisible();
+    await expect(page.getByRole("radio", { name: "Custom" })).toBeVisible();
+    const members = page.getByTestId("selected-technique-set-members");
+    await expect(members.getByText("prompt_sending")).toBeVisible();
+    await expect(members.getByText("jailbreak_system_prompt")).toBeVisible();
+    const preview = page.getByRole("complementary", { name: "Run preview" });
+    await expect(preview.getByText("Jailbreak templates: 2")).toBeVisible();
+    await expect(preview.getByRole("group", {
+      name: "16 planned attacks.",
+    })).toBeVisible();
+    await expect(page.getByText("Include direct baseline comparison")).toBeVisible();
+    await expect(page.getByText(/Also send each selected objective directly/)).toBeVisible();
 
     await page.getByTitle("History").click();
     await expect(page).toHaveURL("/history/attacks");
@@ -551,7 +614,7 @@ test.describe("Scenario catalog, history, and live run routing", () => {
     await page.goto(`/scanner/${SCENARIO_NAME}`);
 
     const form = page.getByRole("form", { name: "Scenario run configuration" });
-    const preview = page.getByTestId("run-estimate");
+    const preview = page.getByRole("complementary", { name: "Run preview" });
     await expect(form).toBeVisible();
     await expect(preview).toBeVisible();
 
@@ -570,12 +633,12 @@ test.describe("Scenario catalog, history, and live run routing", () => {
       const requests = mocks.getEstimateRequests();
       return requests[requests.length - 1];
     }).toEqual(expectedEstimateRequest);
-    await expect(preview.getByText("8", { exact: true })).toBeVisible();
+    await expect(preview.getByRole("group", {
+      name: "8 planned attacks.",
+    })).toBeVisible();
     await expect(preview).not.toContainText("context_compliance");
 
     await page.getByTestId("launch-scenario-btn").click();
-    await expect(page.getByRole("dialog", { name: "Run preview" })).toBeVisible();
-    await page.getByTestId("confirm-launch-scenario-btn").click();
     const expectedLaunchRequest = {
       scenario_name: SCENARIO_NAME,
       target_name: "test-target",
@@ -609,15 +672,28 @@ test.describe("Scenario catalog, history, and live run routing", () => {
     const client = await page.context().newCDPSession(page);
     await client.send("Emulation.setTouchEmulationEnabled", { enabled: true, maxTouchPoints: 1 });
     await page.setViewportSize({ width: 390, height: 844 });
-    await page.goto(`/scanner/${SCENARIO_NAME}`);
+    await page.goto("/scanner");
+    const catalogRow = page.getByTestId(`scenario-card-${SCENARIO_NAME}`);
+    const configureButton = catalogRow.getByRole("button", { name: "Configure run" });
+    await expect(catalogRow).toBeVisible();
+    expect(await catalogRow.getByRole("cell").allInnerTexts()).toEqual([
+      expect.stringContaining("Scenario / purpose"),
+      expect.stringContaining("Configure"),
+      expect.stringContaining("Default datasets"),
+      expect.stringContaining("Default techniques"),
+      expect.stringContaining("Default run size"),
+    ]);
+    expect(await page.evaluate(() => document.documentElement.scrollWidth)).toBeLessThanOrEqual(390);
+    expect((await configureButton.boundingBox())?.height).toBeGreaterThanOrEqual(44);
+    await configureButton.press("Enter");
+    await expect(page).toHaveURL(`/scanner/${SCENARIO_NAME}`);
     await configurePromptSendingRun(page);
 
     const formBox = await page.getByRole("form", { name: "Scenario run configuration" }).boundingBox();
-    const previewBox = await page.getByTestId("run-estimate").boundingBox();
+    const previewBox = await page.getByRole("complementary", { name: "Run preview" }).boundingBox();
     expect(formBox).not.toBeNull();
     expect(previewBox).not.toBeNull();
-    expect(previewBox!.y).toBeGreaterThan(formBox!.y);
-    expect(previewBox!.y + previewBox!.height).toBeLessThanOrEqual(formBox!.y + formBox!.height);
+    expect(previewBox!.y).toBeGreaterThanOrEqual(formBox!.y + formBox!.height);
     expect(await page.evaluate(() => document.documentElement.scrollWidth)).toBeLessThanOrEqual(390);
 
     for (const control of [