perf: publish controlled release benchmark evidence

.gitattributes

@@ -0,0 +1,3 @@
+benchmarks/results/*/*.json linguist-generated=true
+benchmarks/results/*/*.jsonl linguist-generated=true
+benchmarks/results/*/*.log linguist-generated=true

.prettierignore

@@ -2,3 +2,5 @@ coverage/
 dist/
 node_modules/
 pnpm-lock.yaml
+# Preserve downloaded measurements byte-for-byte; check:benchmarks validates their contents.
+benchmarks/results/*/*.json

CHANGELOG.md

@@ -2,6 +2,13 @@
 
 ## Unreleased
 
+- Separate isolated benchmark baselines from historical shared-workstation measurements.
+  Publish the retained DigitalOcean 1.0.5 baseline with provenance and teardown evidence,
+  and validate dataset version labels and summaries in CI before publication.
+  Add controlled release comparisons with separate SQLite durability settings, retain the
+  incomplete MySQL attempt and state-check failure, and derive only fully completed backend plans.
+  Publish a separate complete five-round MySQL comparison of 1.0.5 and 1.0.6 from a fresh runner.
+
 ## 1.0.7 - 2026-11-09
 
 - Add the `singleOwner` option for a process that is the only writer of its

README.md

@@ -17,7 +17,9 @@ promise.
 
 **Production use.** One first-party application with more than 100,000 users
 runs it in production. That is a count of users, not a measure of load. The
-[benchmarks](docs/benchmarks.md) are development measurements.
+[benchmark guide](https://github.com/cardmagic/solid-objects-js/blob/main/docs/benchmarks.md)
+lists the measured releases, environments and durability settings. Reports are
+published independently of package versions.
 
 **It is not** Solid Queue, SolidJS, or the `service_actor` gem, and it is not
 affiliated with Rails or Cloudflare.
@@ -296,6 +298,7 @@ then shows a tested Solid Objects version with its limits:
 - [Choosing Solid Objects](docs/fit.md)
 - [Comparisons with Restate, DBOS, RivetKit, and Graphile Worker](docs/comparisons.md)
 - [Public API](docs/api.md)
+- [Versioned benchmark results](https://github.com/cardmagic/solid-objects-js/blob/main/docs/benchmarks.md)
 - [Versioning](docs/versioning.md)
 - [Operations and recovery](docs/operations.md)
 - [Observability and diagnostics](docs/observability.md)

benchmarks/check-results.mjs

@@ -0,0 +1,208 @@
+import assert from "node:assert/strict"
+import { execFileSync } from "node:child_process"
+import { readFile, writeFile } from "node:fs/promises"
+import { fileURLToPath, pathToFileURL } from "node:url"
+import { resolve } from "node:path"
+import { comparisonRows } from "./comparison.mjs"
+import { completedComparison, comparisonForDatabases } from "./comparison-scope.mjs"
+import {
+  assertPublishedTables,
+  baselineHeadline,
+  comparisonHeadline,
+  controlledTable,
+} from "./publication.mjs"
+
+const writeGuide = process.argv.includes("--write-guide")
+const argumentsValue = process.argv.slice(2).filter((value) => value !== "--write-guide")
+const root = argumentsValue[1]
+  ? pathToFileURL(resolve(argumentsValue[1]) + "/")
+  : new URL("./results/", import.meta.url)
+const catalog = JSON.parse(await readFile(argumentsValue[0] ?? new URL("index.json", root), "utf8"))
+assert.equal(catalog.schemaVersion, 1, "unsupported benchmark catalog")
+const ids = new Set()
+const guideTables = new Map()
+for (const dataset of catalog.datasets) {
+  assert.match(dataset.id, /^[a-zA-Z0-9.-]+$/)
+  assert.ok(!ids.has(dataset.id), "duplicate dataset")
+  ids.add(dataset.id)
+  assert.ok(dataset.environment, "missing environment label")
+  const directory = new URL(`${dataset.id}/`, root)
+  const load = async (name) => JSON.parse(await readFile(new URL(name, directory), "utf8"))
+  if (["controlled-comparison", "partial-controlled-comparison"].includes(dataset.kind)) {
+    const run = await load("run.json")
+    const partial = dataset.kind === "partial-controlled-comparison"
+    assert.equal(run.status, partial ? "failed" : "complete", "wrong parent experiment status")
+    assert.ok(
+      run.destroyedAt && !run.cleanupError && !run.downloadError,
+      "unconfirmed teardown or download",
+    )
+    const result = await load("comparison.json")
+    assert.deepEqual(
+      result.comparison,
+      run.profile.comparison,
+      "comparison differs from archived profile",
+    )
+    let scope = result.comparison
+    if (partial) {
+      assert.equal(run.purpose, "incomplete-diagnostics")
+      assert.ok(run.error || run.recoveryError, "missing parent failure evidence")
+      assert.deepEqual(
+        await completedComparison(directory, result.comparison),
+        result,
+        "partial summary must retain every fully completed backend",
+      )
+      assert.ok(
+        result.completedDatabases.length < result.comparison.databases.length,
+        "the whole sampling plan completed; do not label it partial",
+      )
+      assert.deepEqual(
+        dataset.databases,
+        result.completedDatabases,
+        "mislabeled completed backends",
+      )
+      scope = comparisonForDatabases(result.comparison, result.completedDatabases)
+    }
+    const phases = scope.databases.flatMap((database) =>
+      Array.from({ length: result.comparison.rounds }, (_value, index) =>
+        result.comparison.concurrencies.map(
+          (concurrency) => `${database}-round-${index + 1}-concurrency-${concurrency}`,
+        ),
+      ).flat(),
+    )
+    await checkQuiet(directory, run, phases)
+    const samples = await Promise.all(
+      result.samples.map(async (sample) => {
+        assert.match(sample.file, /^[a-zA-Z0-9.-]+\.json$/)
+        return { ...sample, report: await load(sample.file) }
+      }),
+    )
+    const versions = [...new Set(samples.map((sample) => sample.report.packageVersion))].sort()
+    assert.deepEqual(versions, [...dataset.versions].sort(), "mislabeled runtime versions")
+    assert.deepEqual(
+      comparisonRows(samples, scope),
+      result.rows,
+      "summary differs from raw samples",
+    )
+    const markdown = await readFile(new URL("README.md", directory), "utf8")
+    if (partial)
+      assert.match(
+        markdown,
+        /^# Incomplete/m,
+        "partial parent must remain visibly labeled incomplete",
+      )
+    assertPublishedTables(controlledTable(result.rows), markdown)
+    if (dataset.guideTable)
+      guideTables.set(dataset.id, comparisonHeadline({ ...result, comparison: scope }))
+  } else {
+    assert.ok(
+      ["isolated-baseline", "historical-shared-host"].includes(dataset.kind),
+      "unknown dataset kind",
+    )
+    const names = [
+      "sqlite-counter",
+      "sqlite-cart-outbox",
+      "postgresql-counter",
+      "postgresql-cart-outbox",
+      "mysql-counter",
+      "mysql-cart-outbox",
+      "sqlite-large-state",
+      "sqlite-idle",
+    ]
+    const reports = await Promise.all(names.map((name) => load(`${name}.json`)))
+    assert.deepEqual(
+      [...new Set(reports.map((report) => report.packageVersion))],
+      dataset.versions,
+      "mislabeled runtime version",
+    )
+    if (dataset.kind === "isolated-baseline") {
+      const run = await load("run.json")
+      assert.equal(run.status, "complete")
+      assert.ok(
+        run.destroyedAt && !run.cleanupError && !run.downloadError,
+        "unconfirmed teardown or download",
+      )
+      await checkQuiet(directory, run, names)
+      for (const report of reports)
+        assert.equal(report.source.revision, run.runtimeRevision, "wrong archived revision")
+    }
+    const generated = execFileSync(
+      process.execPath,
+      [
+        fileURLToPath(new URL("./summarize.mjs", import.meta.url)),
+        ...names.map((name) => fileURLToPath(new URL(`${name}.json`, directory))),
+      ],
+      { stdio: ["ignore", "pipe", "pipe"], encoding: "utf8" },
+    )
+    assertPublishedTables(generated, await readFile(new URL("README.md", directory), "utf8"))
+    if (dataset.guideTable)
+      guideTables.set(
+        dataset.id,
+        baselineHeadline(reports.filter((report) => report.methodology.workload === "counter")),
+      )
+  }
+}
+for (const diagnostic of catalog.diagnostics ?? []) {
+  assert.match(diagnostic.id, /^[a-zA-Z0-9][a-zA-Z0-9.-]*$/)
+  assert.ok(!ids.has(diagnostic.id), "incomplete attempts cannot also be comparison datasets")
+  const run = JSON.parse(await readFile(new URL(`${diagnostic.id}/run.json`, root), "utf8"))
+  assert.equal(run.status, "failed", "diagnostic attempt must remain labeled incomplete")
+  assert.equal(run.purpose, "incomplete-diagnostics")
+  assert.ok(
+    run.destroyedAt && !run.cleanupError && !run.downloadError,
+    "diagnostic teardown or download unconfirmed",
+  )
+  assert.match(run.resultsSha256, /^[a-f0-9]{64}$/)
+}
+if (guideTables.size > 0) {
+  const path = new URL("../docs/benchmarks.md", import.meta.url)
+  let guide = await readFile(path, "utf8")
+  for (const [id, table] of guideTables) {
+    const marker = `<!-- benchmark-table: ${id} -->`
+    const endMarker = `<!-- /benchmark-table: ${id} -->`
+    const start = guide.indexOf(marker)
+    const end = guide.indexOf(endMarker, start + marker.length)
+    assert.ok(start >= 0 && end > start, `missing generated guide table: ${id}`)
+    const contentStart = start + marker.length
+    if (writeGuide)
+      guide = guide.slice(0, contentStart) + "\n\n" + table + "\n\n" + guide.slice(end)
+    else assertPublishedTables(table, guide.slice(contentStart, end))
+  }
+  if (writeGuide) await writeFile(path, guide)
+}
+console.log(`Validated ${ids.size} versioned benchmark datasets`)
+
+async function checkQuiet(directory, run, phases) {
+  assert.match(run.resultsSha256, /^[a-f0-9]{64}$/, "missing downloaded archive checksum")
+  const rows = (await readFile(new URL("quiet.jsonl", directory), "utf8"))
+    .trim()
+    .split("\n")
+    .map((line) => JSON.parse(line))
+  const limits = run.profile.quiet
+  for (const phase of phases) {
+    const samples = rows.filter((row) => row.phase === phase).slice(-limits.consecutiveSamples)
+    assert.equal(samples.length, limits.consecutiveSamples, `missing readiness samples: ${phase}`)
+    for (const sample of samples) {
+      assert.equal(sample.quiet, true, `host was not ready: ${phase}`)
+      assert.ok(
+        sample.uptimeSeconds >= limits.minimumUptimeSeconds,
+        `boot had not settled: ${phase}`,
+      )
+      for (const [value, maximum] of [
+        [sample.cpuPercent, limits.maximumCpuPercent],
+        [sample.ioWaitPercent, limits.maximumIoWaitPercent],
+        [sample.stealPercent, limits.maximumStealPercent],
+        [sample.diskBytesPerSecond, limits.maximumDiskBytesPerSecond],
+        [sample.loadPerCpu, limits.maximumLoadPerCpu],
+      ]) {
+        assert.ok(
+          Number.isFinite(value) && value >= 0 && value <= maximum,
+          `readiness threshold exceeded: ${phase}`,
+        )
+      }
+    }
+    assert.ok(
+      samples.at(-1).consecutive >= limits.consecutiveSamples,
+      `incomplete quiet period: ${phase}`,
+    )
+  }
+}

benchmarks/comparison-scope.mjs

@@ -0,0 +1,74 @@
+import assert from "node:assert/strict"
+import { readFile } from "node:fs/promises"
+import {
+  comparisonPlan,
+  comparisonRows,
+  validateComparison,
+  validateComparisonReport,
+} from "./comparison.mjs"
+
+export function comparisonForDatabases(comparison, databases) {
+  return {
+    ...comparison,
+    databases,
+    variants: comparison.variants
+      .filter(
+        (variant) =>
+          !variant.databases || variant.databases.some((database) => databases.includes(database)),
+      )
+      .map((variant) =>
+        variant.databases
+          ? {
+              ...variant,
+              databases: variant.databases.filter((database) => databases.includes(database)),
+            }
+          : variant,
+      ),
+  }
+}
+
+export async function completedComparison(directory, comparison) {
+  validateComparison(comparison)
+  const samples = []
+  const completedDatabases = []
+  let plannedReportCount = 0
+  let emittedReportCount = 0
+  for (const database of comparison.databases) {
+    const reports = []
+    let expected = 0
+    for (let round = 1; round <= comparison.rounds; round++) {
+      for (const concurrency of comparison.concurrencies) {
+        for (const variant of comparisonPlan(comparison, database, round)) {
+          expected++
+          const file = `${database}-round-${round}-concurrency-${concurrency}-${variant.name}.json`
+          let text
+          try {
+            text = await readFile(new URL(file, directory), "utf8")
+          } catch (error) {
+            if (error.code === "ENOENT") continue
+            throw error
+          }
+          if (!text.trim()) continue
+          const report = JSON.parse(text)
+          assert.equal(report.database.adapter, database, "wrong database in archived report")
+          validateComparisonReport(report, variant, comparison, concurrency)
+          reports.push({ variant: variant.name, round, concurrency, file, report })
+        }
+      }
+    }
+    plannedReportCount += expected
+    emittedReportCount += reports.length
+    if (reports.length !== expected) continue
+    completedDatabases.push(database)
+    samples.push(...reports)
+  }
+  assert.ok(completedDatabases.length > 0, "no backend completed its sampling plan")
+  return {
+    comparison,
+    completedDatabases,
+    plannedReportCount,
+    emittedReportCount,
+    samples: samples.map(({ report, ...sample }) => sample),
+    rows: comparisonRows(samples, comparisonForDatabases(comparison, completedDatabases)),
+  }
+}