gpu-mode · msaroufim · Sep 7, 2026 · Sep 7, 2026 · Sep 7, 2026 · Sep 7, 2026
diff --git a/.github/workflows/modal-deploy.yml b/.github/workflows/modal-deploy.yml
@@ -27,10 +27,13 @@ jobs:
       - name: Install Modal
         run: |
           python -m pip install --upgrade pip
-          pip install modal
+          pip install "modal>=1.5.0"
+
+      - name: Warm Torch header cache
+        run: |
+          PYTHONPATH=${{ github.workspace }}/src:${{ github.workspace }}/src/runners modal run src/runners/modal_runner.py::warm_pch
 
       - name: Deploy job
         run: |
           # TODO: Remove pythonpath here
-          PYTHONPATH=${{ github.workspace }}/src modal deploy src/runners/modal_runner_archs.py
-
+          PYTHONPATH=${{ github.workspace }}/src:${{ github.workspace }}/src/runners modal deploy src/runners/modal_runner_archs.py
diff --git a/docs/benchmarks/modal-pch.json b/docs/benchmarks/modal-pch.json
@@ -0,0 +1,58 @@
+{
+  "date": "2026-09-07",
+  "gpu": "T4",
+  "cpu": 4,
+  "memory_mib": 16384,
+  "cuda": "13.3.0",
+  "torch": "2.12.0+cu130",
+  "image_id": "im-VGOID7nd5iWWNRTr1Lw2O7",
+  "cutlass_gemm_version": "4.5.2",
+  "volume": "kernelbot-pch-test-20260907",
+  "ninja_parallelism": "default",
+  "baseline": "/usr/bin/g++; common timing shim in both modes; header tracing outside timed builds",
+  "sampling": "Two variants per case, each with/without PCH in separate containers. CUTLASS has two repetitions per variant/mode. Plain C++ uses the median of ten object compilations per container.",
+  "nvcc_timing": "Recorded for the CUTLASS GEMM runs. The earlier matrix instruments CXX only.",
+  "benchmark_sha256": "b22fbb07c7e78a5426544411f6e5dfbd394bfaa4364cf9d8dd0fe7f92f3a039c",
+  "compiler_sha256": "6d82acd795c3e8f69e63f1c1c62fb8afc52bc9c06a1a6a739a385907f26279d1",
+  "runs": {
+    "simple": "https://modal.com/apps/coreauto/main/ap-ZBKWqPp5sSkMJCFjKmWwPI",
+    "other_cases": "https://modal.com/apps/coreauto/main/ap-5aIt3temjLqpwFSfQhW6LI",
+    "cutlass_gemm": "https://modal.com/apps/coreauto/main/ap-6MXYSgeVomyahsGXOTKuG3",
+    "cutlass_gemm_implicit": "https://modal.com/apps/coreauto/main/ap-cMFfCst1YafIPb9GCly6AQ",
+    "all_warmup_profiles": "https://modal.com/apps/coreauto/main/ap-4ym5VrAiAuC127zircLT5b"
+  },
+  "measurements": [
+    {"case":"inline_simple","variant":0,"repeat":0,"cached":false,"build_s":18.800584544,"cpp_s":17.997382098,"nvcc_s":null,"pch_consumed":false,"pch_key":null,"checks":6,"max_error":null,"outputs":[2.0,2.0,2.0,2.0,2.0],"source_sha256":"4ec65ca0b08feea11ce48b7a0ebd322676aab622d35eba1da63b605643e46435","container_id":"ta-01M1YT654VFX478EV36GSNAGCR"},
+    {"case":"inline_simple","variant":0,"repeat":0,"cached":true,"build_s":9.449116172,"cpp_s":8.415279449,"nvcc_s":null,"pch_consumed":true,"pch_key":"cd04740b9ab36ac9a2a8d533e50fc30250ab89f341186829da169aefb0865f38","checks":6,"max_error":null,"outputs":[2.0,2.0,2.0,2.0,2.0],"source_sha256":"4ec65ca0b08feea11ce48b7a0ebd322676aab622d35eba1da63b605643e46435","container_id":"ta-01M1YT65B8E9149HBHXBTA0NAR"},
+    {"case":"inline_simple","variant":1,"repeat":0,"cached":true,"build_s":10.546373582,"cpp_s":9.191087943000001,"nvcc_s":null,"pch_consumed":true,"pch_key":"cd04740b9ab36ac9a2a8d533e50fc30250ab89f341186829da169aefb0865f38","checks":6,"max_error":null,"outputs":[3.0,3.0,3.0,3.0,3.0],"source_sha256":"7e064d69277762a33d2a65a4d6258e5422c7d2b3535694ce9afd6a6304f6993a","container_id":"ta-01M1YT654V66S3D05ZP1KSGBXR"},
+    {"case":"inline_simple","variant":1,"repeat":0,"cached":false,"build_s":18.893621677,"cpp_s":18.087305365,"nvcc_s":null,"pch_consumed":false,"pch_key":null,"checks":6,"max_error":null,"outputs":[3.0,3.0,3.0,3.0,3.0],"source_sha256":"7e064d69277762a33d2a65a4d6258e5422c7d2b3535694ce9afd6a6304f6993a","container_id":"ta-01M1YT65B8TXRZGSM820Z3Q16R"},
+    {"case":"load_files","variant":0,"repeat":0,"cached":false,"build_s":18.484804069,"cpp_s":17.716654833999996,"nvcc_s":null,"pch_consumed":false,"pch_key":null,"checks":6,"max_error":null,"outputs":[2.0,2.0,2.0,2.0,2.0],"source_sha256":"4ec65ca0b08feea11ce48b7a0ebd322676aab622d35eba1da63b605643e46435","container_id":"ta-01M1YT1MEHZQJTJ91K03EYFKJR"},
+    {"case":"load_files","variant":0,"repeat":0,"cached":true,"build_s":8.844153404,"cpp_s":7.813371175,"nvcc_s":null,"pch_consumed":true,"pch_key":"cd04740b9ab36ac9a2a8d533e50fc30250ab89f341186829da169aefb0865f38","checks":6,"max_error":null,"outputs":[2.0,2.0,2.0,2.0,2.0],"source_sha256":"4ec65ca0b08feea11ce48b7a0ebd322676aab622d35eba1da63b605643e46435","container_id":"ta-01M1YT1MM8C9EBBGEZCWXZAW5R"},
+    {"case":"load_files","variant":1,"repeat":0,"cached":true,"build_s":8.704362362000001,"cpp_s":7.6940678259999995,"nvcc_s":null,"pch_consumed":true,"pch_key":"cd04740b9ab36ac9a2a8d533e50fc30250ab89f341186829da169aefb0865f38","checks":6,"max_error":null,"outputs":[3.0,3.0,3.0,3.0,3.0],"source_sha256":"7e064d69277762a33d2a65a4d6258e5422c7d2b3535694ce9afd6a6304f6993a","container_id":"ta-01M1YT20M1E9G09T9PZBWNQ65R"},
+    {"case":"load_files","variant":1,"repeat":0,"cached":false,"build_s":18.118759269999998,"cpp_s":17.33960136,"nvcc_s":null,"pch_consumed":false,"pch_key":null,"checks":6,"max_error":null,"outputs":[3.0,3.0,3.0,3.0,3.0],"source_sha256":"7e064d69277762a33d2a65a4d6258e5422c7d2b3535694ce9afd6a6304f6993a","container_id":"ta-01M1YT276G72J91NSPNG9VWAWR"},
+    {"case":"plain_cpp","variant":0,"repeat":0,"cached":false,"build_s":0.19980638800000028,"cpp_s":0.12402935050000008,"nvcc_s":null,"pch_consumed":false,"pch_key":null,"checks":1,"max_error":null,"outputs":[3],"source_sha256":"f5fbb559817c22a648c4be4067de69dd806e6e9f2c9be71aa84f2371ecf8d781","container_id":"ta-01M1YT2FP2X908DV2VFHCJPT6R"},
+    {"case":"plain_cpp","variant":0,"repeat":0,"cached":true,"build_s":0.2823680209999999,"cpp_s":0.20570012650000002,"nvcc_s":null,"pch_consumed":false,"pch_key":null,"checks":1,"max_error":null,"outputs":[3],"source_sha256":"f5fbb559817c22a648c4be4067de69dd806e6e9f2c9be71aa84f2371ecf8d781","container_id":"ta-01M1YT2QSWZR9SN6VGQJC16VSR"},
+    {"case":"plain_cpp","variant":1,"repeat":0,"cached":true,"build_s":0.29172716450000014,"cpp_s":0.21366047899999963,"nvcc_s":null,"pch_consumed":false,"pch_key":null,"checks":1,"max_error":null,"outputs":[4],"source_sha256":"4f5b6c5b34f3f71e1e94ad89fb640e348e9bd0232d5cdfc75075c5e827674378","container_id":"ta-01M1YT2TY6A6XZNT3DJDM9AW6R"},
+    {"case":"plain_cpp","variant":1,"repeat":0,"cached":false,"build_s":0.20363044750000014,"cpp_s":0.12606111700000033,"nvcc_s":null,"pch_consumed":false,"pch_key":null,"checks":1,"max_error":null,"outputs":[4],"source_sha256":"4f5b6c5b34f3f71e1e94ad89fb640e348e9bd0232d5cdfc75075c5e827674378","container_id":"ta-01M1YT3294D1MHQ774CC40NP6R"},
+    {"case":"flag_miss","variant":0,"repeat":0,"cached":false,"build_s":18.314668328000003,"cpp_s":17.468810722999997,"nvcc_s":null,"pch_consumed":false,"pch_key":null,"checks":6,"max_error":null,"outputs":[2.0,2.0,2.0,2.0,2.0],"source_sha256":"4ec65ca0b08feea11ce48b7a0ebd322676aab622d35eba1da63b605643e46435","container_id":"ta-01M1YT32ZPHKD6NAA1RQHDFZNR"},
+    {"case":"flag_miss","variant":0,"repeat":0,"cached":true,"build_s":20.427356331,"cpp_s":19.146695692999998,"nvcc_s":null,"pch_consumed":false,"pch_key":null,"checks":6,"max_error":null,"outputs":[2.0,2.0,2.0,2.0,2.0],"source_sha256":"4ec65ca0b08feea11ce48b7a0ebd322676aab622d35eba1da63b605643e46435","container_id":"ta-01M1YT36X83KGN6Y6BSXPHJB5R"},
+    {"case":"flag_miss","variant":1,"repeat":0,"cached":true,"build_s":20.175504439,"cpp_s":19.005651779999997,"nvcc_s":null,"pch_consumed":false,"pch_key":null,"checks":6,"max_error":null,"outputs":[3.0,3.0,3.0,3.0,3.0],"source_sha256":"7e064d69277762a33d2a65a4d6258e5422c7d2b3535694ce9afd6a6304f6993a","container_id":"ta-01M1YT3AQERWZZTC7SBEKE1EFR"},
+    {"case":"flag_miss","variant":1,"repeat":0,"cached":false,"build_s":17.837603579,"cpp_s":17.036646379,"nvcc_s":null,"pch_consumed":false,"pch_key":null,"checks":6,"max_error":null,"outputs":[3.0,3.0,3.0,3.0,3.0],"source_sha256":"7e064d69277762a33d2a65a4d6258e5422c7d2b3535694ce9afd6a6304f6993a","container_id":"ta-01M1YT3MBVRB90BKSZW7DR6FHR"},
+    {"case":"cutlass_gemm","variant":0,"repeat":0,"cached":false,"build_s":19.794986437,"cpp_s":18.943877617,"nvcc_s":[12.304675192000001],"pch_consumed":false,"pch_key":null,"checks":10,"max_error":4.2646570364013314e-06,"outputs":[8.0],"source_sha256":"c0cc1e80088a516e7d3b0a21890de234e88d4cf0b2b278c6e8e74d7912d4a14d","container_id":"ta-01M1YTRK5PMETR512ZRYZAWEJR"},
+    {"case":"cutlass_gemm","variant":0,"repeat":0,"cached":true,"build_s":12.692048463999999,"cpp_s":8.008807361,"nvcc_s":[11.653260732],"pch_consumed":true,"pch_key":"cd04740b9ab36ac9a2a8d533e50fc30250ab89f341186829da169aefb0865f38","checks":10,"max_error":4.2646570364013314e-06,"outputs":[8.0],"source_sha256":"c0cc1e80088a516e7d3b0a21890de234e88d4cf0b2b278c6e8e74d7912d4a14d","container_id":"ta-01M1YTRK5PF1P3622Q8VNYA6KR"},
+    {"case":"cutlass_gemm","variant":1,"repeat":0,"cached":true,"build_s":13.076952955000001,"cpp_s":7.930763236000001,"nvcc_s":[11.847514466],"pch_consumed":true,"pch_key":"cd04740b9ab36ac9a2a8d533e50fc30250ab89f341186829da169aefb0865f38","checks":10,"max_error":8.529314072802663e-06,"outputs":[16.0],"source_sha256":"b926f64a92cfd59840083d417ec09e2db0dd93574c97203ad4ac0d08925ccaee","container_id":"ta-01M1YTRM3CV3B2HBFEF32S6QVR"},
+    {"case":"cutlass_gemm","variant":1,"repeat":0,"cached":false,"build_s":19.559281418,"cpp_s":18.747936609999996,"nvcc_s":[12.55567741],"pch_consumed":false,"pch_key":null,"checks":10,"max_error":8.529314072802663e-06,"outputs":[16.0],"source_sha256":"b926f64a92cfd59840083d417ec09e2db0dd93574c97203ad4ac0d08925ccaee","container_id":"ta-01M1YTRKCWZPE610GWKMMNAHFR"},
+    {"case":"cutlass_gemm","variant":0,"repeat":1,"cached":false,"build_s":21.135683762,"cpp_s":20.226324679999998,"nvcc_s":[12.684764970999998],"pch_consumed":false,"pch_key":null,"checks":10,"max_error":4.2646570364013314e-06,"outputs":[8.0],"source_sha256":"c0cc1e80088a516e7d3b0a21890de234e88d4cf0b2b278c6e8e74d7912d4a14d","container_id":"ta-01M1YTSM3BNSJWKC1QNYJ4A7NR"},
+    {"case":"cutlass_gemm","variant":0,"repeat":1,"cached":true,"build_s":12.152523803,"cpp_s":7.759680262,"nvcc_s":[11.084306554000001],"pch_consumed":true,"pch_key":"cd04740b9ab36ac9a2a8d533e50fc30250ab89f341186829da169aefb0865f38","checks":10,"max_error":4.2646570364013314e-06,"outputs":[8.0],"source_sha256":"c0cc1e80088a516e7d3b0a21890de234e88d4cf0b2b278c6e8e74d7912d4a14d","container_id":"ta-01M1YTSN1GCNZ2DRWEXVY91GQR"},
+    {"case":"cutlass_gemm","variant":1,"repeat":1,"cached":true,"build_s":12.466685846,"cpp_s":7.833615612999999,"nvcc_s":[11.412195861],"pch_consumed":true,"pch_key":"cd04740b9ab36ac9a2a8d533e50fc30250ab89f341186829da169aefb0865f38","checks":10,"max_error":8.529314072802663e-06,"outputs":[16.0],"source_sha256":"b926f64a92cfd59840083d417ec09e2db0dd93574c97203ad4ac0d08925ccaee","container_id":"ta-01M1YTT5X2Z3VGZN0ZXA8J8FFR"},
+    {"case":"cutlass_gemm","variant":1,"repeat":1,"cached":false,"build_s":20.099891596000003,"cpp_s":19.24853497,"nvcc_s":[12.522667313000001],"pch_consumed":false,"pch_key":null,"checks":10,"max_error":8.529314072802663e-06,"outputs":[16.0],"source_sha256":"b926f64a92cfd59840083d417ec09e2db0dd93574c97203ad4ac0d08925ccaee","container_id":"ta-01M1YTT4XD62ZGV4CJYD6AM7WR"},
+    {"case":"cutlass_gemm_implicit","variant":0,"repeat":0,"cached":false,"build_s":49.344148073,"cpp_s":20.027217276,"nvcc_s":[48.269329267],"pch_consumed":false,"pch_key":null,"checks":10,"max_error":4.2646570364013314e-06,"outputs":[8.0],"source_sha256":"c0cc1e80088a516e7d3b0a21890de234e88d4cf0b2b278c6e8e74d7912d4a14d","container_id":"ta-01M1YVFWZ4BD7M4B5P4EKGNRSR"},
+    {"case":"cutlass_gemm_implicit","variant":0,"repeat":0,"cached":true,"build_s":44.628304248000006,"cpp_s":8.165473682,"nvcc_s":[43.488331118],"pch_consumed":true,"pch_key":"cd04740b9ab36ac9a2a8d533e50fc30250ab89f341186829da169aefb0865f38","checks":10,"max_error":4.2646570364013314e-06,"outputs":[8.0],"source_sha256":"c0cc1e80088a516e7d3b0a21890de234e88d4cf0b2b278c6e8e74d7912d4a14d","container_id":"ta-01M1YVFWZ4A8FTFDZ76TX6XT6R"},
+    {"case":"cutlass_gemm_implicit","variant":1,"repeat":0,"cached":true,"build_s":45.660435061,"cpp_s":8.445875664,"nvcc_s":[44.495367471],"pch_consumed":true,"pch_key":"cd04740b9ab36ac9a2a8d533e50fc30250ab89f341186829da169aefb0865f38","checks":10,"max_error":8.529314072802663e-06,"outputs":[16.0],"source_sha256":"b926f64a92cfd59840083d417ec09e2db0dd93574c97203ad4ac0d08925ccaee","container_id":"ta-01M1YVFWSFP54HSM30AAW6BC0R"},
+    {"case":"cutlass_gemm_implicit","variant":1,"repeat":0,"cached":false,"build_s":44.895219539,"cpp_s":18.66360581,"nvcc_s":[44.011524792],"pch_consumed":false,"pch_key":null,"checks":10,"max_error":8.529314072802663e-06,"outputs":[16.0],"source_sha256":"b926f64a92cfd59840083d417ec09e2db0dd93574c97203ad4ac0d08925ccaee","container_id":"ta-01M1YVFWSFP2G8HR0D01V9F6JR"},
+    {"case":"cutlass_gemm_implicit","variant":0,"repeat":1,"cached":false,"build_s":46.818345875999995,"cpp_s":18.425574178,"nvcc_s":[45.895286961000004],"pch_consumed":false,"pch_key":null,"checks":10,"max_error":4.2646570364013314e-06,"outputs":[8.0],"source_sha256":"c0cc1e80088a516e7d3b0a21890de234e88d4cf0b2b278c6e8e74d7912d4a14d","container_id":"ta-01M1YVHXJN3XFWSW7KCQEPWSDR"},
+    {"case":"cutlass_gemm_implicit","variant":0,"repeat":1,"cached":true,"build_s":45.564655846,"cpp_s":8.125612553,"nvcc_s":[44.392049244999995],"pch_consumed":true,"pch_key":"cd04740b9ab36ac9a2a8d533e50fc30250ab89f341186829da169aefb0865f38","checks":10,"max_error":4.2646570364013314e-06,"outputs":[8.0],"source_sha256":"c0cc1e80088a516e7d3b0a21890de234e88d4cf0b2b278c6e8e74d7912d4a14d","container_id":"ta-01M1YVHZH53EH13G4H5Y2PH07R"},
+    {"case":"cutlass_gemm_implicit","variant":1,"repeat":1,"cached":true,"build_s":50.934128441,"cpp_s":9.066257041,"nvcc_s":[49.628436083],"pch_consumed":true,"pch_key":"cd04740b9ab36ac9a2a8d533e50fc30250ab89f341186829da169aefb0865f38","checks":10,"max_error":8.529314072802663e-06,"outputs":[16.0],"source_sha256":"b926f64a92cfd59840083d417ec09e2db0dd93574c97203ad4ac0d08925ccaee","container_id":"ta-01M1YVJ80Q2MBQKQXP74XRF2HR"},
+    {"case":"cutlass_gemm_implicit","variant":1,"repeat":1,"cached":false,"build_s":53.681691927,"cpp_s":21.277357138000003,"nvcc_s":[52.342306416],"pch_consumed":false,"pch_key":null,"checks":10,"max_error":8.529314072802663e-06,"outputs":[16.0],"source_sha256":"b926f64a92cfd59840083d417ec09e2db0dd93574c97203ad4ac0d08925ccaee","container_id":"ta-01M1YVJETDXTHV0W4J3E158HYR"}
+  ]
+}
diff --git a/docs/modal-pch-cache.md b/docs/modal-pch-cache.md
@@ -0,0 +1,102 @@
+# Modal Torch header cache
+
+The runner sets `CXX` to a GCC wrapper that loads a precompiled
+`torch/extension.h` from a Modal Volume. Ordinary `load_inline()` calls need no
+changes. Leave `use_pch=False`; PyTorch's separate native PCH option is unnecessary.
+The submission path assumes `load_inline()`.
+
+Only the Torch host header is cached. CUDA/CUTLASS templates, user headers, kernel
+objects, and shared libraries still compile in each submission container.
+Different sources and module names share a PCH when their host flags match;
+changing a user header after the Torch include still changes the compiled code.
+
+## Profiles and fallback
+
+The CPU warmer builds CPU-only and CUDA-linked profiles for these `extra_cflags`:
+`[]`, `["-O2"]`, `["-O3"]`, and `["-O3", "-ffast-math"]`.
+Keys include installed headers, compiler, Python/Torch versions, host flags, and
+include environment. CUDA-only flags do not change the host key.
+
+Unknown profiles compile normally; sources without the initial Torch include
+bypass PCH. With `no_implicit_headers=True`, explicitly include
+`<torch/extension.h>` first in C++ to use the cache. Python/Triton code that does
+not invoke `CXX` is unaffected.
+
+Misses and bypasses still pay wrapper startup cost. There is no guarantee of a
+speedup. Set `CXX=/usr/bin/g++` before building to bypass the wrapper entirely.
+`KERNELBOT_PCH_DISABLE=1` disables cache lookup but retains wrapper startup.
+
+## Deployment
+
+The main/dev workflow warms profiles before deploying GPU runners. Manually,
+run both commands in the same Modal profile/environment:
+
+```sh
+PYTHONPATH=src:src/runners uv run modal run src/runners/modal_runner.py::warm_pch
+PYTHONPATH=src:src/runners uv run modal deploy src/runners/modal_runner_archs.py
+```
+
+`KERNELBOT_PCH_VOLUME` overrides the default `kernelbot-torch-pch` Volume.
+`warm_pch --profiles cuda-default,cuda-O3` warms selected profiles.
+Rewarm after changing the image or installed headers, then replace existing
+runner containers to refresh their Volume snapshots.
+
+The CPU warmer accepts fixed profiles and commits completed headers. GPU runners
+mount the Volume read-only and use `restrict_modal_access=True`; otherwise the
+Modal upload API could bypass the read-only mount. Restricted runners cannot
+reload the Volume themselves. GCC validates a PCH before using it.
+
+## Benchmark
+
+```sh
+PYTHONPATH=src:src/runners KERNELBOT_PCH_VOLUME=kernelbot-pch-test \
+  uv run modal run scripts/modal_pch_benchmark.py
+# Actual tensor-core GEMM, two tile variants, two repetitions per mode:
+PYTHONPATH=src:src/runners KERNELBOT_PCH_VOLUME=kernelbot-pch-test \
+  uv run modal run scripts/modal_pch_benchmark.py --cases cutlass_gemm,cutlass_gemm_implicit --repeats 2
+```
+
+Every build runs in a fresh T4 container (4 CPU cores, 16 GiB), with normal Ninja
+parallelism. The baseline uses `/usr/bin/g++`; both modes use the same timing
+instrumentation. GCC header tracing verifies actual PCH use in a separate,
+untimed compilation. The tests check changed headers and source against distinct
+expected outputs. The script also retains file-based and plain C++ diagnostics;
+these are not representative submission paths.
+
+The CUTLASS case instantiates SM75 tensor-core GEMMs with FP16 inputs, FP32
+accumulation/output, 128×128×32 and 128×64×32 threadblock tiles. It checks
+`alpha * A @ B.T` for random and zero inputs, tile tails, and a non-default CUDA
+stream against a float64 PyTorch reference (`rtol=atol=2e-4`). This measures compile
+time, not GEMM execution speed.
+
+## Results
+
+Measured on CUDA 13.3 / PyTorch 2.12.0+cu130, T4. Values are medians from separate
+baseline and cached containers; two variants per case, with two repetitions for
+CUTLASS. The table covers `load_inline()` builds.
+
+| Build | Baseline | Cached |
+| --- | ---: | ---: |
+| Minimal CUDA headers | 18.85s | 10.00s |
+| CUTLASS GEMM, default implicit CUDA headers | 48.08s | 45.61s |
+| CUTLASS GEMM, `no_implicit_headers=True` | 19.95s | 12.58s |
+| Unwarmed host flags | 18.08s | 20.30s |
+
+With default implicit CUDA headers, the CUTLASS GEMM improved only **1.05×**,
+with overlapping baseline/cached timing ranges. With `no_implicit_headers=True`,
+it improved **1.59×** (37% less time). In that case, host compilation fell from
+19.10s to 7.88s, while NVCC took 12.54s and 11.53s and became the bottleneck.
+The PCH does not cache CUDA headers or templates. Keeping Torch headers out of
+the CUDA translation unit produces the larger gain for this GEMM; the fixture
+keeps Tensor handling in C++ and explicitly includes its required headers.
+
+All 160 GEMM checks passed, with maximum absolute error 8.53e-6. The remaining
+cases, including the supplemental diagnostics, passed another 76 checks.
+All eight warmup profiles also completed and
+committed on this image.
+
+The `load_inline()` case with unwarmed host flags was slower in this sample.
+Separate containers introduce timing variability, so the measured difference
+does not isolate wrapper overhead. Larger CUDA builds may hide the host-side
+savings entirely.
+[Per-container results and Modal runs](benchmarks/modal-pch.json).
diff --git a/pyproject.toml b/pyproject.toml
@@ -15,7 +15,7 @@ dependencies = [
     "audioop-lts; python_version >= '3.13'",
     "python-dotenv",
     "requests",
-    "modal",
+    "modal>=1.5.0",
     "psycopg2-binary",
     "yoyo-migrations",
     "better_profanity",