diff --git a/README.md b/README.md index 39811eb..db2f648 100644 --- a/README.md +++ b/README.md @@ -259,6 +259,7 @@ scripts/ extract_bazel.py bazel aquery jsonproto → model.json (+ role classify) diff.py role-filtered, TU-set parity diff → worklist + converged triage.py groups diff.json into a systematic-cause worklist + estimate_cost.py transparent engineering-effort range from model + diff serialize.py model ↔ JSON (the contract between stages) tests/ test_engine.py diff/canonicalize/roles/config/TU-set behavior @@ -291,10 +292,33 @@ python3 scripts/diff.py model.cmake.json model.bazel.json cmake2bazel.json > dif # 5. Triage — group the worklist by systematic cause before fixing. python3 scripts/triage.py diff.json + +# 6. Estimate — starts from CMake surface area; use the diff to revise it. +python3 scripts/estimate_cost.py model.cmake.json --diff diff.json \ + --hourly-rate 180 > migration-estimate.json ``` Run as a skill, Claude drives step 2 and the triage/fix loop automatically. +## Estimation + +`estimate_cost.py` is a deliberately visible engineering-effort heuristic. It +reports a low/likely/high hour range, optional cost at a supplied hourly rate, +and scope-risk flags. It does not invent an LLM price: record provider, model +ID, region, token totals, retries, and a pricing snapshot separately. + +Once token totals and a dated pricing snapshot are known, add +`--llm-input-tokens`, `--llm-output-tokens`, `--llm-input-per-million`, and +`--llm-output-per-million` to report that API spend separately from engineering +cost. + +An example that can run without CMake or Bazel is included: + +```bash +python3 scripts/estimate_cost.py examples/cost-estimate.model.cmake.json \ + --diff examples/cost-estimate.diff.json --hourly-rate 180 +``` + ## Tests ```bash @@ -302,6 +326,7 @@ python3 tests/test_engine.py python3 tests/test_extractors.py python3 tests/test_triage.py python3 tests/test_configure.py +python3 tests/test_estimate_cost.py ``` The extractor tests run against fixtures under `tests/` that mirror the diff --git a/SKILL.md b/SKILL.md index 0fdd2fa..b117ecb 100644 --- a/SKILL.md +++ b/SKILL.md @@ -198,6 +198,25 @@ python3 scripts/extract_cmake.py model.cmake.json /tr > the content differ are TODO (see `docs/TODO-configure-time-generation.md`). > This is distinct from build-time codegen (genrules), which is also unmodeled. +### 2a. (Optional) Estimate migration cost +Estimate scope immediately after the CMake extraction, then re-run with the +diff once the first Bazel extraction exists. The estimate is a transparent +engineering-effort range, not a vendor or LLM price quote. +```bash +python3 scripts/estimate_cost.py model.cmake.json --hourly-rate 180 \ + > migration-estimate.initial.json +``` +After step 5, tighten it with observed parity gaps: +```bash +python3 scripts/estimate_cost.py model.cmake.json --diff diff.json \ + --hourly-rate 180 > migration-estimate.json +``` +Its inputs and per-discrepancy weights are in the emitted JSON. Treat codegen, +configure-time generation, and unknown roles as explicit scope risks. Calibrate +the weights against completed migrations; do not present the result as a fixed +bid. Keep LLM/API cost separate until token totals and a dated pricing snapshot +are available. + ### 3. Generate initial BUILD.bazel files *(LLM step)* Read `model.cmake.json`. For each production target emit a `cc_library` / `cc_binary` with `srcs`, `hdrs`, `copts`, `defines`, `includes`, `deps`. Library @@ -309,7 +328,10 @@ Once production parity is reached, opt into test diffing: Summarize: production targets reconciled, rounds taken, suppressions recorded in `cmake2bazel.json` (with rationale), excluded roles (dashboard/codegen) for human follow-up, and — if `include_tests` was on — test-source parity and any -test-binary count gap. +test-binary count gap. Include the final `migration-estimate.json`. For LLM/API +cost, report the provider, model ID, region, token totals, retries, and the +pricing snapshot separately; the repository cannot infer these from build +artifacts. ## What you edit @@ -321,7 +343,8 @@ per-iteration judgment goes into the generated `BUILD.bazel`/`MODULE.bazel` and ```bash python3 tests/test_engine.py && python3 tests/test_extractors.py \ - && python3 tests/test_triage.py && python3 tests/test_configure.py + && python3 tests/test_triage.py && python3 tests/test_configure.py \ + && python3 tests/test_estimate_cost.py ``` Extractor tests run against fixtures that mirror the documented File API and diff --git a/examples/cost-estimate.diff.json b/examples/cost-estimate.diff.json new file mode 100644 index 0000000..f2e0687 --- /dev/null +++ b/examples/cost-estimate.diff.json @@ -0,0 +1,10 @@ +{ + "converged": false, + "discrepancies": [ + {"kind": "flags_diff", "severity": "error"}, + {"kind": "missing_dep", "severity": "error"} + ], + "errors": 2, + "excluded": {}, + "warnings": 0 +} diff --git a/examples/cost-estimate.model.cmake.json b/examples/cost-estimate.model.cmake.json new file mode 100644 index 0000000..6246f3a --- /dev/null +++ b/examples/cost-estimate.model.cmake.json @@ -0,0 +1,19 @@ +{ + "build_system": "cmake", + "configured_files": {}, + "repo_root": "/example/project", + "targets": { + "app": { + "actions": [{"arguments": ["-c", "app/main.cc"], "inputs": [], "mnemonic": "CppCompile", "outputs": []}], + "deps": [{"external": true, "name": "fmt"}], + "kind": "executable", + "role": "production" + }, + "core": { + "actions": [{"arguments": ["-c", "src/core.cc"], "inputs": [], "mnemonic": "CppCompile", "outputs": []}], + "deps": [], + "kind": "static_library", + "role": "production" + } + } +} diff --git a/scripts/estimate_cost.py b/scripts/estimate_cost.py new file mode 100644 index 0000000..da41c12 --- /dev/null +++ b/scripts/estimate_cost.py @@ -0,0 +1,200 @@ +"""Estimate CMake-to-Bazel migration effort from extracted migration artifacts. + +This is deliberately a transparent engineering-effort heuristic, not a quote. +It consumes the CMake reference model and, once available, the parity diff. The +model measures migration surface; the diff measures the remaining work. +LLM/API spend is reported only when priced token usage is supplied separately. + +Usage: + python3 scripts/estimate_cost.py model.cmake.json + python3 scripts/estimate_cost.py model.cmake.json --diff diff.json \ + --hourly-rate 180 > migration-estimate.json +""" + +from __future__ import annotations + +import argparse +import json +import math +from collections import Counter +from typing import Optional + +from model import TargetKind, TargetRole +from serialize import load_model + + +# Hours added for each remaining discrepancy. These are intentionally visible: +# calibrate them with completed migrations rather than treating them as truth. +DIFF_HOURS = { + "missing_target": 1.5, + "kind_mismatch": 1.0, + "missing_tu": 0.20, + "missing_java_src": 0.20, + "defines_diff": 0.20, + "includes_diff": 0.25, + "flags_diff": 0.20, + "link_flags_diff": 0.35, + "missing_dep": 0.75, + "missing_test_tu": 0.15, + "test_binary_count": 0.25, +} + + +def _round_hour(value: float) -> float: + return round(value * 2) / 2 + + +def model_metrics(model) -> dict: + targets = list(model.targets.values()) + roles = Counter(t.role.value for t in targets) + kinds = Counter(t.kind.value for t in targets) + compile_actions = sum( + 1 for target in targets for action in target.actions + if "Compile" in action.mnemonic + ) + external_deps = { + dep.name for target in targets for dep in target.deps if dep.external + } + return { + "targets": len(targets), + "roles": dict(sorted(roles.items())), + "kinds": dict(sorted(kinds.items())), + "compile_actions": compile_actions, + "external_dependencies": len(external_deps), + "configured_files": len(model.configured_files), + "production_targets": roles[TargetRole.PRODUCTION.value], + "test_targets": roles[TargetRole.TEST.value], + "codegen_targets": roles[TargetRole.CODEGEN.value], + "unknown_targets": roles[TargetRole.UNKNOWN.value], + "executables": kinds[TargetKind.EXECUTABLE.value], + } + + +def estimate(model, diff: Optional[dict] = None, hourly_rate: Optional[float] = None, + llm_input_tokens: Optional[int] = None, + llm_output_tokens: Optional[int] = None, + llm_input_per_million: Optional[float] = None, + llm_output_per_million: Optional[float] = None) -> dict: + metrics = model_metrics(model) + # Initial porting effort, before Bazel extraction reveals concrete gaps. + likely = ( + 4.0 + + metrics["production_targets"] * 0.75 + + metrics["compile_actions"] * 0.12 + + metrics["executables"] * 0.75 + + metrics["external_dependencies"] * 1.0 + + metrics["test_targets"] * 0.15 + + metrics["configured_files"] * 1.5 + + metrics["codegen_targets"] * 8.0 + + metrics["unknown_targets"] * 2.0 + ) + discrepancy_counts = Counter() + errors = warnings = 0 + if diff is not None: + discrepancy_counts = Counter(d.get("kind", "unknown") + for d in diff.get("discrepancies", [])) + errors = int(diff.get("errors", 0)) + warnings = int(diff.get("warnings", 0)) + likely += sum(DIFF_HOURS.get(kind, 0.25) * count + for kind, count in discrepancy_counts.items()) + + risk_flags = [] + if metrics["codegen_targets"]: + risk_flags.append("build-time code generation is outside the current MVP") + if metrics["configured_files"]: + risk_flags.append("configure-time generated files need a separate parity review") + if metrics["unknown_targets"]: + risk_flags.append("some target roles are unclassified and need scope review") + if diff is None: + risk_flags.append("no Bazel diff supplied; estimate covers discovery and initial port only") + + risk_multiplier = 1.0 + (0.10 if risk_flags else 0.0) + low = max(2.0, likely * 0.70) + high = likely * (1.60 * risk_multiplier) + likely = _round_hour(likely) + low, high = _round_hour(low), _round_hour(high) + rounds = max(1, math.ceil(1 + errors / max(8, metrics["compile_actions"] ** 0.5 * 3))) + if metrics["codegen_targets"] or metrics["configured_files"]: + rounds += 1 + + result = { + "schema_version": 1, + "estimator": "cmake2bazel-engineering-effort-heuristic-v1", + "input": metrics, + "remaining_diff": { + "available": diff is not None, + "errors": errors, + "warnings": warnings, + "by_kind": dict(sorted(discrepancy_counts.items())), + }, + "estimate": { + "engineering_hours": {"low": low, "likely": likely, "high": high}, + "migration_rounds": {"low": max(1, rounds - 1), "likely": rounds, + "high": rounds + 1 + len(risk_flags)}, + "risk_flags": risk_flags, + }, + "assumptions": [ + "one engineer already familiar with Bazel reviews the migration", + "external dependencies can be resolved without writing new rules", + "generated code, packaging, and install rules are scoped separately", + "LLM/API spend is excluded until model-specific usage and pricing are recorded", + ], + } + if hourly_rate is not None: + result["estimate"]["engineering_cost"] = { + "currency": "USD", + "hourly_rate": hourly_rate, + "low": _round_hour(low * hourly_rate), + "likely": _round_hour(likely * hourly_rate), + "high": _round_hour(high * hourly_rate), + } + llm_values = (llm_input_tokens, llm_output_tokens, llm_input_per_million, + llm_output_per_million) + if any(value is not None for value in llm_values): + if any(value is None for value in llm_values): + result["estimate"]["llm_api_cost"] = { + "available": False, + "reason": "provide input/output token totals and both per-million prices", + } + else: + result["estimate"]["llm_api_cost"] = { + "available": True, + "currency": "USD", + "input_tokens": llm_input_tokens, + "output_tokens": llm_output_tokens, + "input_per_million": llm_input_per_million, + "output_per_million": llm_output_per_million, + "total": round( + llm_input_tokens * llm_input_per_million / 1_000_000 + + llm_output_tokens * llm_output_per_million / 1_000_000, 4), + } + return result + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("model", help="CMake model JSON from extract_cmake.py") + parser.add_argument("--diff", help="optional diff.json from diff.py") + parser.add_argument("--hourly-rate", type=float, + help="optional fully-loaded engineering hourly rate in USD") + parser.add_argument("--llm-input-tokens", type=int, + help="observed total input tokens for this migration") + parser.add_argument("--llm-output-tokens", type=int, + help="observed total output tokens for this migration") + parser.add_argument("--llm-input-per-million", type=float, + help="pricing snapshot: USD per million input tokens") + parser.add_argument("--llm-output-per-million", type=float, + help="pricing snapshot: USD per million output tokens") + args = parser.parse_args() + diff = None + if args.diff: + with open(args.diff) as f: + diff = json.load(f) + print(json.dumps(estimate( + load_model(args.model), diff, args.hourly_rate, args.llm_input_tokens, + args.llm_output_tokens, args.llm_input_per_million, + args.llm_output_per_million), indent=2, sort_keys=True)) + + +if __name__ == "__main__": + main() diff --git a/tests/test_estimate_cost.py b/tests/test_estimate_cost.py new file mode 100644 index 0000000..7170328 --- /dev/null +++ b/tests/test_estimate_cost.py @@ -0,0 +1,76 @@ +"""Tests for the transparent migration-effort estimator.""" + +import os +import sys + +sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "scripts")) + +from estimate_cost import estimate, model_metrics +from model import (Action, BuildSystem, CanonicalModel, ConfiguredFile, + Dependency, Target, TargetKind, TargetRole) + + +def _model(): + model = CanonicalModel(build_system=BuildSystem.CMAKE, repo_root="/work/repo") + model.add(Target( + "app", TargetKind.EXECUTABLE, role=TargetRole.PRODUCTION, + actions=[Action("CppCompile"), Action("CppLink")], + deps=[Dependency("fmt", external=True)], + )) + model.add(Target("generator", TargetKind.UNKNOWN, role=TargetRole.CODEGEN)) + model.add_configured_file(ConfiguredFile("config.h", "/tmp/config.h")) + return model + + +def test_metrics_count_migration_surface(): + metrics = model_metrics(_model()) + assert metrics["production_targets"] == 1 + assert metrics["compile_actions"] == 1 + assert metrics["external_dependencies"] == 1 + assert metrics["codegen_targets"] == 1 + + +def test_diff_increases_estimate_and_identifies_risk(): + initial = estimate(_model()) + with_diff = estimate(_model(), { + "errors": 2, + "warnings": 0, + "discrepancies": [ + {"kind": "missing_dep"}, + {"kind": "flags_diff"}, + ], + }) + assert with_diff["estimate"]["engineering_hours"]["likely"] > \ + initial["estimate"]["engineering_hours"]["likely"] + assert "build-time code generation is outside the current MVP" in \ + with_diff["estimate"]["risk_flags"] + + +def test_optional_hourly_rate_produces_engineering_cost_only(): + result = estimate(_model(), hourly_rate=200) + cost = result["estimate"]["engineering_cost"] + assert cost["currency"] == "USD" + assert cost["likely"] == result["estimate"]["engineering_hours"]["likely"] * 200 + assert "LLM/API spend is excluded" in result["assumptions"][-1] + + +def test_priced_observed_tokens_produce_separate_llm_cost(): + result = estimate(_model(), llm_input_tokens=2_000_000, + llm_output_tokens=500_000, llm_input_per_million=3, + llm_output_per_million=15) + cost = result["estimate"]["llm_api_cost"] + assert cost["available"] + assert cost["total"] == 13.5 + + +if __name__ == "__main__": + import traceback + fns = [v for k, v in sorted(globals().items()) if k.startswith("test_")] + failed = 0 + for fn in fns: + try: + fn(); print(f"PASS {fn.__name__}") + except Exception: + failed += 1; print(f"FAIL {fn.__name__}"); traceback.print_exc() + print(f"\n{len(fns) - failed}/{len(fns)} passed") + sys.exit(1 if failed else 0)