From 2dc7448691797049640a5a0e5abd5db738fae71d Mon Sep 17 00:00:00 2001 From: "pieter.noordhuis" Date: Mon, 9 Nov 2026 11:25:08 +0000 Subject: [PATCH 1/4] Add acceptance test that fails when a package init() is slow go-runewidth v0.0.26/v0.0.27 filled a 2 MB table in init(), adding ~20 ms to every CLI invocation. It arrived through a transitive Charm upgrade and went unnoticed (fixed in #7391). The new acceptance test runs `$CLI --version` with GODEBUG=inittrace=1 and fails if any package's init clock exceeds 5 ms. Each package is judged by its minimum over 5 runs, extended up to 20 runs while a check fails, so load from parallel tests does not cause flakes while a real regression fails on every run. It also records a per-package init time budget in out.initlimits.txt, initialized at 4x the measured minimum (at least 0.1 ms; unlisted packages get 0.1 ms). A package that exceeds its budget gets a new one, so several- fold slowdowns show up as a diff in review without flaking on noise. The test runs on linux/amd64 only: Windows inittrace clocks can be quantized to the system tick, and the set of packages with an init differs per architecture. Co-authored-by: Isaac --- acceptance/bin/check_init_time.py | 157 +++++++++++++++++++++++++ acceptance/inittime/out.initlimits.txt | 38 ++++++ acceptance/inittime/out.test.toml | 4 + acceptance/inittime/output.txt | 1 + acceptance/inittime/script | 12 ++ acceptance/inittime/test.toml | 6 + 6 files changed, 218 insertions(+) create mode 100755 acceptance/bin/check_init_time.py create mode 100644 acceptance/inittime/out.initlimits.txt create mode 100644 acceptance/inittime/out.test.toml create mode 100644 acceptance/inittime/output.txt create mode 100644 acceptance/inittime/script create mode 100644 acceptance/inittime/test.toml diff --git a/acceptance/bin/check_init_time.py b/acceptance/bin/check_init_time.py new file mode 100755 index 00000000000..95b2d9d304a --- /dev/null +++ b/acceptance/bin/check_init_time.py @@ -0,0 +1,157 @@ +#!/usr/bin/env python3 +""" +Fail if any Go package's init() clock time, per GODEBUG=inittrace=1, exceeds a threshold, +and keep a golden file with an upper bound on each package's init time. + +Each package is judged by its minimum over several runs, since load from parallel tests +inflates individual runs but a slow init is slow on every run. Load comes in bursts that +span consecutive runs, so packages over a limit get more runs (up to --max-runs). + +The golden file lists packages with a non-trivial init and their tolerated maximum; any +other package's maximum is MIN_LIMIT_MS. A recorded maximum is kept as long as the package +stays under it, so the file only changes when a package becomes several times slower (or +disappears). In that case the maximum is raised to TOLERANCE times the new measurement and +the diff shows up for review. Delete the file and run with -update to re-record all limits. +""" + +import argparse +import math +import os +import re +import subprocess +import sys + +INIT_LINE = re.compile(r"^init (\S+) @\S+ ms, (\S+) ms clock") + +# 2x is exceeded by load spikes when the file was recorded on a machine ~1.3x faster than +# CI; 3x was not exceeded in any measurement, and 4x leaves margin for slower runners. +TOLERANCE = 4 + +# Limit for packages not in the golden file. Most inits report under 0.01 ms, where +# relative noise is meaningless and the cost is immaterial. +MIN_LIMIT_MS = 0.1 + + +def header(max_ms): + return f"""\ +# Init time BUDGET per Go package, in ms of init() clock time (GODEBUG=inittrace=1), +# measured as the minimum over several runs of `databricks --version` on linux/amd64. +# +# Each budget is initialized at {TOLERANCE}x the package's measured minimum, for headroom. +# The smallest budget is {MIN_LIMIT_MS} ms; packages not listed here have a budget of {MIN_LIMIT_MS} ms. +# A package that exceeds its budget gets a new budget of {TOLERANCE}x its new minimum: the test +# fails on the diff, which you accept with -update after review. Budgets never shrink; +# delete this file and run with -update to re-measure all of them. +# +# Independently, the test fails if any package takes over {max_ms:g} ms; -update cannot raise that. +# Generated by acceptance/bin/check_init_time.py. +""" + + +def is_recorded(pkg): + # Names and set of FIPS module packages depend on GOFIPS140, which differs between + # `task test` and plain `go test`. + return not pkg.startswith("crypto/internal/fips140") + + +def measure(command): + # Some inits search PATH (github.com/atotto/clipboard looks up clipboard tools), so their + # time depends on the machine's PATH; a fixed PATH keeps the budgets comparable. + env = dict(os.environ, GODEBUG="inittrace=1", PATH=os.defpath) + cp = subprocess.run(command, env=env, stdout=subprocess.DEVNULL, stderr=subprocess.PIPE, text=True) + if cp.returncode != 0: + sys.exit(f"command failed with exit code {cp.returncode}:\n{cp.stderr}") + clocks = {m.group(1): float(m.group(2)) for m in map(INIT_LINE.match, cp.stderr.splitlines()) if m} + if not clocks: + sys.exit(f"no inittrace output from {command}:\n{cp.stderr}") + return clocks + + +def read_limits(path): + limits = {} + if os.path.exists(path): + with open(path) as f: + for line in f: + if line.strip() and not line.startswith("#"): + limit, pkg = line.split() + limits[pkg] = float(limit) + return limits + + +def format_limit(ms): + # Aligned on the decimal point so cheap and expensive packages are easy to tell apart. + whole, frac = (f"{ms:.1f}" if ms >= 1 else f"{ms:.2f}").split(".") + return f"{whole:>3}.{frac:<2}" + + +def new_limit(ms): + # Round up to two significant digits so the limit is never below TOLERANCE * ms. + limit = TOLERANCE * ms + scale = 10 if limit >= 1 else 100 + return math.ceil(limit * scale) / scale + + +def over_limit(runs, limits): + pkgs = filter(is_recorded, runs[0]) + return [p for p in pkgs if min(r[p] for r in runs) > limits.get(p, MIN_LIMIT_MS)] + + +def find_failures(runs, max_ms): + failures = [] + for pkg in runs[0]: + clocks = [r[pkg] for r in runs] + if min(clocks) > max_ms: + failures.append( + f"init of {pkg} took {min(clocks):.2f} ms (min) / {max(clocks):.2f} ms (max) " + f"over {len(runs)} runs; threshold is {max_ms} ms" + ) + return failures + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("--runs", type=int, default=5) + parser.add_argument("--max-runs", type=int, default=20) + parser.add_argument("--max-ms", type=float, required=True) + parser.add_argument("--limits", required=True, help="golden file name, read from $TESTDIR and written to cwd") + parser.add_argument("command", nargs="+") + args = parser.parse_args() + + limits_path = os.path.join(os.environ["TESTDIR"], args.limits) + # When recording from scratch, list every package whose limit exceeds MIN_LIMIT_MS so + # each gets TOLERANCE headroom. Afterwards only exceeded limits are added, so packages + # measuring near MIN_LIMIT_MS / TOLERANCE don't flip in and out of the file. + record_all = not os.path.exists(limits_path) + limits = read_limits(limits_path) + + runs = [measure(args.command) for _ in range(args.runs)] + while len(runs) < args.max_runs and (find_failures(runs, args.max_ms) or over_limit(runs, limits)): + runs.append(measure(args.command)) + + exceeded = set(over_limit(runs, limits)) + with open(args.limits, "w") as f: + f.write(header(args.max_ms)) + for pkg in sorted(filter(is_recorded, runs[0])): + limit = limits.get(pkg) + if pkg in exceeded or (record_all and limit is None): + limit = new_limit(min(r[pkg] for r in runs)) + limit = limit if limit > MIN_LIMIT_MS else None + if limit is not None: + f.write(f"{format_limit(limit)} {pkg}\n") + + failures = find_failures(runs, args.max_ms) + if not failures: + print("OK: package init times are within threshold") + return + + print("FAIL: Go package init() is slow; this adds latency to every CLI invocation.") + for f in failures: + print(f" {f}") + print("To investigate:") + print(" GODEBUG=inittrace=1 databricks --version 2>&1 | sort -k5 -n -r | head") + print(" go mod why -m ") + sys.exit(1) + + +if __name__ == "__main__": + main() diff --git a/acceptance/inittime/out.initlimits.txt b/acceptance/inittime/out.initlimits.txt new file mode 100644 index 00000000000..35300dd7c7e --- /dev/null +++ b/acceptance/inittime/out.initlimits.txt @@ -0,0 +1,38 @@ +# Init time BUDGET per Go package, in ms of init() clock time (GODEBUG=inittrace=1), +# measured as the minimum over several runs of `databricks --version` on linux/amd64. +# +# Each budget is initialized at 4x the package's measured minimum, for headroom. +# The smallest budget is 0.1 ms; packages not listed here have a budget of 0.1 ms. +# A package that exceeds its budget gets a new budget of 4x its new minimum: the test +# fails on the diff, which you accept with -update after review. Budgets never shrink; +# delete this file and run with -update to re-measure all of them. +# +# Independently, the test fails if any package takes over 5 ms; -update cannot raise that. +# Generated by acceptance/bin/check_init_time.py. + 2.5 encoding/gob + 0.29 github.com/BurntSushi/toml/internal + 0.72 github.com/Masterminds/semver/v3 + 0.26 github.com/charmbracelet/bubbletea + 0.15 github.com/charmbracelet/x/ansi + 1.2 github.com/databricks/cli/bundle/config + 0.14 github.com/databricks/cli/bundle/direct/dresources + 0.44 github.com/databricks/cli/bundle/internal/validation/generated + 0.17 github.com/databricks/cli/bundle/libraries + 0.26 github.com/databricks/cli/bundle/terraform_dabs_map + 3.1 github.com/databricks/cli/cmd/air + 0.13 github.com/databricks/cli/cmd/root + 0.60 github.com/databricks/cli/libs/dyn/dynvar + 0.31 github.com/databricks/cli/libs/localenv + 1.0 github.com/databricks/cli/libs/structs/structpath + 0.23 github.com/databricks/cli/libs/tags + 0.15 github.com/databricks/cli/libs/telemetry + 0.40 github.com/databricks/databricks-sdk-go/apierr + 0.15 github.com/databricks/databricks-sdk-go/config + 0.16 github.com/databricks/databricks-sdk-go/useragent + 0.12 github.com/google/s2a-go/internal/proto/common_go_proto + 0.14 go.yaml.in/yaml/v3 + 0.21 golang.org/x/crypto/ssh + 0.13 golang.org/x/net/trace + 0.29 google.golang.org/grpc + 0.18 google.golang.org/protobuf/internal/detrand + 0.17 google.golang.org/protobuf/internal/encoding/json diff --git a/acceptance/inittime/out.test.toml b/acceptance/inittime/out.test.toml new file mode 100644 index 00000000000..44cce516e73 --- /dev/null +++ b/acceptance/inittime/out.test.toml @@ -0,0 +1,4 @@ +Cloud = false +GOOS.darwin = false +GOOS.linux = true +GOOS.windows = false diff --git a/acceptance/inittime/output.txt b/acceptance/inittime/output.txt new file mode 100644 index 00000000000..3f500562ff4 --- /dev/null +++ b/acceptance/inittime/output.txt @@ -0,0 +1 @@ +OK: package init times are within threshold diff --git a/acceptance/inittime/script b/acceptance/inittime/script new file mode 100644 index 00000000000..3b9deff0a1b --- /dev/null +++ b/acceptance/inittime/script @@ -0,0 +1,12 @@ +# Package init() runs on every CLI invocation, so a slow one (often arriving through a +# transitive dependency upgrade) delays every command. Today the slowest package takes +# ~1 ms; go-runewidth v0.0.27 took ~20 ms. out.initlimits.txt makes new inits and +# several-fold slowdowns visible in review; the 5 ms limit cannot be updated away. + +# The set of packages with an init differs per architecture; CI runs on amd64. +if [ "$(uname -m)" != x86_64 ]; then + echo "SKIP_TEST out.initlimits.txt is recorded for amd64" + exit 0 +fi + +check_init_time.py --max-ms 5 --limits out.initlimits.txt -- $CLI --version diff --git a/acceptance/inittime/test.toml b/acceptance/inittime/test.toml new file mode 100644 index 00000000000..1ce38ad5899 --- /dev/null +++ b/acceptance/inittime/test.toml @@ -0,0 +1,6 @@ +# Linux only: inittrace clocks elsewhere are too coarse or noisy to compare against a +# few-millisecond threshold (on Windows they can be quantized to the ~15.6 ms system tick). +# Absent GOOS keys default to enabled, so the other OSes must be disabled explicitly. +GOOS.linux = true +GOOS.darwin = false +GOOS.windows = false From 96b9183eb3a29ba70f6741dcc2868f6f64460201 Mon Sep 17 00:00:00 2001 From: "pieter.noordhuis" Date: Tue, 10 Nov 2026 07:52:45 +0000 Subject: [PATCH 2/4] Format init time budgets with a fixed two-decimal width Co-authored-by: Isaac --- acceptance/bin/check_init_time.py | 5 +-- acceptance/inittime/out.initlimits.txt | 54 +++++++++++++------------- 2 files changed, 29 insertions(+), 30 deletions(-) diff --git a/acceptance/bin/check_init_time.py b/acceptance/bin/check_init_time.py index 95b2d9d304a..9b9a9466047 100755 --- a/acceptance/bin/check_init_time.py +++ b/acceptance/bin/check_init_time.py @@ -79,9 +79,8 @@ def read_limits(path): def format_limit(ms): - # Aligned on the decimal point so cheap and expensive packages are easy to tell apart. - whole, frac = (f"{ms:.1f}" if ms >= 1 else f"{ms:.2f}").split(".") - return f"{whole:>3}.{frac:<2}" + # Fixed width so cheap and expensive packages are easy to tell apart. + return f"{ms:5.2f}" def new_limit(ms): diff --git a/acceptance/inittime/out.initlimits.txt b/acceptance/inittime/out.initlimits.txt index 35300dd7c7e..90b2076e80d 100644 --- a/acceptance/inittime/out.initlimits.txt +++ b/acceptance/inittime/out.initlimits.txt @@ -9,30 +9,30 @@ # # Independently, the test fails if any package takes over 5 ms; -update cannot raise that. # Generated by acceptance/bin/check_init_time.py. - 2.5 encoding/gob - 0.29 github.com/BurntSushi/toml/internal - 0.72 github.com/Masterminds/semver/v3 - 0.26 github.com/charmbracelet/bubbletea - 0.15 github.com/charmbracelet/x/ansi - 1.2 github.com/databricks/cli/bundle/config - 0.14 github.com/databricks/cli/bundle/direct/dresources - 0.44 github.com/databricks/cli/bundle/internal/validation/generated - 0.17 github.com/databricks/cli/bundle/libraries - 0.26 github.com/databricks/cli/bundle/terraform_dabs_map - 3.1 github.com/databricks/cli/cmd/air - 0.13 github.com/databricks/cli/cmd/root - 0.60 github.com/databricks/cli/libs/dyn/dynvar - 0.31 github.com/databricks/cli/libs/localenv - 1.0 github.com/databricks/cli/libs/structs/structpath - 0.23 github.com/databricks/cli/libs/tags - 0.15 github.com/databricks/cli/libs/telemetry - 0.40 github.com/databricks/databricks-sdk-go/apierr - 0.15 github.com/databricks/databricks-sdk-go/config - 0.16 github.com/databricks/databricks-sdk-go/useragent - 0.12 github.com/google/s2a-go/internal/proto/common_go_proto - 0.14 go.yaml.in/yaml/v3 - 0.21 golang.org/x/crypto/ssh - 0.13 golang.org/x/net/trace - 0.29 google.golang.org/grpc - 0.18 google.golang.org/protobuf/internal/detrand - 0.17 google.golang.org/protobuf/internal/encoding/json + 2.50 encoding/gob + 0.29 github.com/BurntSushi/toml/internal + 0.72 github.com/Masterminds/semver/v3 + 0.26 github.com/charmbracelet/bubbletea + 0.15 github.com/charmbracelet/x/ansi + 1.20 github.com/databricks/cli/bundle/config + 0.14 github.com/databricks/cli/bundle/direct/dresources + 0.44 github.com/databricks/cli/bundle/internal/validation/generated + 0.17 github.com/databricks/cli/bundle/libraries + 0.26 github.com/databricks/cli/bundle/terraform_dabs_map + 3.10 github.com/databricks/cli/cmd/air + 0.13 github.com/databricks/cli/cmd/root + 0.60 github.com/databricks/cli/libs/dyn/dynvar + 0.31 github.com/databricks/cli/libs/localenv + 1.00 github.com/databricks/cli/libs/structs/structpath + 0.23 github.com/databricks/cli/libs/tags + 0.15 github.com/databricks/cli/libs/telemetry + 0.40 github.com/databricks/databricks-sdk-go/apierr + 0.15 github.com/databricks/databricks-sdk-go/config + 0.16 github.com/databricks/databricks-sdk-go/useragent + 0.12 github.com/google/s2a-go/internal/proto/common_go_proto + 0.14 go.yaml.in/yaml/v3 + 0.21 golang.org/x/crypto/ssh + 0.13 golang.org/x/net/trace + 0.29 google.golang.org/grpc + 0.18 google.golang.org/protobuf/internal/detrand + 0.17 google.golang.org/protobuf/internal/encoding/json From d187bdb33acf0de6dd088ad237a26c7a0c2d9900 Mon Sep 17 00:00:00 2001 From: "pieter.noordhuis" Date: Tue, 10 Nov 2026 10:36:59 +0000 Subject: [PATCH 3/4] Log init times on all OSes and fail on total init time on Linux Co-authored-by: Isaac --- acceptance/bin/check_init_time.py | 142 +++++-------------------- acceptance/inittime/out.initlimits.txt | 38 ------- acceptance/inittime/out.test.toml | 3 - acceptance/inittime/output.txt | 2 +- acceptance/inittime/script | 16 +-- acceptance/inittime/test.toml | 6 -- 6 files changed, 30 insertions(+), 177 deletions(-) delete mode 100644 acceptance/inittime/out.initlimits.txt delete mode 100644 acceptance/inittime/test.toml diff --git a/acceptance/bin/check_init_time.py b/acceptance/bin/check_init_time.py index 9b9a9466047..6e2412f11b4 100755 --- a/acceptance/bin/check_init_time.py +++ b/acceptance/bin/check_init_time.py @@ -1,21 +1,14 @@ #!/usr/bin/env python3 """ -Fail if any Go package's init() clock time, per GODEBUG=inittrace=1, exceeds a threshold, -and keep a golden file with an upper bound on each package's init time. +Report each Go package's init() clock time (GODEBUG=inittrace=1) on stderr and, on Linux, +fail if the total init time exceeds --max-ms. -Each package is judged by its minimum over several runs, since load from parallel tests -inflates individual runs but a slow init is slow on every run. Load comes in bursts that -span consecutive runs, so packages over a limit get more runs (up to --max-runs). - -The golden file lists packages with a non-trivial init and their tolerated maximum; any -other package's maximum is MIN_LIMIT_MS. A recorded maximum is kept as long as the package -stays under it, so the file only changes when a package becomes several times slower (or -disappears). In that case the maximum is raised to TOLERANCE times the new measurement and -the diff shows up for review. Delete the file and run with -update to re-record all limits. +The total is judged by its minimum over several runs, since load from parallel tests +inflates individual runs but slow inits are slow on every run. While the total is over the +threshold, more runs are added (up to --max-runs) to ride out bursts of load. """ import argparse -import math import os import re import subprocess @@ -23,41 +16,9 @@ INIT_LINE = re.compile(r"^init (\S+) @\S+ ms, (\S+) ms clock") -# 2x is exceeded by load spikes when the file was recorded on a machine ~1.3x faster than -# CI; 3x was not exceeded in any measurement, and 4x leaves margin for slower runners. -TOLERANCE = 4 - -# Limit for packages not in the golden file. Most inits report under 0.01 ms, where -# relative noise is meaningless and the cost is immaterial. -MIN_LIMIT_MS = 0.1 - - -def header(max_ms): - return f"""\ -# Init time BUDGET per Go package, in ms of init() clock time (GODEBUG=inittrace=1), -# measured as the minimum over several runs of `databricks --version` on linux/amd64. -# -# Each budget is initialized at {TOLERANCE}x the package's measured minimum, for headroom. -# The smallest budget is {MIN_LIMIT_MS} ms; packages not listed here have a budget of {MIN_LIMIT_MS} ms. -# A package that exceeds its budget gets a new budget of {TOLERANCE}x its new minimum: the test -# fails on the diff, which you accept with -update after review. Budgets never shrink; -# delete this file and run with -update to re-measure all of them. -# -# Independently, the test fails if any package takes over {max_ms:g} ms; -update cannot raise that. -# Generated by acceptance/bin/check_init_time.py. -""" - - -def is_recorded(pkg): - # Names and set of FIPS module packages depend on GOFIPS140, which differs between - # `task test` and plain `go test`. - return not pkg.startswith("crypto/internal/fips140") - def measure(command): - # Some inits search PATH (github.com/atotto/clipboard looks up clipboard tools), so their - # time depends on the machine's PATH; a fixed PATH keeps the budgets comparable. - env = dict(os.environ, GODEBUG="inittrace=1", PATH=os.defpath) + env = dict(os.environ, GODEBUG="inittrace=1") cp = subprocess.run(command, env=env, stdout=subprocess.DEVNULL, stderr=subprocess.PIPE, text=True) if cp.returncode != 0: sys.exit(f"command failed with exit code {cp.returncode}:\n{cp.stderr}") @@ -67,89 +28,36 @@ def measure(command): return clocks -def read_limits(path): - limits = {} - if os.path.exists(path): - with open(path) as f: - for line in f: - if line.strip() and not line.startswith("#"): - limit, pkg = line.split() - limits[pkg] = float(limit) - return limits - - -def format_limit(ms): - # Fixed width so cheap and expensive packages are easy to tell apart. - return f"{ms:5.2f}" - - -def new_limit(ms): - # Round up to two significant digits so the limit is never below TOLERANCE * ms. - limit = TOLERANCE * ms - scale = 10 if limit >= 1 else 100 - return math.ceil(limit * scale) / scale - - -def over_limit(runs, limits): - pkgs = filter(is_recorded, runs[0]) - return [p for p in pkgs if min(r[p] for r in runs) > limits.get(p, MIN_LIMIT_MS)] - - -def find_failures(runs, max_ms): - failures = [] - for pkg in runs[0]: - clocks = [r[pkg] for r in runs] - if min(clocks) > max_ms: - failures.append( - f"init of {pkg} took {min(clocks):.2f} ms (min) / {max(clocks):.2f} ms (max) " - f"over {len(runs)} runs; threshold is {max_ms} ms" - ) - return failures - - def main(): parser = argparse.ArgumentParser() parser.add_argument("--runs", type=int, default=5) parser.add_argument("--max-runs", type=int, default=20) parser.add_argument("--max-ms", type=float, required=True) - parser.add_argument("--limits", required=True, help="golden file name, read from $TESTDIR and written to cwd") parser.add_argument("command", nargs="+") args = parser.parse_args() - limits_path = os.path.join(os.environ["TESTDIR"], args.limits) - # When recording from scratch, list every package whose limit exceeds MIN_LIMIT_MS so - # each gets TOLERANCE headroom. Afterwards only exceeded limits are added, so packages - # measuring near MIN_LIMIT_MS / TOLERANCE don't flip in and out of the file. - record_all = not os.path.exists(limits_path) - limits = read_limits(limits_path) - runs = [measure(args.command) for _ in range(args.runs)] - while len(runs) < args.max_runs and (find_failures(runs, args.max_ms) or over_limit(runs, limits)): + while len(runs) < args.max_runs and min(sum(r.values()) for r in runs) > args.max_ms: runs.append(measure(args.command)) - exceeded = set(over_limit(runs, limits)) - with open(args.limits, "w") as f: - f.write(header(args.max_ms)) - for pkg in sorted(filter(is_recorded, runs[0])): - limit = limits.get(pkg) - if pkg in exceeded or (record_all and limit is None): - limit = new_limit(min(r[pkg] for r in runs)) - limit = limit if limit > MIN_LIMIT_MS else None - if limit is not None: - f.write(f"{format_limit(limit)} {pkg}\n") - - failures = find_failures(runs, args.max_ms) - if not failures: - print("OK: package init times are within threshold") - return - - print("FAIL: Go package init() is slow; this adds latency to every CLI invocation.") - for f in failures: - print(f" {f}") - print("To investigate:") - print(" GODEBUG=inittrace=1 databricks --version 2>&1 | sort -k5 -n -r | head") - print(" go mod why -m ") - sys.exit(1) + total = min(sum(r.values()) for r in runs) + clocks = {pkg: min(r[pkg] for r in runs) for pkg in runs[0]} + print(f"Minimum init() clock time in ms over {len(runs)} runs:", file=sys.stderr) + print(f"{total:5.2f} total", file=sys.stderr) + for pkg, ms in sorted(clocks.items(), key=lambda kv: (-kv[1], kv[0])): + print(f"{ms:5.2f} {pkg}", file=sys.stderr) + + # Clocks elsewhere are too coarse or noisy to enforce (on Windows they can be quantized + # to the ~15.6 ms system tick). + if total > args.max_ms and sys.platform == "linux": + print( + f"FAIL: package init() takes {total:.2f} ms in total, over {args.max_ms:g} ms; this delays every CLI invocation." + ) + print("To investigate:") + print(" GODEBUG=inittrace=1 databricks --version 2>&1 | sort -k5 -n -r | head") + print(" go mod why -m ") + sys.exit(1) + print(f"OK: total package init() time is within {args.max_ms:g} ms") if __name__ == "__main__": diff --git a/acceptance/inittime/out.initlimits.txt b/acceptance/inittime/out.initlimits.txt deleted file mode 100644 index 90b2076e80d..00000000000 --- a/acceptance/inittime/out.initlimits.txt +++ /dev/null @@ -1,38 +0,0 @@ -# Init time BUDGET per Go package, in ms of init() clock time (GODEBUG=inittrace=1), -# measured as the minimum over several runs of `databricks --version` on linux/amd64. -# -# Each budget is initialized at 4x the package's measured minimum, for headroom. -# The smallest budget is 0.1 ms; packages not listed here have a budget of 0.1 ms. -# A package that exceeds its budget gets a new budget of 4x its new minimum: the test -# fails on the diff, which you accept with -update after review. Budgets never shrink; -# delete this file and run with -update to re-measure all of them. -# -# Independently, the test fails if any package takes over 5 ms; -update cannot raise that. -# Generated by acceptance/bin/check_init_time.py. - 2.50 encoding/gob - 0.29 github.com/BurntSushi/toml/internal - 0.72 github.com/Masterminds/semver/v3 - 0.26 github.com/charmbracelet/bubbletea - 0.15 github.com/charmbracelet/x/ansi - 1.20 github.com/databricks/cli/bundle/config - 0.14 github.com/databricks/cli/bundle/direct/dresources - 0.44 github.com/databricks/cli/bundle/internal/validation/generated - 0.17 github.com/databricks/cli/bundle/libraries - 0.26 github.com/databricks/cli/bundle/terraform_dabs_map - 3.10 github.com/databricks/cli/cmd/air - 0.13 github.com/databricks/cli/cmd/root - 0.60 github.com/databricks/cli/libs/dyn/dynvar - 0.31 github.com/databricks/cli/libs/localenv - 1.00 github.com/databricks/cli/libs/structs/structpath - 0.23 github.com/databricks/cli/libs/tags - 0.15 github.com/databricks/cli/libs/telemetry - 0.40 github.com/databricks/databricks-sdk-go/apierr - 0.15 github.com/databricks/databricks-sdk-go/config - 0.16 github.com/databricks/databricks-sdk-go/useragent - 0.12 github.com/google/s2a-go/internal/proto/common_go_proto - 0.14 go.yaml.in/yaml/v3 - 0.21 golang.org/x/crypto/ssh - 0.13 golang.org/x/net/trace - 0.29 google.golang.org/grpc - 0.18 google.golang.org/protobuf/internal/detrand - 0.17 google.golang.org/protobuf/internal/encoding/json diff --git a/acceptance/inittime/out.test.toml b/acceptance/inittime/out.test.toml index 44cce516e73..18b1a88417e 100644 --- a/acceptance/inittime/out.test.toml +++ b/acceptance/inittime/out.test.toml @@ -1,4 +1 @@ Cloud = false -GOOS.darwin = false -GOOS.linux = true -GOOS.windows = false diff --git a/acceptance/inittime/output.txt b/acceptance/inittime/output.txt index 3f500562ff4..eb32da7ebc2 100644 --- a/acceptance/inittime/output.txt +++ b/acceptance/inittime/output.txt @@ -1 +1 @@ -OK: package init times are within threshold +OK: total package init() time is within 10 ms diff --git a/acceptance/inittime/script b/acceptance/inittime/script index 3b9deff0a1b..61d7249ca06 100644 --- a/acceptance/inittime/script +++ b/acceptance/inittime/script @@ -1,12 +1,4 @@ -# Package init() runs on every CLI invocation, so a slow one (often arriving through a -# transitive dependency upgrade) delays every command. Today the slowest package takes -# ~1 ms; go-runewidth v0.0.27 took ~20 ms. out.initlimits.txt makes new inits and -# several-fold slowdowns visible in review; the 5 ms limit cannot be updated away. - -# The set of packages with an init differs per architecture; CI runs on amd64. -if [ "$(uname -m)" != x86_64 ]; then - echo "SKIP_TEST out.initlimits.txt is recorded for amd64" - exit 0 -fi - -check_init_time.py --max-ms 5 --limits out.initlimits.txt -- $CLI --version +# Package init() runs on every CLI invocation, so slow inits (often arriving through a +# transitive dependency upgrade) delay every command; go-runewidth v0.0.27 alone took ~20 ms. +# The total is ~5 ms on linux/amd64. +check_init_time.py --max-ms 10 -- $CLI --version 2> LOG.inittime diff --git a/acceptance/inittime/test.toml b/acceptance/inittime/test.toml deleted file mode 100644 index 1ce38ad5899..00000000000 --- a/acceptance/inittime/test.toml +++ /dev/null @@ -1,6 +0,0 @@ -# Linux only: inittrace clocks elsewhere are too coarse or noisy to compare against a -# few-millisecond threshold (on Windows they can be quantized to the ~15.6 ms system tick). -# Absent GOOS keys default to enabled, so the other OSes must be disabled explicitly. -GOOS.linux = true -GOOS.darwin = false -GOOS.windows = false From 990715253ce63957fa7d70eea3c2e5f0b9aa169b Mon Sep 17 00:00:00 2001 From: "pieter.noordhuis" Date: Tue, 10 Nov 2026 10:41:56 +0000 Subject: [PATCH 4/4] Drop historical reference from inittime script Co-authored-by: Isaac --- acceptance/inittime/script | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/acceptance/inittime/script b/acceptance/inittime/script index 61d7249ca06..585dbc6dd80 100644 --- a/acceptance/inittime/script +++ b/acceptance/inittime/script @@ -1,4 +1,3 @@ # Package init() runs on every CLI invocation, so slow inits (often arriving through a -# transitive dependency upgrade) delay every command; go-runewidth v0.0.27 alone took ~20 ms. -# The total is ~5 ms on linux/amd64. +# transitive dependency upgrade) delay every command. The total is ~5 ms on linux/amd64. check_init_time.py --max-ms 10 -- $CLI --version 2> LOG.inittime