From 77d42caaa87fbe85be2e143ed39b68d1852499df Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Domagoj=20=C5=A0ari=C4=87?= Date: Wed, 4 Nov 2026 00:03:23 +0100 Subject: [PATCH] feat: tools to compare two builds fairly, with their spread Adds tools/ab.sh, which runs two or more builds of an entry in turns on one machine: a work folder and an untimed load per arm, ABBA order, SCHED_FIFO verified instead of assumed, and a stop when the machine changes. tools/summarize-ab.mjs prints the median and spread of the legs per test, and says when two arms differ by more than that spread. No change to the runner or the result files. Co-Authored-By: Claude Opus 5.5 --- tools/README.md | 45 +++++++++++ tools/ab.sh | 166 +++++++++++++++++++++++++++++++++++++++++ tools/summarize-ab.mjs | 130 ++++++++++++++++++++++++++++++++ 3 files changed, 341 insertions(+) create mode 100644 tools/README.md create mode 100755 tools/ab.sh create mode 100644 tools/summarize-ab.mjs diff --git a/tools/README.md b/tools/README.md new file mode 100644 index 0000000..0dff11a --- /dev/null +++ b/tools/README.md @@ -0,0 +1,45 @@ +# Tools + +## Comparing two builds + +The runner times one system once. A difference between two runs of two builds can be the builds, or the order they ran in, a neighbour on the machine, a different disk, or a snapshot one build made for the other. `ab.sh` takes the last four out, and `summarize-ab.mjs` says how large the difference is next to the noise between runs of the same build. Linux, bash 4.4 or later. + +```bash +tools/ab.sh --entry rama --dataset ~/data/fpabench-10m --work ~/ab/work --out ~/ab/out \ + --arm old=$PWD,RAMA_PATH=$HOME/rama-old,RAMA_NODE_ADDON_BINARY=$HOME/rama-old/rama.node \ + --arm new=$PWD,RAMA_PATH=$HOME/rama-new,RAMA_NODE_ADDON_BINARY=$HOME/rama-new/rama.node \ + --legs 3 --tests R1,W3 -- --expected ~/run/expected.json +node tools/summarize-ab.mjs ~/ab/out +``` + +An `--arm` is `NAME=`. Each `--env NAME=VALUE` after it is an environment variable for that arm; a value may contain commas. Everything after `--` goes to `runner/run.mjs` as it is. The other options: + +| Option | What it does | +| --- | --- | +| `--entry`, `--dataset`, `--tests`, `--machine` | As in the runner. `--dataset` is a folder or a published size. | +| `--work` | Each arm gets its own folder in it. Downloaded datasets are shared. | +| `--out` | One folder per leg, `-L`, with the runner's files, `run.log`, and `leg.txt`: the host, the load, the scheduling and the exit code. | +| `--legs` | How many times each arm runs. The default is 6, the least is 5. | +| `--timeout` | Seconds a leg may take. A leg over it is a failed leg, never a wait. | +| `--cpus` | Pin the runs to these CPUs, such as `4-23`. At least 2. Without it the runner and any server it starts inherit `SCHED_FIFO` 50 on every CPU, and can starve the rest of the machine: pass it on a machine that has other work. | + +What it does: + +- **Own work folder.** Every arm has its own `--work` folder, and one untimed load of its own first, so no arm times a snapshot another build made. +- **Turns.** The legs run `A B`, `B A`, `A B`, `B A`, ... (the runs go A B B A A B B A), so neither arm is always first or last. With three or more arms each leg is the next row of a rotating Latin square. The order is fully balanced when `--legs` is a multiple of the number of arms, and `ab.sh` warns when it is not. Running again into the same `--out` replaces the legs of the arms. +- **Real-time.** Each leg tries `chrt -f 50`, then the same through `sudo -n`. The check runs the same command as a leg (the prefix, the pin, `env` with a variable and `PATH`, then `node`) and trusts it only when the process finds `node`, sees the variable, and reports class `FF` and a priority above 0 in `ps`. The arm's variables are set after `sudo`, which would otherwise drop them. Otherwise the leg runs normally and is marked `not realtime` in `leg.txt` and in the summary. +- **Same machine.** It records the host name, the boot id and the machine id, and stops when any of them changes between legs. + +The summary prints, per test and arm, the median over the legs and its spread, which is (max - min) / median. Two arms are "beyond spread" only when there are at least 5 legs in each and all three hold: + +1. the ranges (min to max) of their legs do not overlap; +2. the difference of the medians is larger than the range (max - min) of each of the two arms; +3. that difference is at least 1% of the first arm's median. + +Anything else is "within spread", "fewer than 5 legs", or "check failed in a leg". + +**False alarms.** If both arms were the same build, rule 1 alone flags a test with probability 2 / C(2n, n) for n legs each: 10% at 3 legs, 0.8% at 5, 0.2% at 6. Rules 2 and 3 only lower that; simulated with normally distributed noise, the three rules together flag 0.2% of the tests at 5 legs and 0.04% at 6. That is per test: over m tests the chance of at least one false flag is 1 - (1 - p)^m, so for 20 tests at 5 legs about 15% by rule 1 alone and about 3.5% with all three rules. It says nothing about a difference smaller than the spread: that is "within spread", not "no difference". + +It refuses results from machines that differ in their CPU, CPU count, memory, kernel, Node version, dataset or tests version. `--csv` prints the same numbers for a spreadsheet. + +A timed run needs a machine nothing else uses. `ab.sh` does not check that, and a run with other work on the machine shows up as a large spread. diff --git a/tools/ab.sh b/tools/ab.sh new file mode 100755 index 0000000..41d4ff6 --- /dev/null +++ b/tools/ab.sh @@ -0,0 +1,166 @@ +#!/bin/bash +# Runs two or more builds of the same entry against each other, in turns, on this machine. +# +# tools/ab.sh --entry rama --dataset ~/data/fpabench-10m --out ~/ab/out --work ~/ab/work \ +# --arm A=/path/to/checkout --env RAMA_PATH=/path/a --env RAMA_NODE_ADDON_BINARY=/path/a/rama.node \ +# --arm B=/path/to/checkout --env RAMA_PATH=/path/b --env RAMA_NODE_ADDON_BINARY=/path/b/rama.node \ +# [--legs 6] [--tests R1,W3] [--machine ] [--timeout ] [--cpus 4-23] \ +# -- --expected ~/run/expected.json +# +# An arm is `--arm NAME=`, followed by any number of `--env NAME=VALUE` for that arm (a +# value may contain commas). The checkout may be the same for every arm (two builds of one system) or +# differ (two versions of the runner). What follows `--` goes to runner/run.mjs as it is, such as +# --expected and --quick. +# +# Each arm has its own work folder. Each gets one untimed load first, so no arm times a snapshot another +# one made. Then the legs run in turns that keep the order balanced (A B, B A, A B, ...; for 3 or more arms +# a rotating Latin square), so that no arm is always first or last. At least 5 legs per arm. +# Each leg runs under SCHED_FIFO 50 when the machine allows it, and the result says whether it did. Without +# --cpus the runner and any server it starts get FIFO 50 on every CPU and can starve the rest of the +# machine: pass --cpus on a machine that has other work. The run stops when the machine changes between legs. +# Never time on a machine that other work uses. Running again into the same --out replaces the legs of the arms. +# The results are in /-L/, and tools/summarize-ab.mjs prints the table. + +set -euo pipefail + +here=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +entry= dataset= out= work= machine= tests= cpus= legs=6 leg_timeout=7200 +arms=() arm_envs=() +while [ $# -gt 0 ]; do + case $1 in + --entry) entry=$2; shift 2 ;; + --dataset) dataset=$2; shift 2 ;; + --out) out=$2; shift 2 ;; + --work) work=$2; shift 2 ;; + --arm) + case $2 in ?*=?*) ;; *) echo "--arm needs NAME=, not $2" >&2; exit 2 ;; esac + arms+=("$2"); arm_envs+=(""); shift 2 ;; + --env) + [ ${#arms[@]} -gt 0 ] || { echo "--env comes after the --arm it belongs to" >&2; exit 2; } + case $2 in [A-Za-z_]*=*) ;; *) echo "--env needs NAME=VALUE, not $2" >&2; exit 2 ;; esac + arm_envs[${#arms[@]}-1]+="$2"$'\n'; shift 2 ;; + --legs) legs=$2; shift 2 ;; + --tests) tests=$2; shift 2 ;; + --machine) machine=$2; shift 2 ;; + --timeout) leg_timeout=$2; shift 2 ;; + --cpus) cpus=$2; shift 2 ;; + --) shift; break ;; + *) echo "unknown option $1" >&2; exit 2 ;; + esac +done +extra=("$@") +if [ -z "$entry" ] || [ -z "$dataset" ] || [ -z "$out" ] || [ -z "$work" ] || [ ${#arms[@]} -lt 2 ]; then + sed -n '2,10p' "$0" | sed 's/^# \{0,1\}//' >&2 + echo "needs --entry, --dataset, --out, --work and at least two --arm" >&2 + exit 2 +fi +[[ $legs =~ ^[0-9]+$ ]] && [ "$legs" -ge 5 ] || { echo "--legs $legs: at least 5 legs per arm (fewer cannot tell a difference from noise)" >&2; exit 2; } +if [ $((legs % ${#arms[@]})) != 0 ]; then echo "WARNING: $legs legs is not a multiple of ${#arms[@]} arms: the order of the arms is not fully balanced" >&2; fi +case $dataset in 10m|100m|1b|10b) ;; *) dataset=$(cd "$dataset" && pwd) ;; esac +mkdir -p "$out" "$work/datasets" +out=$(cd "$out" && pwd) +work=$(cd "$work" && pwd) + +# --- the machine: the legs must all run on the one --------------------------------------------------------- + +fingerprint() { echo "$(hostname) $(cat /proc/sys/kernel/random/boot_id 2>/dev/null || echo no-boot-id) $(cat /etc/machine-id 2>/dev/null || echo no-machine-id)"; } +host=$(fingerprint) +echo "$host" > "$out/host.txt" +check_host() { [ "$(fingerprint)" = "$host" ] || { echo "the machine changed ($(fingerprint), it was $host): stopping" >&2; exit 3; }; } + +# A single CPU would starve the runner and a server of each other under a real-time class. +if [ -n "$cpus" ]; then + n=0 + IFS=, read -ra parts <<< "$cpus" + for p in "${parts[@]}"; do case $p in *-*) n=$((n + ${p#*-} - ${p%-*} + 1)) ;; *) n=$((n + 1)) ;; esac; done + [ $n -ge 2 ] || { echo "--cpus $cpus is one CPU: a real-time runner needs at least 2" >&2; exit 2; } +fi +pin=() +[ -z "$cpus" ] || pin=(taskset -c "$cpus") + +# --- real-time scheduling, verified instead of assumed ----------------------------------------------------- +# What the shell can start under SCHED_FIFO 50 is tried directly, then through sudo. The probe has the shape +# of a leg: the prefix, the pin, then env with a variable and PATH, then a process that must find node, see +# the variable and report its class. A candidate counts only when that process reports class FF and a +# non-zero real-time priority. + +rt=() +sched="not realtime" +probe() { "$@" "${pin[@]}" env AB_PROBE=seen HOME="$HOME" PATH="$PATH" sh -c 'echo "$AB_PROBE $(node --version >/dev/null 2>&1 && echo node) $(ps -o cls=,rtprio= -p $$)"' 2>/dev/null | tr -s ' ' | sed 's/^ //; s/ $//'; } +for candidate in "chrt -f 50" "sudo -n chrt -f 50 setpriv --reuid=$(id -u) --regid=$(id -g) --clear-groups --"; do + read -ra c <<< "$candidate" + got=$(probe "${c[@]}" || true) + if [[ $got =~ ^seen\ node\ (FF\ [1-9][0-9]*)$ ]]; then rt=("${c[@]}"); sched=${BASH_REMATCH[1]}; break; fi +done +[ ${#rt[@]} -gt 0 ] || echo "WARNING: SCHED_FIFO is not available (no rtprio limit, no passwordless sudo, or node not found under it): the legs run \"not realtime\" and say so" >&2 +{ [ ${#rt[@]} -eq 0 ] || [ -n "$cpus" ]; } || echo "WARNING: without --cpus the runner and any server it starts run FIFO 50 on all CPUs" >&2 +echo "scheduling: $sched" + +# --- one leg ------------------------------------------------------------------------------------------------ + +arm_name() { echo "${1%%=*}"; } +arm_dir() { realpath "${1#*=}"; } + +# run_leg ; the runner gets what followed `--` +run_leg() { + local idx=$1 name=$2 only=$3 realtime=$4 + local spec=${arms[$1]} dir envs=() leg=$out/$2 prefix=() tflag=() + dir=$(arm_dir "$spec") + while IFS= read -r kv; do [ -z "$kv" ] || envs+=("$kv"); done <<< "${arm_envs[$idx]}" + [ "$realtime" = 1 ] && [ ${#rt[@]} -gt 0 ] && prefix=("${rt[@]}") + [ -z "$only" ] || tflag=(--tests "$only") + rm -rf "${leg:?}" + mkdir -p "$leg" + { + echo "host=$(fingerprint)" + echo "load=$(cut -d' ' -f1-3 /proc/loadavg 2>/dev/null || echo unknown)" + echo "sched=$([ ${#prefix[@]} -gt 0 ] && echo "$sched" || echo "not realtime")" + echo "start=$(date -u +%FT%TZ)" + echo "run=$((++runs))" + } > "$leg/leg.txt" + local rc=0 + # env comes AFTER the real-time prefix: sudo resets the environment, so assignments before it never + # arrive. PATH and HOME are passed on for the same reason; an arm's own PATH or HOME wins. + (cd "$dir" && timeout "$leg_timeout" "${prefix[@]}" "${pin[@]}" env HOME="$HOME" PATH="$PATH" "${envs[@]}" node runner/run.mjs --entry "$entry" --dataset "$dataset" \ + --work "$work/$(arm_name "$spec")" --out "$leg" ${machine:+--machine "$machine"} "${tflag[@]}" "${extra[@]}") > "$leg/run.log" 2>&1 || rc=$? + { echo "end=$(date -u +%FT%TZ)"; echo "rc=$rc"; } >> "$leg/leg.txt" + return $rc +} + +# The arms, in order, for leg i (0-based). Two arms: A B, B A, A B, ... so the runs go A B B A A B B A. +# Three or more: row i of a Latin square, the cyclic one over the sequence 0 1 k-1 2 k-2 ... shifted by i, +# which for an even k also balances which arm follows which when legs is a multiple of k. +leg_order() { + local i=$1 k=${#arms[@]} j seq=(0) lo=1 hi=$((${#arms[@]} - 1)) + while [ $lo -le $hi ]; do + seq+=($lo); lo=$((lo + 1)) + if [ $lo -le $hi ]; then seq+=($hi); hi=$((hi - 1)); fi + done + for j in "${seq[@]}"; do echo $(((j + i) % k)); done +} + +failed=0 runs=0 +for idx in "${!arms[@]}"; do + name=$(arm_name "${arms[$idx]}") + rm -rf "${out:?}/$name-prep" "${out:?}/$name"-L[0-9]* + mkdir -p "$work/$name" + # A real folder where the link goes (an earlier run, or a runner that made one) is removed when empty, + # and never linked into. + if [ -e "$work/$name/datasets" ] && [ ! -L "$work/$name/datasets" ]; then + rmdir "$work/$name/datasets" || { echo "$work/$name/datasets is a folder with files in it: move it away" >&2; exit 2; } + fi + ln -sfn "$work/datasets" "$work/$name/datasets" + echo "prepare $name: one untimed load" + check_host + run_leg "$idx" "$name-prep" L1 0 || { echo "the load of $name failed, see $out/$name-prep/run.log" >&2; exit 1; } +done +for ((i = 0; i < legs; i++)); do + for idx in $(leg_order $i); do + name=$(arm_name "${arms[$idx]}") + check_host + echo "leg $((i + 1)): $name" + run_leg "$idx" "$name-L$((i + 1))" "$tests" 1 || { echo "leg $((i + 1)) of $name failed (rc $?), see $out/$name-L$((i + 1))/run.log" >&2; failed=1; } + done +done +node "$here/summarize-ab.mjs" "$out" || true +exit $failed diff --git a/tools/summarize-ab.mjs b/tools/summarize-ab.mjs new file mode 100644 index 0000000..73acc03 --- /dev/null +++ b/tools/summarize-ab.mjs @@ -0,0 +1,130 @@ +// Reads the results of tools/ab.sh and prints, per test, each arm's median over its legs, the spread of +// the legs, and whether the arms differ by more than that spread. +// +// node tools/summarize-ab.mjs [--csv] +// +// A test's value in one leg is the `time` the runner wrote for it, itself a median of the leg's runs. An +// arm's value is the median of its legs, and its spread is (max - min) / median over the legs. +// Two arms differ "beyond spread" only when, with at least 5 legs in each, all of these hold: +// 1. the ranges [min, max] of their legs do not overlap, +// 2. |median difference| is larger than the range (max - min) of each of the two arms, and +// 3. |median difference| is at least 1% of the first arm's median. +// If the arms were the same build, rule 1 alone fires with probability 2 / C(2n, n) per test: 0.8% at 5 +// legs each, 0.2% at 6; rules 2 and 3 only lower it (simulated with normal noise: 0.2% at 5 legs, +// 0.04% at 6). Over m tests the chance of at least one false flag is +// 1 - (1 - p)^m. A test that is not `ok` in some leg has no value, and no verdict. +// It refuses to compare results from different machines. + +import fs from 'node:fs'; +import path from 'node:path'; + +const args = process.argv.slice(2); +const dir = args.find((a) => !a.startsWith('--')); +const MIN_LEGS = 5; +const MIN_EFFECT_PCT = 1; +const csv = args.includes('--csv'); +if (!dir) { + console.error('usage: node tools/summarize-ab.mjs [--csv]'); + process.exit(2); +} + +const median = (xs) => { + const s = [...xs].sort((a, b) => a - b); + return s.length % 2 ? s[(s.length - 1) / 2] : (s[s.length / 2 - 1] + s[s.length / 2]) / 2; +}; +/** The key=value lines of a leg's leg.txt. */ +const keyValues = (file) => (fs.existsSync(file) ? Object.fromEntries(fs.readFileSync(file, 'utf8').split('\n').filter((l) => l.includes('=')).map((l) => [l.slice(0, l.indexOf('=')), l.slice(l.indexOf('=') + 1)])) : {}); + +// --- the legs: -L, in the folder ----------------------------------------------------------------------- + +const legs = []; +for (const name of fs.readdirSync(dir)) { + const m = /^(.+)-L(\d+)$/.exec(name); + if (!m || !fs.statSync(path.join(dir, name)).isDirectory()) continue; + const results = fs.readdirSync(path.join(dir, name)).filter((f) => f.endsWith('.json') && !f.endsWith('.answers.json')); + const info = keyValues(path.join(dir, name, 'leg.txt')); + if (results.length !== 1) { + console.error(`${name}: ${results.length ? 'more than one result file' : 'no result file'}, rc=${info.rc ?? '?'}: the leg is left out`); + continue; + } + legs.push({ arm: m[1], n: Number(m[2]), info, result: JSON.parse(fs.readFileSync(path.join(dir, name, results[0]), 'utf8')) }); +} +if (!legs.length) { + console.error(`${dir}: no -L folder with a result`); + process.exit(1); +} +legs.sort((a, b) => a.n - b.n); +const arms = [...new Set(legs.map((l) => l.arm))].sort((a, b) => legs.findIndex((l) => l.arm === a) - legs.findIndex((l) => l.arm === b)); + +// --- one machine only ------------------------------------------------------------------------------------------ + +const machineOf = (r) => ({ entry: r.entry, scale: r.scale, machine: r.machine, platform: r.host?.platform, arch: r.host?.arch, cpus: r.host?.cpus, cpu: r.host?.cpu, memoryGB: r.host?.memoryGB, node: r.host?.node, testsVersion: r.testsVersion, dataset: r.dataset?.sha256 }); +const first = machineOf(legs[0].result); +for (const l of legs) { + const differs = Object.entries(machineOf(l.result)).filter(([k, v]) => v !== first[k]); + if (differs.length) { + console.error(`not comparable: ${l.arm}-L${l.n} differs from ${legs[0].arm}-L${legs[0].n} in ${differs.map(([k, v]) => `${k} (${first[k]} against ${v})`).join(', ')}`); + process.exit(1); + } +} +const fingerprints = new Set(legs.map((l) => l.info.host).filter(Boolean)); +if (fingerprints.size > 1) { + console.error(`not comparable: the legs ran on ${fingerprints.size} different hosts or boots`); + process.exit(1); +} + +// --- the numbers ----------------------------------------------------------------------------------------------- + +const ids = [...new Set(legs.flatMap((l) => Object.keys(l.result.tests)))]; +const stats = (id, arm) => { + const of = legs.filter((l) => l.arm === arm); + const ok = of.filter((l) => l.result.tests[id]?.check === 'ok' && l.result.tests[id].time !== undefined); + const xs = ok.map((l) => l.result.tests[id].time); + if (!xs.length) return { legs: of.length, ok: 0 }; + const med = median(xs); + return { legs: of.length, ok: ok.length, median: med, min: Math.min(...xs), max: Math.max(...xs), spread: med ? ((Math.max(...xs) - Math.min(...xs)) / med) * 100 : 0 }; +}; +const verdict = (a, b) => { + if (!a.ok || !b.ok) return 'no value'; + if (a.ok < a.legs || b.ok < b.legs) return 'check failed in a leg'; + if (a.legs < MIN_LEGS || b.legs < MIN_LEGS) return `fewer than ${MIN_LEGS} legs`; + const apart = a.max < b.min || b.max < a.min; + const delta = Math.abs(b.median - a.median); + return apart && delta > a.max - a.min && delta > b.max - b.min && delta >= (a.median * MIN_EFFECT_PCT) / 100 ? 'beyond spread' : 'within spread'; +}; +const rows = []; +for (const id of ids) { + if (legs.every((l) => l.result.tests[id]?.time === undefined)) continue; + const s = Object.fromEntries(arms.map((a) => [a, stats(id, a)])); + const rest = arms.slice(1).map((a) => ({ arm: a, delta: s[a].ok && s[arms[0]].ok ? ((s[a].median - s[arms[0]].median) / s[arms[0]].median) * 100 : null, verdict: verdict(s[arms[0]], s[a]) })); + rows.push({ id, s, rest }); +} + +// --- output ---------------------------------------------------------------------------------------------------- + +if (csv) { + console.log('test,arm,legs,ok,median_s,min_s,max_s,spread_pct,delta_vs_first_pct,verdict'); + for (const r of rows) + for (const a of arms) { + const v = r.s[a]; + const d = r.rest.find((x) => x.arm === a); + console.log([r.id, a, v.legs, v.ok, v.median ?? '', v.min ?? '', v.max ?? '', v.spread?.toFixed(1) ?? '', d?.delta?.toFixed(1) ?? '', d?.verdict ?? ''].join(',')); + } + process.exit(0); +} + +const g = legs[0].result; +const rt = (arm) => legs.filter((l) => l.arm === arm).map((l) => l.info.sched ?? '?'); +console.log(`${g.entry} ${g.scale}, machine ${g.machine}: ${g.host.cpu}, ${g.host.cpus} CPUs, ${g.host.memoryGB} GB, ${g.host.platform}`); +console.log(`arms: ${arms.map((a) => `${a} (${legs.filter((l) => l.arm === a).length} legs)`).join(', ')}; order of the runs ${[...legs].sort((a, b) => Number(a.info.run ?? 0) - Number(b.info.run ?? 0) || (a.info.start ?? '').localeCompare(b.info.start ?? '')).map((l) => l.arm).join(' ')}`); +for (const a of arms) { + const r = rt(a); + const bad = r.filter((x) => !/^FF \d+$/.test(x)).length; + console.log(`${a}: scheduling ${bad ? `NOT realtime in ${bad} of ${r.length} legs (${[...new Set(r)].join('; ')})` : `${r[0]} in all ${r.length} legs`}; exit codes ${legs.filter((l) => l.arm === a).map((l) => l.info.rc ?? '?').join(' ')}`); +} +console.log(''); +const w = Math.max(4, ...ids.map((i) => i.length)); +const cell = (v) => (v.ok ? `${v.median.toPrecision(3)} s ±${v.spread.toFixed(0)}%` : '-'); +console.log(['test'.padEnd(w), ...arms.map((a) => a.padStart(16)), ...arms.slice(1).map((a) => `${a} vs ${arms[0]}`.padStart(22))].join(' ')); +for (const r of rows) + console.log([r.id.padEnd(w), ...arms.map((a) => cell(r.s[a]).padStart(16)), ...r.rest.map((x) => `${x.delta === null ? '-' : `${x.delta >= 0 ? '+' : ''}${x.delta.toFixed(1)}%`} ${x.verdict}`.padStart(22))].join(' '));