#!/usr/bin/env python3
"""
Did the background load cost frames?
Reads the CSV written by perf_metrics.gd (HYPER_TPS_FRAMELOG) plus the .phases
marks written by frametime-load-test.sh, and compares the three phases.
REPORTS THE TAIL, NOT JUST THE MEDIAN. A generator that adds 0.3 ms to a typical
frame and 40 ms to one frame in a hundred is unshippable, and a median hides
that completely — a stutter IS a tail event. p99 and "frames over 2x baseline
median" are the numbers that decide this.
RECOVERY IS THE CONTROL. If loaded is worse than baseline but recovery does not
come back down, what was measured is drift or thermal ramp, not the load, and
the run says so instead of blaming the generator.
python3 scripts/frametime-report.py /tmp/frametime-123456.csv
"""
import sys, os
import numpy as np
def load(csv):
rows = []
with open(csv) as f:
next(f, None)
for line in f:
p = line.strip().split(",")
if len(p) == 2:
try:
rows.append((int(p[0]), float(p[1])))
except ValueError:
pass
return np.array(rows, dtype=np.float64)
def phases(path):
out = []
if not os.path.exists(path):
return out
with open(path) as f:
for line in f:
p = line.split()
if len(p) == 4 and p[0] == "#":
out.append((p[2], int(p[3])))
return out
def main():
if len(sys.argv) < 2:
raise SystemExit(__doc__)
csv = sys.argv[1]
d = load(csv)
if d.size == 0:
raise SystemExit(f"no frames in {csv} — was HYPER_TPS_FRAMELOG set?")
marks = phases(csv + ".phases")
if len(marks) < 2:
raise SystemExit(f"no phase marks in {csv}.phases — run the driver")
# perf_metrics logs Time.get_ticks_msec (since engine start); the driver
# marks wall-clock. Align on the FIRST mark rather than assuming a shared
# epoch, which they do not have.
t = d[:, 0] - d[0, 0]
wall0 = marks[0][1]
segs = []
for i, (name, ms) in enumerate(marks):
start = ms - wall0
end = (marks[i + 1][1] - wall0) if i + 1 < len(marks) else 1 << 60
m = (t >= start) & (t < end)
if m.sum() > 30:
segs.append((name, d[m, 1]))
if not segs:
raise SystemExit("phases did not line up with the frame log")
print(f"{len(d)} frames, {len(segs)} phases\n")
print(f"{'phase':>10} {'frames':>8} {'median':>9} {'p95':>9} {'p99':>9} "
f"{'max':>9}")
stats = {}
for name, v in segs:
s = (float(np.median(v)), float(np.percentile(v, 95)),
float(np.percentile(v, 99)), float(v.max()))
stats[name] = (v, s)
print(f"{name:>10} {len(v):>8} {s[0]:>8.2f}m {s[1]:>8.2f}m "
f"{s[2]:>8.2f}m {s[3]:>8.2f}m")
if "baseline" not in stats:
return 0
base_v, base = stats["baseline"]
thresh = 2.0 * base[0]
print(f"\nstutters (frames over 2x baseline median = {thresh:.1f} ms)")
for name, (v, _) in stats.items():
pct = 100.0 * float((v > thresh).sum()) / len(v)
print(f"{name:>10} {pct:>7.2f} %")
if "loaded" not in stats:
return 0
load_v, ld = stats["loaded"]
print(f"\nloaded vs baseline: median {ld[0] - base[0]:+.2f} ms "
f"p99 {ld[2] - base[2]:+.2f} ms")
# RECOVERY IS THE CONTROL.
if "recovery" in stats:
_, rc = stats["recovery"]
drift = rc[0] - base[0]
if abs(drift) > 0.5 * abs(ld[0] - base[0]) and (ld[0] - base[0]) > 0:
print(f"\n⚠ RECOVERY DID NOT COME BACK (median {drift:+.2f} ms vs "
f"baseline).\n What was measured is drift or thermal ramp, "
f"not the load. Inconclusive.")
return 2
if ld[2] - base[2] > 0.25 * base[2] or ld[0] - base[0] > 0.1 * base[0]:
print("\nVERDICT: the load IS visible. Reduce --jobs, raise --nice, or "
"throttle\n the generator's duty cycle and re-run.")
return 1
print("\nVERDICT: no measurable cost at these settings. Raise --jobs until "
"there is,\n so the ceiling is known rather than assumed.")
return 0
if __name__ == "__main__":
sys.exit(main())