#!/usr/bin/env python3
"""
ONE CONFIG AT A TIME, against a strict correctness gate.
The first three benchmark runs measured only time. decoder_tile_size=256 came
out "2x faster" while returning 70% exact zeros, and onestep_latent came out
"1.9x faster" while visibly degrading the terrain. Speed without a correctness
gate is not a measurement, it is a way of being confidently wrong.
THE GATE, in order of severity:
zeros fraction of pixels exactly 0 structural breakage
nans NaN in the climate channels structural breakage
corr Pearson r vs the fp32 reference is it the SAME world?
rmse metres, vs the fp32 reference how far off, in real units
relief min/max, vs the reference did the range collapse?
A config that changes the terrain is not "faster", it is DIFFERENT. Those are
reported separately so the choice is explicit rather than accidental.
python test_config.py --save-reference # from the live fp32 service
python test_config.py --label "fp16" -- --dtype fp16
"""
import argparse, json, os, signal, subprocess, sys, time, urllib.request
import numpy as np
PORT = 8819
MODEL = "xandergos/terrain-diffusion-30m"
VENV = "/storage/projects/code/godot/terrain-diffusion/.venv/bin/python"
CWD = "/storage/projects/code/godot/terrain-diffusion"
REF_PATH = "/tmp/td-reference.npz"
# The check region must EXERCISE what is being tested. The first one picked was
# ocean floor -- flat, so any config would have correlated near 1.0 and passed.
# This is the mountain region every visual comparison in the session used.
CHECK_I, CHECK_J = 0, 2048
TIME_AT = 61_000_000 # timing uses fresh coordinates, never the check region
def fetch(port, i1, j1, n):
u = f"http://127.0.0.1:{port}/terrain?i1={i1}&j1={j1}&i2={i1+n}&j2={j1+n}&scale=1"
t0 = time.time()
with urllib.request.urlopen(u, timeout=3600) as r:
h = int(r.headers["X-Height"]); w = int(r.headers["X-Width"]); b = r.read()
dt = time.time() - t0
e = np.frombuffer(b[:h*w*2], dtype="<i2").astype(np.float64).reshape(h, w)
c = (np.frombuffer(b[h*w*2:], dtype="<f4").reshape(h, w, 4)
if len(b) > h*w*2 else None)
return e, c, dt
def gate(e, c, ref):
zeros = float((e == 0).mean())
nans = int(np.isnan(c).sum()) if c is not None else 0
corr = rmse = float("nan")
if ref is not None and ref.shape == e.shape:
a, b = ref.ravel(), e.ravel()
if a.std() > 0 and b.std() > 0:
corr = float(np.corrcoef(a, b)[0, 1])
rmse = float(np.sqrt(((a - b) ** 2).mean()))
return dict(zeros=zeros, nans=nans, corr=corr, rmse=rmse,
lo=float(e.min()), hi=float(e.max()), sd=float(e.std()))
def verdict(g, ref_g):
if g["zeros"] > 0.02 or g["nans"] > 0:
return "BROKEN", "structural: zeros or NaN"
if g["corr"] == g["corr"]:
if g["corr"] > 0.9999 and g["rmse"] < 2.0:
return "IDENTICAL", "same terrain"
if g["corr"] > 0.99:
return "NEAR", f"same shape, rmse {g['rmse']:.1f} m"
return "DIFFERENT", f"corr {g['corr']:.4f} — a different world"
return "NOREF", "no reference to compare"
def main():
ap = argparse.ArgumentParser()
ap.add_argument("--label", default="config")
ap.add_argument("--save-reference", action="store_true")
ap.add_argument("--live-port", type=int, default=8817)
ap.add_argument("--n", type=int, default=256)
ap.add_argument("--time-n", type=int, default=512)
ap.add_argument("extra", nargs="*")
a = ap.parse_args()
if a.save_reference:
e, c, _ = fetch(a.live_port, CHECK_I, CHECK_J, a.n)
np.savez(REF_PATH, e=e)
g = gate(e, c, None)
print(f"reference saved from the live service on port {a.live_port}")
print(f" region ({CHECK_I},{CHECK_J}) {a.n}x{a.n}")
print(f" relief {g['lo']:.0f} .. {g['hi']:.0f} m sd {g['sd']:.1f} "
f"zeros {100*g['zeros']:.2f}% nan {g['nans']}")
return 0
ref = np.load(REF_PATH)["e"] if os.path.exists(REF_PATH) else None
extra = [x for x in a.extra if x != "--"]
cmd = [VENV, "-m", "terrain_diffusion.inference.api", MODEL, "--seed", "1234",
"--host", "127.0.0.1", "--port", str(PORT), "--log-mode", "info"] + extra
safe = ''.join(c if c.isalnum() or c in '-_.' else '_' for c in a.label)
log = open(f"/tmp/tc-{safe}.log", "w")
p = subprocess.Popen(cmd, cwd=CWD, stdout=log, stderr=subprocess.STDOUT,
preexec_fn=os.setsid)
try:
t0 = time.time()
while time.time() - t0 < 900:
if p.poll() is not None:
print(f" {a.label}: SERVER DIED — see /tmp/tc-*.log"); return 1
try:
urllib.request.urlopen(f"http://127.0.0.1:{PORT}/health", timeout=2).read()
break
except Exception:
time.sleep(2)
load = time.time() - t0
fetch(PORT, TIME_AT - 500_000, TIME_AT - 500_000, 256) # warm
ts = [fetch(PORT, TIME_AT + 40_000*k, TIME_AT + 40_000*k, a.time_n)[2]
for k in range(3)]
e, c, _ = fetch(PORT, CHECK_I, CHECK_J, a.n)
mem = 0
try:
out = subprocess.run(["nvidia-smi", "--query-compute-apps=pid,used_memory",
"--format=csv,noheader,nounits"],
capture_output=True, text=True, timeout=20).stdout
for line in out.strip().splitlines():
x, y = [t.strip() for t in line.split(",")[:2]]
if x.isdigit() and abs(int(x) - p.pid) < 8:
mem = int(y)
except Exception:
pass
g = gate(e, c, ref)
v, why = verdict(g, None)
km2 = (a.time_n * 30 / 1000) ** 2
avg = sum(ts) / len(ts)
print(f"\n {a.label}")
print(f" flags {' '.join(extra) or '(none — plain fp32)'}")
print(f" load {load:6.1f} s")
print(f" {a.time_n}^2 tile {avg:6.2f} s (best {min(ts):.2f}) "
f"{km2/avg:6.1f} km2/s")
print(f" VRAM {mem:6d} MiB")
print(f" zeros {100*g['zeros']:6.2f} % nan {g['nans']}")
print(f" relief {g['lo']:.0f} .. {g['hi']:.0f} m sd {g['sd']:.1f}")
if ref is not None:
print(f" vs fp32 ref corr {g['corr']:.6f} rmse {g['rmse']:.2f} m")
print(f" VERDICT {v} ({why})")
return 0
finally:
try: os.killpg(os.getpgid(p.pid), signal.SIGTERM)
except Exception: pass
p.wait(timeout=60)
for _ in range(30):
o = subprocess.run(["nvidia-smi", "--query-compute-apps=pid",
"--format=csv,noheader"], capture_output=True,
text=True).stdout
if str(p.pid) not in o: break
time.sleep(2)
if __name__ == "__main__":
sys.exit(main())