OverviewFilesCommitsIssuesDocsReleasesPeers

terrain-lookdev commits / 9822add

lookdev: test one config at a time against a strict gate — the 14x had no merit
test_config.py runs a single config and compares its output to a saved fp32 reference on a FIXED mountain region, reporting correlation and RMSE in metres alongside the timing. Speed without that is not a measurement. (The first check region picked was ocean floor -- flat, so every config would have correlated near 1.0 and passed. A gate that cannot fail is not a gate.) CONTROL FIRST: plain fp32 returns corr 1.000000, rmse 0.00 m. The harness is sound before anything is judged by it. config verdict 512 tile VRAM vs fp32 reference fp32 plain (control) IDENTICAL 11.96 s 3522 MiB corr 1.000000 rmse 0.00 fp16 DIFFERENT 8.84 s 1560 MiB corr 0.925421 rmse 436.66 decoder_tile 256 BROKEN 4.24 s 1698 MiB 82.4% zeros, 53984 NaN torch.compile IDENTICAL 14.94 s 2794 MiB corr 1.000000 rmse 0.02 EVERY PART OF THE EARLIER 14x FAILS IN ISOLATION. decoder_tile_size 256 is broken BY ITSELF -- 82% of the tile is exactly zero. Nothing to do with compile. That was the whole "2x". fp16 is not a free precision win. RMSE 437 m against a field whose sd is 364 m, and the relief compresses 528..2348 -> 257..1857. It is a DIFFERENT world, which is why it did not look right. The pipeline blanket-casts with .to(), and diffusers warns about exactly this at startup -- a warning I read and did not act on. torch.compile is the only lever that is genuinely correct, and it is SLOWER here (14.94 s vs 11.96 s) with a 248 s compile. It does cut VRAM 3522 -> 2794 MiB, which is the one real gain in the whole exercise. The earlier numbers were fast BECAUSE they were wrong. Recorded here so the result is not rediscovered. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
9822add2803d1acf3484b122abfa4e40acd0ae49
by plankatron <noreply@anthropic.com> · 2026-08-31T18:53:47.000Z

Changed files (1) +160 -0

test_config.py +160 -0
……@@ -0,0 +1,160 @@
1#!/usr/bin/env python3
2"""
3ONE CONFIG AT A TIME, against a strict correctness gate.
4
5The first three benchmark runs measured only time. decoder_tile_size=256 came
6out "2x faster" while returning 70% exact zeros, and onestep_latent came out
7"1.9x faster" while visibly degrading the terrain. Speed without a correctness
8gate is not a measurement, it is a way of being confidently wrong.
9
10THE GATE, in order of severity:
11 zeros fraction of pixels exactly 0 structural breakage
12 nans NaN in the climate channels structural breakage
13 corr Pearson r vs the fp32 reference is it the SAME world?
14 rmse metres, vs the fp32 reference how far off, in real units
15 relief min/max, vs the reference did the range collapse?
16
17A config that changes the terrain is not "faster", it is DIFFERENT. Those are
18reported separately so the choice is explicit rather than accidental.
19
20 python test_config.py --save-reference # from the live fp32 service
21 python test_config.py --label "fp16" -- --dtype fp16
22"""
23import argparse, json, os, signal, subprocess, sys, time, urllib.request
24import numpy as np
25
26PORT = 8819
27MODEL = "xandergos/terrain-diffusion-30m"
28VENV = "/storage/projects/code/godot/terrain-diffusion/.venv/bin/python"
29CWD = "/storage/projects/code/godot/terrain-diffusion"
30REF_PATH = "/tmp/td-reference.npz"
31# The check region must EXERCISE what is being tested. The first one picked was
32# ocean floor -- flat, so any config would have correlated near 1.0 and passed.
33# This is the mountain region every visual comparison in the session used.
34CHECK_I, CHECK_J = 0, 2048
35TIME_AT = 61_000_000 # timing uses fresh coordinates, never the check region
36
37
38def fetch(port, i1, j1, n):
39 u = f"http://127.0.0.1:{port}/terrain?i1={i1}&j1={j1}&i2={i1+n}&j2={j1+n}&scale=1"
40 t0 = time.time()
41 with urllib.request.urlopen(u, timeout=3600) as r:
42 h = int(r.headers["X-Height"]); w = int(r.headers["X-Width"]); b = r.read()
43 dt = time.time() - t0
44 e = np.frombuffer(b[:h*w*2], dtype="<i2").astype(np.float64).reshape(h, w)
45 c = (np.frombuffer(b[h*w*2:], dtype="<f4").reshape(h, w, 4)
46 if len(b) > h*w*2 else None)
47 return e, c, dt
48
49
50def gate(e, c, ref):
51 zeros = float((e == 0).mean())
52 nans = int(np.isnan(c).sum()) if c is not None else 0
53 corr = rmse = float("nan")
54 if ref is not None and ref.shape == e.shape:
55 a, b = ref.ravel(), e.ravel()
56 if a.std() > 0 and b.std() > 0:
57 corr = float(np.corrcoef(a, b)[0, 1])
58 rmse = float(np.sqrt(((a - b) ** 2).mean()))
59 return dict(zeros=zeros, nans=nans, corr=corr, rmse=rmse,
60 lo=float(e.min()), hi=float(e.max()), sd=float(e.std()))
61
62
63def verdict(g, ref_g):
64 if g["zeros"] > 0.02 or g["nans"] > 0:
65 return "BROKEN", "structural: zeros or NaN"
66 if g["corr"] == g["corr"]:
67 if g["corr"] > 0.9999 and g["rmse"] < 2.0:
68 return "IDENTICAL", "same terrain"
69 if g["corr"] > 0.99:
70 return "NEAR", f"same shape, rmse {g['rmse']:.1f} m"
71 return "DIFFERENT", f"corr {g['corr']:.4f} — a different world"
72 return "NOREF", "no reference to compare"
73
74
75def main():
76 ap = argparse.ArgumentParser()
77 ap.add_argument("--label", default="config")
78 ap.add_argument("--save-reference", action="store_true")
79 ap.add_argument("--live-port", type=int, default=8817)
80 ap.add_argument("--n", type=int, default=256)
81 ap.add_argument("--time-n", type=int, default=512)
82 ap.add_argument("extra", nargs="*")
83 a = ap.parse_args()
84
85 if a.save_reference:
86 e, c, _ = fetch(a.live_port, CHECK_I, CHECK_J, a.n)
87 np.savez(REF_PATH, e=e)
88 g = gate(e, c, None)
89 print(f"reference saved from the live service on port {a.live_port}")
90 print(f" region ({CHECK_I},{CHECK_J}) {a.n}x{a.n}")
91 print(f" relief {g['lo']:.0f} .. {g['hi']:.0f} m sd {g['sd']:.1f} "
92 f"zeros {100*g['zeros']:.2f}% nan {g['nans']}")
93 return 0
94
95 ref = np.load(REF_PATH)["e"] if os.path.exists(REF_PATH) else None
96 extra = [x for x in a.extra if x != "--"]
97 cmd = [VENV, "-m", "terrain_diffusion.inference.api", MODEL, "--seed", "1234",
98 "--host", "127.0.0.1", "--port", str(PORT), "--log-mode", "info"] + extra
99 log = open(f"/tmp/tc-{a.label.replace(' ', '_')}.log", "w")
100 p = subprocess.Popen(cmd, cwd=CWD, stdout=log, stderr=subprocess.STDOUT,
101 preexec_fn=os.setsid)
102 try:
103 t0 = time.time()
104 while time.time() - t0 < 900:
105 if p.poll() is not None:
106 print(f" {a.label}: SERVER DIED — see /tmp/tc-*.log"); return 1
107 try:
108 urllib.request.urlopen(f"http://127.0.0.1:{PORT}/health", timeout=2).read()
109 break
110 except Exception:
111 time.sleep(2)
112 load = time.time() - t0
113
114 fetch(PORT, TIME_AT - 500_000, TIME_AT - 500_000, 256) # warm
115 ts = [fetch(PORT, TIME_AT + 40_000*k, TIME_AT + 40_000*k, a.time_n)[2]
116 for k in range(3)]
117 e, c, _ = fetch(PORT, CHECK_I, CHECK_J, a.n)
118
119 mem = 0
120 try:
121 out = subprocess.run(["nvidia-smi", "--query-compute-apps=pid,used_memory",
122 "--format=csv,noheader,nounits"],
123 capture_output=True, text=True, timeout=20).stdout
124 for line in out.strip().splitlines():
125 x, y = [t.strip() for t in line.split(",")[:2]]
126 if x.isdigit() and abs(int(x) - p.pid) < 8:
127 mem = int(y)
128 except Exception:
129 pass
130
131 g = gate(e, c, ref)
132 v, why = verdict(g, None)
133 km2 = (a.time_n * 30 / 1000) ** 2
134 avg = sum(ts) / len(ts)
135 print(f"\n {a.label}")
136 print(f" flags {' '.join(extra) or '(none — plain fp32)'}")
137 print(f" load {load:6.1f} s")
138 print(f" {a.time_n}^2 tile {avg:6.2f} s (best {min(ts):.2f}) "
139 f"{km2/avg:6.1f} km2/s")
140 print(f" VRAM {mem:6d} MiB")
141 print(f" zeros {100*g['zeros']:6.2f} % nan {g['nans']}")
142 print(f" relief {g['lo']:.0f} .. {g['hi']:.0f} m sd {g['sd']:.1f}")
143 if ref is not None:
144 print(f" vs fp32 ref corr {g['corr']:.6f} rmse {g['rmse']:.2f} m")
145 print(f" VERDICT {v} ({why})")
146 return 0
147 finally:
148 try: os.killpg(os.getpgid(p.pid), signal.SIGTERM)
149 except Exception: pass
150 p.wait(timeout=60)
151 for _ in range(30):
152 o = subprocess.run(["nvidia-smi", "--query-compute-apps=pid",
153 "--format=csv,noheader"], capture_output=True,
154 text=True).stdout
155 if str(p.pid) not in o: break
156 time.sleep(2)
157
158
159if __name__ == "__main__":
160 sys.exit(main())