#!/usr/bin/env python3
"""Generate diffusion tiles STEADILY, without a browser driving the view.
The lookdev's streamer is view-driven on purpose: /explore asks for what is on
screen and the server caps the pending set at 16 so a moving view cannot grow a
queue it will never want. That is right for looking at terrain and useless for
filling a world -- covering 119x119 by panning means panning for seven hours.
This walks a REGION instead, nearest-first from a centre, and hands the tiles to
the SAME TileStore the server uses. So it shares the worker, the cache layout,
the backoff, and the .hblk publishing; the only thing it replaces is who decides
what to ask for. Nothing here knows the tile format.
python bake_tiles.py # the whole 119x119 world window
python bake_tiles.py --radius 20 # a 41x41 disc around the centre
python bake_tiles.py --centre 10,-5 --radius 8
python bake_tiles.py --limit 200 # stop after N new tiles
python bake_tiles.py --dry-run # say what it would do, generate nothing
Interrupting is safe. Every tile is written before the next is started, and a
re-run skips whatever is already cached -- so ^C then re-run resumes.
"""
import argparse, math, os, signal, sys, time
HERE = os.path.dirname(os.path.abspath(__file__))
sys.path.insert(0, HERE)
import server # config + fetch, no Flask app is started
from tiles import TileStore
# The world window the ENGINE can address, from the same expression the shader
# emitter uses: erosion_heightmap_world(28*2^15*1.0, 256*30) -> n = 119,
# tx0 = ty0 = -(n/2) = -59. Generating outside it is not wasted exactly -- the
# cache is the cache -- but the tile CANNOT be loaded by the map, and 130 such
# tiles already sit in the cache being refused. Default to the addressable set.
OCTREE_SPAN_M = 28 * (2 ** 15) * 1.0
TILE_SPAN_M = 256 * 30.0
WORLD_N = int(OCTREE_SPAN_M / TILE_SPAN_M)
WORLD_LO = -(WORLD_N // 2)
WORLD_HI = WORLD_LO + WORLD_N - 1
def preflight(td_url):
"""Fail HERE if the backend is not up.
tiles.py backs off on failure precisely because a downed service once
burned a whole queue into 962 errors in seconds. Better to not start.
Only a CONNECTION failure counts as down. A tiny probe tile is not a
representative request and the service is entitled to reject it -- treating
that as "down" would refuse to bake against a perfectly healthy backend.
"""
import urllib.error
try:
server.fetch_diffusion(td_url, 0, 0, 8)
return True
except urllib.error.HTTPError:
return True # it answered, and that is all this asks
except (urllib.error.URLError, ConnectionError, OSError) as ex:
print(f"diffusion service not answering at {td_url}\n {ex}\n\n"
" cd /storage/projects/code/godot/terrain-diffusion\n"
" .venv/bin/python -m terrain_diffusion.inference.api \\\n"
" xandergos/terrain-diffusion-30m --seed 1234 --no-compile \\\n"
" --host 127.0.0.1 --port 8817", file=sys.stderr)
return False
def targets(centre, radius, lo, hi):
"""Region tiles, nearest-first. Ties broken deterministically so an
interrupted bake and its resume walk the same order."""
cx, cy = centre
out = []
for tx in range(lo, hi + 1):
for ty in range(lo, hi + 1):
d2 = (tx - cx) ** 2 + (ty - cy) ** 2
if radius is not None and d2 > radius * radius:
continue
out.append((d2, tx, ty))
out.sort()
return [(tx, ty) for _d, tx, ty in out]
def human(sec):
if sec < 90:
return f"{sec:.0f}s"
if sec < 5400:
return f"{sec/60:.1f}m"
return f"{sec/3600:.1f}h"
def main():
ap = argparse.ArgumentParser()
ap.add_argument("--centre", default="0,0", metavar="TX,TY")
ap.add_argument("--radius", type=int, default=None,
help="tiles from centre; default = the whole world window")
ap.add_argument("--limit", type=int, default=0, help="stop after N new tiles")
ap.add_argument("--all", action="store_true",
help="ignore the 119x119 addressable window")
ap.add_argument("--td-url", default="http://127.0.0.1:8817")
ap.add_argument("--tile-px", type=int, default=256)
ap.add_argument("--depth", type=int, default=24,
help="pending tiles kept queued; the worker is serial either way")
ap.add_argument("--dry-run", action="store_true")
a = ap.parse_args()
cx, cy = [int(v) for v in a.centre.split(",")]
lo, hi = (-10**6, 10**6) if a.all else (WORLD_LO, WORLD_HI)
want = targets((cx, cy), a.radius, lo, hi)
print(f"world window : {WORLD_N}x{WORLD_N} tiles, tx/ty {WORLD_LO}..{WORLD_HI}"
+ (" (IGNORED, --all)" if a.all else ""))
print(f"region : centre ({cx},{cy})"
+ (f", radius {a.radius}" if a.radius is not None else ", whole window")
+ f" -> {len(want)} tiles")
print(f"npy cache : {server.TILE_CACHE}")
print(f"hblk out : {server.HBLK_CACHE}")
if a.dry_run:
have = sum(1 for t in want
if os.path.exists(os.path.join(
server.TILE_CACHE, f"t2_{a.tile_px}_{t[0]}_{t[1]}.npy")))
todo = len(want) - have
print(f"\ncached {have}, would generate {todo}"
+ (f" (capped to {a.limit})" if a.limit and todo > a.limit else ""))
print(f"at the measured 1.72 s/tile median that is "
f"~{human(min(todo, a.limit or todo) * 1.72)}")
return 0
if not preflight(a.td_url):
return 1
def _fetch(i1, j1, n):
return server.fetch_diffusion(a.td_url, i1, j1, n)[:2]
ts = TileStore(_fetch, server.TILE_CACHE, tile_px=a.tile_px,
log=lambda *_: None, hblk_dir=server.HBLK_CACHE)
n = ts.backfill_hblk()
if n:
print(f"backfilled {n} .hblk from the existing cache")
todo = [t for t in want if ts.state(*t) not in ("cached", "disk")]
if a.limit:
todo = todo[:a.limit]
print(f"\n{len(want) - len(todo)} already cached, generating {len(todo)}\n")
if not todo:
return 0
stop = {"now": False}
signal.signal(signal.SIGINT, lambda *_: stop.__setitem__("now", True))
t0 = time.time()
done0 = ts.stats["generated"]
err0 = ts.stats["errors"]
i = 0
last = ""
while (i < len(todo) or ts.queue_depth()) and not stop["now"]:
room = a.depth - ts.queue_depth()
if room > 0 and i < len(todo):
ts.request(todo[i:i + room], (cx, cy))
i += room
time.sleep(0.4)
done = ts.stats["generated"] - done0
errs = ts.stats["errors"] - err0
el = time.time() - t0
rate = done / el if el > 0 else 0
eta = (len(todo) - done) / rate if rate > 0 else 0
last = (f"\r {done}/{len(todo)} {100*done/len(todo):5.1f}% "
f"{rate*3600:6.0f} tiles/h elapsed {human(el)} "
f"eta {human(eta) if rate > 0 else '--'}"
+ (f" errors {errs}" if errs else "") + " ")
sys.stdout.write(last); sys.stdout.flush()
# A backend that went away mid-bake: tiles.py backs off, so this is a
# slow bleed rather than a burst. Stop rather than grind for hours.
if errs >= 20 and done == 0:
print("\n\naborting: 20 failures and no successes — is the service up?")
return 1
el = time.time() - t0
done = ts.stats["generated"] - done0
print(f"\n\n{'interrupted' if stop['now'] else 'done'}: "
f"{done} tiles in {human(el)}"
+ (f" ({done/el*3600:.0f} tiles/h)" if el > 0 and done else ""))
if stop["now"]:
print("re-run the same command to resume; cached tiles are skipped.")
return 0
if __name__ == "__main__":
sys.exit(main())