| … | … | @@ -36,19 +36,28 @@ DESIGN, and the reasons are measured: |
| 36 | 36 | the wrong model for a queue whose ordering depends on a moving camera. |
| 37 | 37 | The worker now picks the nearest to the CURRENT centre each iteration. |
| 38 | 38 | """ |
| 39 | | import os, queue, threading, time |
| 39 | import os, queue, re, threading, time |
| 40 | 40 | import numpy as np |
| 41 | 41 | |
| 42 | 42 | TILE_M = 30.0 # model native metres per pixel |
| 43 | 43 | |
| 44 | 44 | |
| 45 | 45 | class TileStore: |
| 46 | | def __init__(self, fetch_fn, cache_dir, tile_px=256, log=print): |
| 46 | def __init__(self, fetch_fn, cache_dir, tile_px=256, log=print, |
| 47 | hblk_dir=None): |
| 47 | 48 | self.fetch = fetch_fn # (i1, j1, n) -> (elev, climate) |
| 48 | 49 | self.dir = cache_dir |
| 49 | 50 | self.tile_px = tile_px |
| 50 | 51 | self.log = log |
| 52 | # PUBLISH AS WE GENERATE, because the engine reads a DIFFERENT |
| 53 | # directory in a DIFFERENT format, and a manual conversion step between |
| 54 | # the two is a step that gets forgotten. Browsing then looks like it did |
| 55 | # nothing. The .npy cache stays authoritative -- .hblk is a derived |
| 56 | # view, regenerable from it at any time by stitch_block --per-tile. |
| 57 | self.hblk_dir = hblk_dir |
| 51 | 58 | os.makedirs(self.dir, exist_ok=True) |
| 59 | if self.hblk_dir: |
| 60 | os.makedirs(self.hblk_dir, exist_ok=True) |
| 52 | 61 | self._mem = {} # (tx,ty) -> np.int16 array |
| 53 | 62 | self._state = {} # (tx,ty) -> 'cached'|'queued'|'busy'|'error' |
| 54 | 63 | self._want = {} # (tx,ty) -> True, the pending set |
| … | … | @@ -129,6 +138,45 @@ class TileStore: |
| 129 | 138 | return k |
| 130 | 139 | |
| 131 | 140 | # ---- the single worker ------------------------------------------------- |
| 141 | def hblk_path(self, tx, ty): |
| 142 | return os.path.join(self.hblk_dir, f"tile_{tx}_{ty}.hblk") |
| 143 | |
| 144 | def _publish_hblk(self, tx, ty, arr): |
| 145 | """Write the engine-format twin. Never fatal: the .npy is the cache.""" |
| 146 | if not self.hblk_dir: |
| 147 | return |
| 148 | try: |
| 149 | from stitch_block import write_hblk, tile_origin |
| 150 | ox, oz = tile_origin(tx, ty, self.tile_px, TILE_M, self.APRON) |
| 151 | write_hblk(self.hblk_path(tx, ty), arr, ox, oz, TILE_M) |
| 152 | except Exception as ex: |
| 153 | self.log(f" tile ({tx},{ty}) hblk FAILED: {ex}") |
| 154 | |
| 155 | def backfill_hblk(self): |
| 156 | """Publish every cached .npy that has no .hblk yet. |
| 157 | |
| 158 | The cache predates this path by 841 tiles, and a legacy prefix sits |
| 159 | alongside the current one -- so walk the SAME pattern the converter |
| 160 | does rather than every file in the directory. |
| 161 | """ |
| 162 | if not self.hblk_dir: |
| 163 | return 0 |
| 164 | pat = re.compile(rf"^t2_{self.tile_px}_(-?\d+)_(-?\d+)\.npy$") |
| 165 | n = 0 |
| 166 | for f in os.listdir(self.dir): |
| 167 | m = pat.match(f) |
| 168 | if not m: |
| 169 | continue |
| 170 | tx, ty = int(m.group(1)), int(m.group(2)) |
| 171 | if os.path.exists(self.hblk_path(tx, ty)): |
| 172 | continue |
| 173 | try: |
| 174 | self._publish_hblk(tx, ty, np.load(os.path.join(self.dir, f))) |
| 175 | n += 1 |
| 176 | except Exception as ex: |
| 177 | self.log(f" backfill ({tx},{ty}) FAILED: {ex}") |
| 178 | return n |
| 179 | |
| 132 | 180 | def _run(self): |
| 133 | 181 | while True: |
| 134 | 182 | k = self._take_nearest() |
| … | … | @@ -146,6 +194,7 @@ class TileStore: |
| 146 | 194 | e, _c = self.fetch(tx * n - a_, ty * n - a_, n + 2 * a_) |
| 147 | 195 | a = np.clip(e, -32768, 32767).astype(np.int16) |
| 148 | 196 | np.save(self.path(tx, ty), a) |
| 197 | self._publish_hblk(tx, ty, a) |
| 149 | 198 | self._mem[(tx, ty)] = a |
| 150 | 199 | dt = time.time() - t0 |
| 151 | 200 | self.stats["generated"] += 1 |