| … | … | @@ -73,6 +73,12 @@ function hashFile (filePath) { |
| 73 | 73 | return hashChunk(fs.readFileSync(filePath)) |
| 74 | 74 | } |
| 75 | 75 | |
| 76 | // Bumped when the persisted index gains a field the fast manifest-diff boot |
| 77 | // path relies on. On a version mismatch (or absence) the boot does ONE full |
| 78 | // _scan() to repopulate + re-stamp, then fast-paths thereafter. v1 added |
| 79 | // `linkname` on file entries (so symlinks are reconstructable without lstat). |
| 80 | const SCAN_SCHEMA_VERSION = 1 |
| 81 | |
| 76 | 82 | // ── DeployServer ───────────────────────────────────────────── |
| 77 | 83 | |
| 78 | 84 | export class DeployServer extends ReadyResource { |
| … | … | @@ -100,6 +106,16 @@ export class DeployServer extends ReadyResource { |
| 100 | 106 | this.manifest = null |
| 101 | 107 | this.stats = { totalConnections: 0, filesServed: 0, bytesServed: 0 } |
| 102 | 108 | |
| 109 | // Symlink-target blob dir. Durability follows the corestore: a |
| 110 | // caller-supplied (persistent) store — e.g. the sidecar — keeps blobs in a |
| 111 | // durable sibling dir so the fast manifest-diff boot path can serve symlink |
| 112 | // content across restarts WITHOUT re-walking to re-materialize them. An OWN |
| 113 | // (ephemeral tmp) store gets no persistent fast-path benefit, so it uses a |
| 114 | // tmp dir that _close() cleans up (no litter). content-addressed either way. |
| 115 | this._symlinkBlobsDir = opts.symlinkBlobsDir || (this._ownStore |
| 116 | ? path.join(os.tmpdir(), 'hyper-deploy-symlinks-' + crypto.randomBytes(8).toString('hex')) |
| 117 | : path.join(path.dirname(this.dir), '.hyper-deploy-symlink-blobs')) |
| 118 | |
| 103 | 119 | this.ready().catch(safetyCatch) |
| 104 | 120 | } |
| 105 | 121 | |
| … | … | @@ -135,8 +151,12 @@ export class DeployServer extends ReadyResource { |
| 135 | 151 | ) |
| 136 | 152 | } |
| 137 | 153 | |
| 138 | | // Index directory |
| 139 | | await this._scan() |
| 154 | // Index directory. Prefer the fast manifest-diff boot path (reconstruct the |
| 155 | // manifest from the persistent fileIndex + walk only NEW hash-dirs); fall |
| 156 | // back to a full rescan whenever the persisted schema is absent/older or |
| 157 | // anything is inconsistent — correctness over speed. |
| 158 | fs.mkdirSync(this._symlinkBlobsDir, { recursive: true }) |
| 159 | await this._scanBoot() |
| 140 | 160 | |
| 141 | 161 | // Start swarm |
| 142 | 162 | if (!this.swarm) this.swarm = new Hyperswarm() |
| … | … | @@ -164,9 +184,13 @@ export class DeployServer extends ReadyResource { |
| 164 | 184 | } |
| 165 | 185 | if (this._fileIndex) await this._fileIndex.close() |
| 166 | 186 | if (this._ownStore && this._corestore) await this._corestore.close() |
| 167 | | if (this._symlinkBlobsDir) { |
| 187 | // Ephemeral (own tmp corestore → no persistent fast-path benefit): clean up |
| 188 | // the tmp symlink-blob dir on close (no litter). Durable (caller-supplied |
| 189 | // corestore, e.g. the sidecar): KEEP the blobs so the fast manifest-diff |
| 190 | // boot path can serve symlink content across restarts — they are GC'd |
| 191 | // alongside the tree store, not on process exit. |
| 192 | if (this._ownStore && this._symlinkBlobsDir) { |
| 168 | 193 | try { fs.rmSync(this._symlinkBlobsDir, { recursive: true, force: true }) } catch {} |
| 169 | | this._symlinkBlobsDir = null |
| 170 | 194 | } |
| 171 | 195 | } |
| 172 | 196 | |
| … | … | @@ -223,10 +247,8 @@ export class DeployServer extends ReadyResource { |
| 223 | 247 | * @returns {{ addedFiles: number, addedBytes: number }} |
| 224 | 248 | */ |
| 225 | 249 | async _scanIncremental (subPaths) { |
| 226 | | if (!this._symlinkBlobsDir) { |
| 227 | | this._symlinkBlobsDir = path.join(os.tmpdir(), 'hyper-deploy-symlinks-' + crypto.randomBytes(8).toString('hex')) |
| 228 | | fs.mkdirSync(this._symlinkBlobsDir, { recursive: true }) |
| 229 | | } |
| 250 | // Durable symlink-blob dir (set in the constructor, ensured in _open). |
| 251 | fs.mkdirSync(this._symlinkBlobsDir, { recursive: true }) |
| 230 | 252 | if (!this.manifest) { |
| 231 | 253 | // First-ever scan or after a full reset — fall back to legacy full scan, |
| 232 | 254 | // since incremental can't synthesize the manifest skeleton from nothing. |
| … | … | @@ -269,7 +291,8 @@ export class DeployServer extends ReadyResource { |
| 269 | 291 | await this._fileIndex.putFile(f.path, { |
| 270 | 292 | contentHash, |
| 271 | 293 | size: f.size, |
| 272 | | mtime |
| 294 | mtime, |
| 295 | linkname: f.isSymlink ? f.linkTarget : null |
| 273 | 296 | }) |
| 274 | 297 | |
| 275 | 298 | if (f.size >= CHUNK_THRESHOLD) { |
| … | … | @@ -332,19 +355,129 @@ export class DeployServer extends ReadyResource { |
| 332 | 355 | return { addedFiles, addedBytes } |
| 333 | 356 | } |
| 334 | 357 | |
| 358 | /** |
| 359 | * Boot indexer. Fast manifest-diff path when the persisted index carries the |
| 360 | * current scan schema; otherwise a full _scan() that repopulates + re-stamps |
| 361 | * the schema. ANY failure in the fast path falls back to a full _scan(). |
| 362 | */ |
| 363 | async _scanBoot () { |
| 364 | let schema = null |
| 365 | try { schema = await this._fileIndex.getIndexMeta('scanSchema') } catch {} |
| 366 | if (schema && schema.version === SCAN_SCHEMA_VERSION) { |
| 367 | try { |
| 368 | await this._scanFromManifest() |
| 369 | return |
| 370 | } catch (err) { |
| 371 | this.emit('manifest-scan-fallback', err) |
| 372 | // fall through to a full, always-correct rescan |
| 373 | } |
| 374 | } |
| 375 | await this._scan() |
| 376 | try { await this._fileIndex.putIndexMeta('scanSchema', { version: SCAN_SCHEMA_VERSION }) } catch {} |
| 377 | } |
| 378 | |
| 379 | /** |
| 380 | * Fast boot: reconstruct the manifest from the persistent fileIndex (ONE |
| 381 | * sequential hyperbee scan) and reconcile only the hash-dir delta on disk. |
| 382 | * The served `<dir>/<hash>/` dirs are content-addressed + immutable, so an |
| 383 | * unchanged dir is never re-walked/re-stat'd. Throws on any inconsistency so |
| 384 | * _scanBoot can fall back to a full scan (correctness over speed). |
| 385 | */ |
| 386 | async _scanFromManifest () { |
| 387 | const persisted = await this._fileIndex.listFiles('') |
| 388 | if (persisted.length === 0) throw new Error('empty index — full scan required') |
| 389 | |
| 390 | // Current top-level hash-dirs on disk (one readdir, NOT a recursive walk). |
| 391 | const currentDirs = new Set() |
| 392 | for (const name of fs.readdirSync(this.dir)) { |
| 393 | let st |
| 394 | try { st = fs.statSync(path.join(this.dir, name)) } catch { continue } |
| 395 | if (st.isDirectory()) currentDirs.add(name) |
| 396 | } |
| 397 | |
| 398 | // Group persisted entries by first path segment (= hash-dir name). |
| 399 | const filesByDir = new Map() |
| 400 | for (const e of persisted) { |
| 401 | const slash = e.path.indexOf('/') |
| 402 | const seg = slash === -1 ? e.path : e.path.slice(0, slash) |
| 403 | if (!filesByDir.has(seg)) filesByDir.set(seg, []) |
| 404 | filesByDir.get(seg).push(e) |
| 405 | } |
| 406 | |
| 407 | // Prune GC'd dirs: indexed dirs no longer on disk (bidirectional diff). |
| 408 | const goneDirs = [] |
| 409 | for (const dir of filesByDir.keys()) if (!currentDirs.has(dir)) goneDirs.push(dir) |
| 410 | let pruned = 0 |
| 411 | for (const dir of goneDirs) { |
| 412 | for (const e of filesByDir.get(dir)) { await this._fileIndex.delFile(e.path); pruned++ } |
| 413 | filesByDir.delete(dir) |
| 414 | } |
| 415 | |
| 416 | // Reconstruct the manifest from surviving (unchanged) dirs — no walk/stat. |
| 417 | // Regular files resolve their diskPath by pure path join (zero fs calls); |
| 418 | // only symlinks touch the fs (existsSync + self-heal of the durable blob). |
| 419 | this.manifest = { files: [], totalSize: 0, rootTreeHash: null } |
| 420 | for (const entries of filesByDir.values()) { |
| 421 | for (const e of entries) { |
| 422 | this.manifest.files.push({ |
| 423 | path: e.path, |
| 424 | size: e.size, |
| 425 | diskPath: this._diskPathFor(e), |
| 426 | hash: e.contentHash || null |
| 427 | }) |
| 428 | this.manifest.totalSize += e.size |
| 429 | } |
| 430 | } |
| 431 | const reused = this.manifest.files.length |
| 432 | |
| 433 | // New dirs (on disk, absent from the index) → incremental index (files + |
| 434 | // trees + symlink blobs), appending to this.manifest. |
| 435 | const newDirs = [] |
| 436 | for (const dir of currentDirs) if (!filesByDir.has(dir)) newDirs.push(dir) |
| 437 | if (newDirs.length) await this._scanIncremental(newDirs) |
| 438 | |
| 439 | this.emit('scan-cache-stats', { |
| 440 | hits: reused, |
| 441 | misses: this.manifest.files.length - reused, |
| 442 | hashedBytes: 0, |
| 443 | totalFiles: this.manifest.files.length, |
| 444 | prunedFiles: pruned, |
| 445 | newDirs: newDirs.length, |
| 446 | fastPath: true |
| 447 | }) |
| 448 | // manifest.rootTreeHash is left null on the fast path — vestigial for a |
| 449 | // multi-tree store (clients fetch specific tree hashes, which stay |
| 450 | // individually persisted). The full-scan fallback still computes it. |
| 451 | this.emit('indexed', this.manifest.files.length, this.manifest.totalSize) |
| 452 | } |
| 453 | |
| 454 | /** |
| 455 | * diskPath for a persisted file entry. Symlink entries (non-null linkname) |
| 456 | * resolve to the durable content-addressed blob, self-healed from the |
| 457 | * persisted linkname if missing (no tree re-read). Regular files resolve to |
| 458 | * their in-tree path (stable — dirs are immutable), with no fs call. |
| 459 | */ |
| 460 | _diskPathFor (entry) { |
| 461 | if (entry.linkname != null && entry.contentHash) { |
| 462 | const blob = path.join(this._symlinkBlobsDir, entry.contentHash) |
| 463 | if (!fs.existsSync(blob)) { |
| 464 | try { fs.writeFileSync(blob, Buffer.from(entry.linkname, 'utf8')) } catch {} |
| 465 | } |
| 466 | return blob |
| 467 | } |
| 468 | return path.join(this.dir, entry.path) |
| 469 | } |
| 470 | |
| 335 | 471 | async _scan () { |
| 336 | 472 | const files = walkDir(this.dir) |
| 337 | 473 | let totalSize = 0 |
| 338 | 474 | |
| 339 | | // Scratch dir for symlink-target blobs. Each unique link-target |
| 340 | | // string gets materialized to <scratch>/<contentHash> so that the |
| 341 | | // existing diskPath-based content-request flow can serve it via |
| 342 | | // fs.readFileSync. Single source of bytes; the protocol layer does |
| 343 | | // not need to know about symlinks. |
| 344 | | if (!this._symlinkBlobsDir) { |
| 345 | | this._symlinkBlobsDir = path.join(os.tmpdir(), 'hyper-deploy-symlinks-' + crypto.randomBytes(8).toString('hex')) |
| 346 | | fs.mkdirSync(this._symlinkBlobsDir, { recursive: true }) |
| 347 | | } |
| 475 | // Durable dir for symlink-target blobs (set in the constructor, ensured in |
| 476 | // _open). Each unique link-target string is materialized to |
| 477 | // <symlinkBlobsDir>/<contentHash> so the diskPath-based content-request flow |
| 478 | // can serve it via fs.readFileSync. Single source of bytes; the protocol |
| 479 | // layer needn't know about symlinks. |
| 480 | fs.mkdirSync(this._symlinkBlobsDir, { recursive: true }) |
| 348 | 481 | |
| 349 | 482 | // Mtime cache stats — measured at debug/observability level. When |
| 350 | 483 | // the fileIndex already has a matching (size, mtime) for a path, |
| … | … | @@ -395,7 +528,10 @@ export class DeployServer extends ReadyResource { |
| 395 | 528 | await this._fileIndex.putFile(f.path, { |
| 396 | 529 | contentHash, |
| 397 | 530 | size: f.size, |
| 398 | | mtime |
| 531 | mtime, |
| 532 | // Persist the symlink target so the fast boot path can rebuild the |
| 533 | // diskPath (durable blob) without an lstat. null for regular files. |
| 534 | linkname: f.isSymlink ? f.linkTarget : null |
| 399 | 535 | }) |
| 400 | 536 | } |
| 401 | 537 | |