| … | … | @@ -17,6 +17,19 @@ import { |
| 17 | 17 | normalizeStorageClass |
| 18 | 18 | } from '../constants.js' |
| 19 | 19 | |
| 20 | function seedAdmissionTimeoutError (timeoutMs) { |
| 21 | const err = new Error(`Seed admission timed out after ${timeoutMs}ms`) |
| 22 | err.code = 'STORAGE_SEED_ADMISSION_TIMEOUT' |
| 23 | return err |
| 24 | } |
| 25 | |
| 26 | function seedAdmissionAbortError () { |
| 27 | const err = new Error('Seed admission aborted') |
| 28 | err.name = 'AbortError' |
| 29 | err.code = 'ABORT_ERR' |
| 30 | return err |
| 31 | } |
| 32 | |
| 20 | 33 | /** |
| 21 | 34 | * AppLifecycle — owns seeding, unseeding, and manifest indexing for a RelayNode. |
| 22 | 35 | * |
| … | … | @@ -30,6 +43,10 @@ export class AppLifecycle extends EventEmitter { |
| 30 | 43 | super() |
| 31 | 44 | this.node = node |
| 32 | 45 | this._seedMutex = false |
| 46 | // Serialization tail for seed admission. Callers queue behind it via |
| 47 | // _queueSeedAdmission; the tail itself never rejects, so stop paths can |
| 48 | // `await lifecycle._seedTail` to know the chain has drained. |
| 49 | this._seedTail = Promise.resolve() |
| 33 | 50 | // Drives whose live resources (ranges, drive, bridge) have been settled |
| 34 | 51 | // but whose durable delete has not yet completed. On failure the entry is |
| 35 | 52 | // restored to the registry and parked here so a retry can finish the |
| … | … | @@ -38,6 +55,10 @@ export class AppLifecycle extends EventEmitter { |
| 38 | 55 | // Active auxiliary share-bundle fetches. unseedApp drains these before |
| 39 | 56 | // releasing drive debt so a concurrent fetch cannot race with teardown. |
| 40 | 57 | this._auxShareBundleResources = new Set() |
| 58 | // Drive keys whose storage lease this lifecycle already holds, so a |
| 59 | // re-entrant call (seed → evict → unseed) doesn't deadlock on the |
| 60 | // non-reentrant per-key mutation lock. |
| 61 | this._heldDriveLeases = new Set() |
| 41 | 62 | } |
| 42 | 63 | |
| 43 | 64 | /** |
| … | … | @@ -48,6 +69,82 @@ export class AppLifecycle extends EventEmitter { |
| 48 | 69 | return this.node.appRegistry.apps |
| 49 | 70 | } |
| 50 | 71 | |
| 72 | /** |
| 73 | * Serialize a seed-admission closure behind every earlier one. |
| 74 | * |
| 75 | * Why a queue and not a spin-wait mutex: a caller that gives up (timeout) |
| 76 | * or a node that is shutting down (scope abort) must ALSO cancel the work |
| 77 | * it queued. The old `while (this._seedMutex) await sleep(50)` loop could |
| 78 | * not do that — the waiter's closure still ran when the mutex freed, so a |
| 79 | * seed that had already reported failure to its caller went on to mutate |
| 80 | * the registry and storage-admission ledger afterwards. |
| 81 | * |
| 82 | * Cancellation only applies while a closure is WAITING. Once it starts it |
| 83 | * owns the admission slot and runs to completion; aborting mid-seed would |
| 84 | * strand a half-registered drive, which is exactly what the retirement |
| 85 | * owner in unseedApp exists to avoid. |
| 86 | * |
| 87 | * @param {() => Promise<any>} run the admission-critical closure |
| 88 | * @returns {Promise<any>} the closure's result, or a rejection with code |
| 89 | * `STORAGE_SEED_ADMISSION_TIMEOUT` / `ABORT_ERR` if it never ran. |
| 90 | */ |
| 91 | _queueSeedAdmission (run) { |
| 92 | const node = this.node || {} |
| 93 | const scope = node._scope || null |
| 94 | const signal = scope && scope.signal ? scope.signal : null |
| 95 | const configured = Number(node.config && node.config.seedAdmissionTimeoutMs) |
| 96 | const timeoutMs = Number.isFinite(configured) && configured > 0 ? configured : 0 |
| 97 | |
| 98 | const gate = { started: false, cancelled: false } |
| 99 | let resolveCaller = null |
| 100 | let rejectCaller = null |
| 101 | const caller = new Promise((resolve, reject) => { |
| 102 | resolveCaller = resolve |
| 103 | rejectCaller = reject |
| 104 | }) |
| 105 | |
| 106 | let timer = null |
| 107 | let onAbort = null |
| 108 | const disarm = () => { |
| 109 | if (timer) { clearTimeout(timer); timer = null } |
| 110 | if (onAbort && signal) { signal.removeEventListener('abort', onAbort); onAbort = null } |
| 111 | } |
| 112 | const cancel = (err) => { |
| 113 | if (gate.started || gate.cancelled) return |
| 114 | gate.cancelled = true |
| 115 | disarm() |
| 116 | rejectCaller(err) |
| 117 | } |
| 118 | |
| 119 | const previous = this._seedTail || Promise.resolve() |
| 120 | this._seedTail = previous.then(async () => { |
| 121 | if (gate.cancelled) return |
| 122 | gate.started = true |
| 123 | disarm() |
| 124 | try { |
| 125 | resolveCaller(await run()) |
| 126 | } catch (err) { |
| 127 | rejectCaller(err) |
| 128 | } |
| 129 | }) |
| 130 | |
| 131 | if (signal && signal.aborted) { |
| 132 | cancel(seedAdmissionAbortError()) |
| 133 | } else { |
| 134 | // Deliberately NOT unref'd: a caller is blocked on this timeout, so the |
| 135 | // event loop must stay alive long enough to deliver it. It is cleared as |
| 136 | // soon as the closure starts or the wait is cancelled, so it can only |
| 137 | // hold the process for at most seedAdmissionTimeoutMs. |
| 138 | if (timeoutMs > 0) timer = setTimeout(() => cancel(seedAdmissionTimeoutError(timeoutMs)), timeoutMs) |
| 139 | if (signal) { |
| 140 | onAbort = () => cancel(seedAdmissionAbortError()) |
| 141 | signal.addEventListener('abort', onAbort, { once: true }) |
| 142 | } |
| 143 | } |
| 144 | |
| 145 | return caller |
| 146 | } |
| 147 | |
| 51 | 148 | /** |
| 52 | 149 | * Load the durable drive registry (and run the one-shot seeded-apps.json |
| 53 | 150 | * migration when empty). Does NOT re-seed drives — callers that need the |
| … | … | @@ -188,8 +285,24 @@ export class AppLifecycle extends EventEmitter { |
| 188 | 285 | |
| 189 | 286 | async seedApp (appKeyHex, opts = {}) { |
| 190 | 287 | const node = this.node |
| 288 | // Recovery inventories have not sealed, so the relay does not yet know |
| 289 | // what it already owns — admitting new content here could double-count |
| 290 | // storage against a commitment the recovery pass is about to adopt. |
| 291 | // Checked before every other gate (including "seeding not enabled") so |
| 292 | // the caller gets the specific, retryable cause rather than a generic |
| 293 | // refusal that hides the fact that startup simply hasn't finished. |
| 294 | if (node._storageIngressReady === false) { |
| 295 | const err = new Error('storage recovery inventory has not sealed') |
| 296 | err.code = 'STORAGE_RECOVERY_INVENTORY_PENDING' |
| 297 | throw err |
| 298 | } |
| 191 | 299 | if (!node.seeder) throw new Error('Seeding not enabled') |
| 192 | 300 | if (!isValidHexKey(appKeyHex)) throw new Error('Invalid app key: must be 64 hex characters') |
| 301 | // Canonicalize before ANYTHING keys off it. AppRegistry stores lowercase, |
| 302 | // so a mixed-case caller otherwise misses the live entry, seeds a second |
| 303 | // drive for the same content, and serializes under a different |
| 304 | // storage-admission mutation key than the call it is racing. |
| 305 | appKeyHex = appKeyHex.toLowerCase() |
| 193 | 306 | |
| 194 | 307 | const contentType = normalizeContentType(opts.type, 'app') |
| 195 | 308 | const blind = opts.blind === true |
| … | … | @@ -254,9 +367,97 @@ export class AppLifecycle extends EventEmitter { |
| 254 | 367 | // those re-pins now via _reconcileSeedOptsOnRepin: cap-up + unanchored |
| 255 | 368 | // retriggers replication; cap-down emits a warning; same/missing is |
| 256 | 369 | // a no-op (matches prior behavior). |
| 370 | const mutationKey = `drive:${appKeyHex}` |
| 371 | return this._withDriveLease(appKeyHex, () => this._seedAppAdmitted( |
| 372 | appKeyHex, normalizedOpts, contentType, parentKey, mountPath, privacyTier, mutationKey |
| 373 | )) |
| 374 | } |
| 375 | |
| 376 | /** |
| 377 | * Run `run` while holding this drive's per-key storage lease. |
| 378 | * |
| 379 | * The same lease is taken by seedApp, unseedApp, the PVSS auxiliary fetch, |
| 380 | * and HyperGateway's HTTP reads, so a durable retirement cannot begin while |
| 381 | * a stalled HTTP response is still reading pinned core sessions, and a new |
| 382 | * read cannot enter after retirement starts. |
| 383 | * |
| 384 | * runKeyMutation is NOT re-entrant, so a held key is tracked and re-entered |
| 385 | * directly — a seed that evicts and unseeds the same key would otherwise |
| 386 | * wait on a lock it is itself holding. |
| 387 | */ |
| 388 | /** |
| 389 | * True when the registry can accept a durable write right now. Non-throwing |
| 390 | * probe used to stop storage-PRODUCING work (seed, repair) early while still |
| 391 | * allowing serve-only paths to run under a read-only physical authority. |
| 392 | */ |
| 393 | _durableWritesAvailable () { |
| 394 | const registry = this.node && this.node.appRegistry |
| 395 | if (!registry || typeof registry.assertDurableWritesAvailable !== 'function') return true |
| 396 | try { |
| 397 | registry.assertDurableWritesAvailable() |
| 398 | return true |
| 399 | } catch (_) { |
| 400 | return false |
| 401 | } |
| 402 | } |
| 403 | |
| 404 | _withDriveLease (appKeyHex, run, opts = {}) { |
| 405 | const admission = this.node && this.node.storageAdmission |
| 406 | if (!admission || typeof admission.runKeyMutation !== 'function') return run() |
| 407 | if (this._heldDriveLeases.has(appKeyHex)) return run() |
| 408 | // stop() retires every seeded drive AFTER the authority stops accepting |
| 409 | // mutations, and runKeyMutation fails closed in that state. Gating a |
| 410 | // retirement on it would leave shutdown unable to close its own drives. |
| 411 | // Seeds keep failing closed here — refusing new content while storage is |
| 412 | // stopping is the correct answer; refusing to release it is not. |
| 413 | if (opts.allowWhenStopping && typeof admission.mutationAdmission === 'function' && |
| 414 | admission.mutationAdmission().allowed !== true) { |
| 415 | return run() |
| 416 | } |
| 417 | return admission.runKeyMutation(`drive:${appKeyHex}`, async () => { |
| 418 | this._heldDriveLeases.add(appKeyHex) |
| 419 | try { |
| 420 | return await run() |
| 421 | } finally { |
| 422 | this._heldDriveLeases.delete(appKeyHex) |
| 423 | } |
| 424 | }) |
| 425 | } |
| 426 | |
| 427 | /** |
| 428 | * seedApp's body, already serialized on this drive's storage-admission |
| 429 | * mutation key. Split out so the already-seeded fast path runs UNDER that |
| 430 | * lock: evaluating it outside lets two concurrent callers both conclude the |
| 431 | * drive needs seeding. |
| 432 | */ |
| 433 | async _seedAppAdmitted (appKeyHex, normalizedOpts, contentType, parentKey, mountPath, privacyTier, mutationKey) { |
| 434 | const node = this.node |
| 435 | |
| 436 | // Fail before ANY storage adoption. Seeding ends in a durable registry |
| 437 | // write, so if the physical authority cannot accept one there is no point |
| 438 | // opening a Corestore session, building a Hyperdrive, or taking a storage |
| 439 | // reservation first — each of those is a resource we would then have to |
| 440 | // unwind, and the reservation error (STORAGE_ADMISSION_BLOCKED) would |
| 441 | // mask the real cause. |
| 442 | if (node.appRegistry && typeof node.appRegistry.assertDurableWritesAvailable === 'function') { |
| 443 | node.appRegistry.assertDurableWritesAvailable() |
| 444 | } |
| 445 | |
| 257 | 446 | if (this.seededApps.has(appKeyHex)) { |
| 258 | 447 | const existing = this.seededApps.get(appKeyHex) |
| 259 | 448 | if (existing && existing.discoveryKey) { |
| 449 | // Returning alreadySeeded ACKNOWLEDGES durable state to the caller. |
| 450 | // That is only honest if the storage ledger holds a committed record |
| 451 | // for this drive. When admission cannot acknowledge, the live entry is |
| 452 | // stale relative to durable state (crash between the registry write |
| 453 | // and the ledger commit, or a fail-closed authority) — reconcile |
| 454 | // instead of reporting a success nothing durable backs. |
| 455 | if (node.storageAdmission && typeof node.storageAdmission.canAcknowledge === 'function' && |
| 456 | !node.storageAdmission.canAcknowledge(mutationKey)) { |
| 457 | const err = new Error('seeded drive storage authority requires reconciliation') |
| 458 | err.code = 'STORAGE_RECONCILIATION_REQUIRED' |
| 459 | throw err |
| 460 | } |
| 260 | 461 | const dkHex = typeof existing.discoveryKey === 'string' |
| 261 | 462 | ? existing.discoveryKey |
| 262 | 463 | : b4a.toString(existing.discoveryKey, 'hex') |
| … | … | @@ -272,16 +473,17 @@ export class AppLifecycle extends EventEmitter { |
| 272 | 473 | // else: placeholder entry from load() — fall through to seed properly. |
| 273 | 474 | } |
| 274 | 475 | |
| 275 | | // Acquire seed mutex to prevent concurrent eviction races |
| 276 | | while (this._seedMutex) { |
| 277 | | await new Promise(resolve => setTimeout(resolve, 50)) |
| 278 | | } |
| 279 | | this._seedMutex = true |
| 280 | | try { |
| 281 | | return await this._seedAppInner(appKeyHex, normalizedOpts, contentType, parentKey, mountPath, privacyTier) |
| 282 | | } finally { |
| 283 | | this._seedMutex = false |
| 284 | | } |
| 476 | // Serialize against concurrent eviction races. Queued (not yet started) |
| 477 | // admissions are cancellable by shutdown or by seedAdmissionTimeoutMs, so |
| 478 | // a caller that has already been told "no" never mutates state later. |
| 479 | return this._queueSeedAdmission(async () => { |
| 480 | this._seedMutex = true |
| 481 | try { |
| 482 | return await this._seedAppInner(appKeyHex, normalizedOpts, contentType, parentKey, mountPath, privacyTier) |
| 483 | } finally { |
| 484 | this._seedMutex = false |
| 485 | } |
| 486 | }) |
| 285 | 487 | } |
| 286 | 488 | |
| 287 | 489 | async _seedAppInner (appKeyHex, opts, contentType, parentKey, mountPath, privacyTier) { |
| … | … | @@ -314,6 +516,7 @@ export class AppLifecycle extends EventEmitter { |
| 314 | 516 | const admissionCap = Number.isFinite(opts.maxStorage) && opts.maxStorage > 0 |
| 315 | 517 | ? Math.floor(opts.maxStorage) |
| 316 | 518 | : null |
| 519 | let admissionReservation = null |
| 317 | 520 | if (node.storageAdmission && typeof node.storageAdmission.reserve === 'function' && admissionCap !== null) { |
| 318 | 521 | const reservation = node.storageAdmission.reserve(`drive:${appKeyHex}`, admissionCap) |
| 319 | 522 | if (!reservation || reservation.allowed !== true) { |
| … | … | @@ -322,53 +525,61 @@ export class AppLifecycle extends EventEmitter { |
| 322 | 525 | err.storageAdmission = reservation |
| 323 | 526 | throw err |
| 324 | 527 | } |
| 325 | | // Commit on successful seed (after drive creation); for now track it. |
| 326 | | this._pendingAdmission = reservation |
| 327 | | } |
| 328 | | |
| 329 | | // Evict oldest app if storage capacity would be exceeded |
| 330 | | if (node.config.enableEviction !== false && node.seeder && node.seeder.totalBytesStored >= node.config.maxStorageBytes && this.seededApps.size > 0) { |
| 331 | | const TWENTY_FOUR_HOURS = 24 * 60 * 60 * 1000 |
| 332 | | let oldestKey = null |
| 333 | | let oldestTime = Infinity |
| 334 | | |
| 335 | | for (const [appKey, entry] of this.seededApps) { |
| 336 | | if (entry.startedAt < oldestTime) { |
| 337 | | oldestTime = entry.startedAt |
| 338 | | oldestKey = appKey |
| 528 | // Held in the `reserved` state until the drive exists AND its registry |
| 529 | // entry is durably persisted, then committed below. EVERY exit path |
| 530 | // between here and that commit must roll it back, so the eviction check |
| 531 | // and the Hyperdrive construction moved inside the try below — both can |
| 532 | // throw, and a reservation abandoned in `reserved` state is not merely a |
| 533 | // leaked byte count: reserve() rejects the next attempt on this key with |
| 534 | // `storage-reservation-in-progress` and canAcknowledge() stays false, so |
| 535 | // the drive can never be re-seeded, re-pinned, or acknowledged again. |
| 536 | admissionReservation = reservation |
| 537 | } |
| 538 | |
| 539 | let drive = null |
| 540 | try { |
| 541 | // Evict oldest app if storage capacity would be exceeded |
| 542 | if (node.config.enableEviction !== false && node.seeder && node.seeder.totalBytesStored >= node.config.maxStorageBytes && this.seededApps.size > 0) { |
| 543 | const TWENTY_FOUR_HOURS = 24 * 60 * 60 * 1000 |
| 544 | let oldestKey = null |
| 545 | let oldestTime = Infinity |
| 546 | |
| 547 | for (const [appKey, entry] of this.seededApps) { |
| 548 | if (entry.startedAt < oldestTime) { |
| 549 | oldestTime = entry.startedAt |
| 550 | oldestKey = appKey |
| 551 | } |
| 339 | 552 | } |
| 340 | | } |
| 341 | 553 | |
| 342 | | const shouldEvict = oldestKey && ( |
| 343 | | (opts.replicationFactor && opts.replicationFactor > (this.seededApps.get(oldestKey)?.replicationFactor || 1)) || |
| 344 | | (Date.now() - oldestTime > TWENTY_FOUR_HOURS) |
| 345 | | ) |
| 554 | const shouldEvict = oldestKey && ( |
| 555 | (opts.replicationFactor && opts.replicationFactor > (this.seededApps.get(oldestKey)?.replicationFactor || 1)) || |
| 556 | (Date.now() - oldestTime > TWENTY_FOUR_HOURS) |
| 557 | ) |
| 346 | 558 | |
| 347 | | if (shouldEvict) { |
| 348 | | await node._evictOldestApp() |
| 349 | | } else { |
| 350 | | throw new Error('Storage capacity exceeded and no eligible app to evict') |
| 559 | if (shouldEvict) { |
| 560 | await node._evictOldestApp() |
| 561 | } else { |
| 562 | throw new Error('Storage capacity exceeded and no eligible app to evict') |
| 563 | } |
| 351 | 564 | } |
| 352 | | } |
| 353 | 565 | |
| 354 | | const publisherPubkey = opts.publisherPubkey |
| 355 | | ? (typeof opts.publisherPubkey === 'string' |
| 356 | | ? opts.publisherPubkey |
| 357 | | : b4a.toString(opts.publisherPubkey, 'hex')) |
| 358 | | : null |
| 566 | const publisherPubkey = opts.publisherPubkey |
| 567 | ? (typeof opts.publisherPubkey === 'string' |
| 568 | ? opts.publisherPubkey |
| 569 | : b4a.toString(opts.publisherPubkey, 'hex')) |
| 570 | : null |
| 359 | 571 | |
| 360 | | const appKey = b4a.from(appKeyHex, 'hex') |
| 361 | | // Use a per-app corestore session so drive.close() does NOT propagate to |
| 362 | | // the shared root store. A session shares the same key-addressed hypercore |
| 363 | | // objects (same _root.cores map, same key derivation for explicit keys) but |
| 364 | | // its _close() only tears down this session's refs — the root store stays |
| 365 | | // open for all other seeded drives. Without this, any unseed path (custody |
| 366 | | // expiry, eviction, manual unseed) would call corestore.close() on the root |
| 367 | | // store and wedge the entire relay until systemd restarted the process. |
| 368 | | // See: .planning/debug/CAPTURED-TRACE-2026-05-18.md (root cause confirmed). |
| 369 | | const drive = new Hyperdrive(node.store.session(), appKey) |
| 572 | const appKey = b4a.from(appKeyHex, 'hex') |
| 573 | // Use a per-app corestore session so drive.close() does NOT propagate to |
| 574 | // the shared root store. A session shares the same key-addressed hypercore |
| 575 | // objects (same _root.cores map, same key derivation for explicit keys) but |
| 576 | // its _close() only tears down this session's refs — the root store stays |
| 577 | // open for all other seeded drives. Without this, any unseed path (custody |
| 578 | // expiry, eviction, manual unseed) would call corestore.close() on the root |
| 579 | // store and wedge the entire relay until systemd restarted the process. |
| 580 | // See: .planning/debug/CAPTURED-TRACE-2026-05-18.md (root cause confirmed). |
| 581 | drive = new Hyperdrive(node.store.session(), appKey) |
| 370 | 582 | |
| 371 | | try { |
| 372 | 583 | // 2026-05-24: hard timeout on drive.ready(). On a drive whose |
| 373 | 584 | // underlying hypercore is in a bad state (corrupted index, hung |
| 374 | 585 | // session, etc.), ready() can hang indefinitely — and because |
| … | … | @@ -446,6 +657,11 @@ export class AppLifecycle extends EventEmitter { |
| 446 | 657 | ? Math.floor(opts.maxStorage) |
| 447 | 658 | : null |
| 448 | 659 | |
| 660 | // Captured before the write so a failed durable persist can restore the |
| 661 | // exact pre-seed view (usually the null-discoveryKey placeholder that |
| 662 | // load() created), rather than leaving a half-seeded entry behind. |
| 663 | const previousEntry = node.appRegistry.get(appKeyHex) || null |
| 664 | |
| 449 | 665 | node.appRegistry.set(appKeyHex, { |
| 450 | 666 | drive, |
| 451 | 667 | discoveryKey, |
| … | … | @@ -475,7 +691,38 @@ export class AppLifecycle extends EventEmitter { |
| 475 | 691 | unseedFreezeMs, |
| 476 | 692 | durability, |
| 477 | 693 | maxStorage |
| 478 | | }) |
| 694 | }, { persist: false }) |
| 695 | |
| 696 | // Durability gate: persist EXPLICITLY and await it before reporting the |
| 697 | // seed as successful. AppRegistry.set() persists fire-and-forget, so a |
| 698 | // failed durable write used to be swallowed — seedApp returned a |
| 699 | // discoveryKey, the drive served traffic, and the entry simply was not |
| 700 | // there after the next restart. On failure, restore the pre-seed view |
| 701 | // and rethrow; the enclosing catch closes the drive. |
| 702 | if (typeof node.appRegistry.persistEntry === 'function') { |
| 703 | try { |
| 704 | await node.appRegistry.persistEntry(appKeyHex, { throwOnError: true }) |
| 705 | } catch (err) { |
| 706 | if (previousEntry) node.appRegistry.set(appKeyHex, previousEntry, { persist: false }) |
| 707 | else node.appRegistry.delete(appKeyHex, { persist: false }) |
| 708 | throw err |
| 709 | } |
| 710 | } |
| 711 | |
| 712 | // Durable state now exists for this drive, so the reservation becomes a |
| 713 | // committed commitment. Ordered AFTER persistEntry for the same reason |
| 714 | // _reconcileSeedOptsOnRepin commits last: a crash between the persist and |
| 715 | // the commit leaves a durable entry that startup re-adopts via |
| 716 | // adoptRecovery(), whereas committing first would leave a commitment in |
| 717 | // the ledger with no durable entry backing it. |
| 718 | // |
| 719 | // owns() guards the commit because commit() on a token this authority no |
| 720 | // longer holds trips failClosed() and wedges the whole relay. |
| 721 | if (admissionReservation && typeof node.storageAdmission.owns === 'function' && |
| 722 | node.storageAdmission.owns(admissionReservation)) { |
| 723 | node.storageAdmission.commit(admissionReservation) |
| 724 | admissionReservation = null |
| 725 | } |
| 479 | 726 | |
| 480 | 727 | // 2026-05-23: register persistent download ranges on the drive's |
| 481 | 728 | // cores so they actively pull missing blocks from any peer that |
| … | … | @@ -510,11 +757,38 @@ export class AppLifecycle extends EventEmitter { |
| 510 | 757 | this.emit('seeding', { appKey: appKeyHex, discoveryKey: b4a.toString(discoveryKey, 'hex') }) |
| 511 | 758 | return { discoveryKey: b4a.toString(discoveryKey, 'hex') } |
| 512 | 759 | } catch (err) { |
| 513 | | try { await drive.close() } catch (_) {} |
| 760 | // Drive first (it may not exist yet — the eviction check and the |
| 761 | // Hyperdrive constructor both throw with `drive` still null), then undo |
| 762 | // the admission reservation so a failed seed leaves the ledger exactly as |
| 763 | // it found it. |
| 764 | if (drive) { try { await drive.close() } catch (_) {} } |
| 765 | this._rollbackSeedAdmission(admissionReservation) |
| 514 | 766 | throw err |
| 515 | 767 | } |
| 516 | 768 | } |
| 517 | 769 | |
| 770 | /** |
| 771 | * Undo a still-reserved drive admission from _seedAppInner's failure paths. |
| 772 | * |
| 773 | * rollback() — not release() — is reserve()'s counterpart. rollback restores |
| 774 | * the token's `previous` record, which on the reseed-a-placeholder path is |
| 775 | * the adoptRecovery() commitment made for this drive at startup. release() |
| 776 | * would delete that record outright, dropping a durable commitment from the |
| 777 | * ledger and understating the relay's storage debt by the drive's whole cap. |
| 778 | * |
| 779 | * owns() guards the call so a reservation that was already committed (or |
| 780 | * superseded by another holder) is left alone rather than clobbered. |
| 781 | */ |
| 782 | _rollbackSeedAdmission (reservation) { |
| 783 | const node = this.node |
| 784 | if (!reservation || !node || !node.storageAdmission) return |
| 785 | const authority = node.storageAdmission |
| 786 | if (typeof authority.rollback !== 'function' || typeof authority.owns !== 'function') return |
| 787 | try { |
| 788 | if (authority.owns(reservation)) authority.rollback(reservation) |
| 789 | } catch (_) {} |
| 790 | } |
| 791 | |
| 518 | 792 | /** |
| 519 | 793 | * Fire-and-forget wrapper around _eagerReplicate that participates in |
| 520 | 794 | * the LifecycleScope cancellation contract (see lifecycle-scope.js + |
| … | … | @@ -933,11 +1207,62 @@ export class AppLifecycle extends EventEmitter { |
| 933 | 1207 | |
| 934 | 1208 | return { ok: true, changed: true, incrementalBytes, replicationStarted: true } |
| 935 | 1209 | } |
| 1210 | /** |
| 1211 | * Read /manifest.json for indexing. |
| 1212 | * |
| 1213 | * With a storage proof, the read is taken from a checkout pinned to the |
| 1214 | * proven drive version and is strictly LOCAL (`wait: false`). Two reasons: |
| 1215 | * |
| 1216 | * - Honesty: the metadata we index must be the bytes the proof covers. A |
| 1217 | * live `drive.get()` can resolve against a newer version — or a forked |
| 1218 | * chain — so the registry would advertise a version nobody proved. |
| 1219 | * - Leak: the old read raced `drive.get()` against a 5s timer. Losing the |
| 1220 | * race rejected the caller but left the get outstanding, so a timed-out |
| 1221 | * manifest lookup kept an in-flight network request alive against a |
| 1222 | * drive the caller had already given up on. |
| 1223 | * |
| 1224 | * The checkout is always closed, including on the mismatch and error paths. |
| 1225 | * Without a proof this falls back to the legacy timed live read. |
| 1226 | */ |
| 1227 | async _readPinnedManifest (drive, proof) { |
| 1228 | if (!proof || typeof drive.checkout !== 'function') { |
| 1229 | return Promise.race([ |
| 1230 | drive.get('/manifest.json'), |
| 1231 | new Promise((_resolve, reject) => setTimeout(() => reject(new Error('manifest timeout')), 5000)) |
| 1232 | ]) |
| 1233 | } |
| 1234 | |
| 1235 | let checkout = null |
| 1236 | try { |
| 1237 | checkout = drive.checkout(proof.driveVersion) |
| 1238 | if (!checkout) return null |
| 1239 | if (typeof checkout.ready === 'function') await checkout.ready() |
| 1240 | |
| 1241 | // Fork/length stability: the checkout must be the exact metadata-core |
| 1242 | // state the proof covers. A fork swap between proving and reading would |
| 1243 | // otherwise authorize indexing bytes from a different chain. |
| 1244 | const snapshot = proof.metaCoreSnapshot |
| 1245 | const core = checkout.db && checkout.db.core |
| 1246 | if (snapshot && core) { |
| 1247 | if (Number(core.fork ?? 0) !== Number(snapshot.fork ?? 0) || |
| 1248 | Number(core.length ?? -1) !== Number(snapshot.length ?? -2)) { |
| 1249 | return null |
| 1250 | } |
| 1251 | } |
| 1252 | |
| 1253 | return await checkout.get('/manifest.json', { wait: false }) |
| 1254 | } finally { |
| 1255 | if (checkout && typeof checkout.close === 'function') { |
| 1256 | try { await checkout.close() } catch (_) {} |
| 1257 | } |
| 1258 | } |
| 1259 | } |
| 1260 | |
| 936 | 1261 | /** |
| 937 | 1262 | * Read manifest.json from a drive and deduplicate by appId. |
| 938 | 1263 | * If an older version of the same app is already seeded, unseed it. |
| 939 | 1264 | */ |
| 940 | | async _indexAppManifest (appKeyHex, drive) { |
| 1265 | async _indexAppManifest (appKeyHex, drive, proof = null) { |
| 941 | 1266 | const node = this.node |
| 942 | 1267 | // Blind drives: publisher's privacy contract says "do not inspect." |
| 943 | 1268 | // We don't open /manifest.json, don't persist any manifest-derived |
| … | … | @@ -953,10 +1278,7 @@ export class AppLifecycle extends EventEmitter { |
| 953 | 1278 | if (existingForBlindCheck && existingForBlindCheck.blind === true) return |
| 954 | 1279 | |
| 955 | 1280 | try { |
| 956 | | const manifestBuf = await Promise.race([ |
| 957 | | drive.get('/manifest.json'), |
| 958 | | new Promise((_resolve, reject) => setTimeout(() => reject(new Error('manifest timeout')), 5000)) |
| 959 | | ]) |
| 1281 | const manifestBuf = await this._readPinnedManifest(drive, proof) |
| 960 | 1282 | if (!manifestBuf) return |
| 961 | 1283 | |
| 962 | 1284 | const manifest = JSON.parse(manifestBuf.toString()) |
| … | … | @@ -1049,6 +1371,10 @@ export class AppLifecycle extends EventEmitter { |
| 1049 | 1371 | async repairUnanchored (appKeyHex, opts = {}) { |
| 1050 | 1372 | const node = this.node |
| 1051 | 1373 | if (!node.appRegistry) return false |
| 1374 | // Repair pulls blocks and re-anchors — both storage-producing. Under a |
| 1375 | // read-only physical authority the relay may still SERVE what it holds, |
| 1376 | // but it must not grow. Stop before drive.update()/download(). |
| 1377 | if (!this._durableWritesAvailable()) return false |
| 1052 | 1378 | const entry = node.appRegistry.get(appKeyHex) |
| 1053 | 1379 | if (!entry || !entry.drive) return false |
| 1054 | 1380 | if (entry.anchored === true) return true // already anchored, nothing to do |
| … | … | @@ -1236,6 +1562,7 @@ export class AppLifecycle extends EventEmitter { |
| 1236 | 1562 | async runRepairPass (opts = {}) { |
| 1237 | 1563 | const node = this.node |
| 1238 | 1564 | if (!node.appRegistry) return { checked: 0, repaired: 0, stillUnanchored: 0 } |
| 1565 | if (!this._durableWritesAvailable()) return { checked: 0, repaired: 0, stillUnanchored: 0 } |
| 1239 | 1566 | const scope = node && node._scope |
| 1240 | 1567 | if (scope && scope.aborted) return { checked: 0, repaired: 0, stillUnanchored: 0 } |
| 1241 | 1568 | |
| … | … | @@ -1289,11 +1616,75 @@ export class AppLifecycle extends EventEmitter { |
| 1289 | 1616 | if (pending.length > 0) await Promise.allSettled(pending) |
| 1290 | 1617 | } |
| 1291 | 1618 | |
| 1619 | /** |
| 1620 | * Ordered, retry-safe teardown of one auxiliary share-bundle resource. |
| 1621 | * |
| 1622 | * Order matters and is the inverse of acquisition: stop new replication |
| 1623 | * (listener) → stop discovery → stop the download tracker → close the |
| 1624 | * snapshot session → close the core → close a store we own. Closing the |
| 1625 | * core before its snapshot session, or the store before its cores, throws |
| 1626 | * SESSION_CLOSED / "Mutex has been destroyed" out of teardown. |
| 1627 | * |
| 1628 | * Each step records its own settled flag, so a step that rejects leaves |
| 1629 | * every DOWNSTREAM owner untouched and the resource parked in |
| 1630 | * _auxShareBundleResources for a retry — the same ownership discipline |
| 1631 | * unseedApp uses for drive retirement. The error propagates; callers that |
| 1632 | * must not fail (best-effort cleanup) catch it themselves. |
| 1633 | */ |
| 1292 | 1634 | async _releaseAuxShareBundleResource (resource) { |
| 1293 | 1635 | if (!resource) return |
| 1636 | const node = this.node || {} |
| 1637 | |
| 1638 | if (!resource.listenerSettled) { |
| 1639 | const swarm = node.swarm |
| 1640 | if (resource.onConnection && swarm && typeof swarm.removeListener === 'function') { |
| 1641 | swarm.removeListener('connection', resource.onConnection) |
| 1642 | } |
| 1643 | resource.listenerSettled = true |
| 1644 | } |
| 1645 | |
| 1646 | if (!resource.discoverySettled) { |
| 1647 | if (resource.discovery && typeof resource.discovery.destroy === 'function') { |
| 1648 | await resource.discovery.destroy() |
| 1649 | } else if (resource.core && node.swarm && typeof node.swarm.leave === 'function') { |
| 1650 | await node.swarm.leave(resource.core.discoveryKey) |
| 1651 | } |
| 1652 | resource.discoverySettled = true |
| 1653 | } |
| 1654 | |
| 1655 | if (!resource.trackerSettled) { |
| 1656 | if (resource.tracker && typeof resource.tracker.destroy === 'function') { |
| 1657 | resource.tracker.destroy() |
| 1658 | } |
| 1659 | resource.trackerSettled = true |
| 1660 | } |
| 1661 | |
| 1662 | if (!resource.snapshotSettled) { |
| 1663 | if (resource.snapshotCore && typeof resource.snapshotCore.close === 'function') { |
| 1664 | await resource.snapshotCore.close() |
| 1665 | } |
| 1666 | resource.snapshotSettled = true |
| 1667 | } |
| 1668 | |
| 1669 | if (!resource.coreSettled) { |
| 1670 | if (resource.core && typeof resource.core.close === 'function') { |
| 1671 | await resource.core.close() |
| 1672 | } |
| 1673 | resource.coreSettled = true |
| 1674 | } |
| 1675 | |
| 1676 | if (!resource.storeSettled) { |
| 1677 | // Only close a store this read OWNS. When _readShareBundle falls back to |
| 1678 | // node.store the corestore belongs to the node — closing it here would |
| 1679 | // tear down every drive on the relay. |
| 1680 | if (resource.auxStore && resource.auxStore !== node.store && |
| 1681 | typeof resource.auxStore.close === 'function') { |
| 1682 | await resource.auxStore.close() |
| 1683 | } |
| 1684 | resource.storeSettled = true |
| 1685 | } |
| 1686 | |
| 1294 | 1687 | this._auxShareBundleResources.delete(resource) |
| 1295 | | try { if (resource.tracker && typeof resource.tracker.destroy === 'function') resource.tracker.destroy() } catch (_) {} |
| 1296 | | try { if (resource.snapshotCore && typeof resource.snapshotCore.close === 'function') await resource.snapshotCore.close() } catch (_) {} |
| 1297 | 1688 | } |
| 1298 | 1689 | |
| 1299 | 1690 | async _readShareBundle (shareBundleKey, opts = {}) { |
| … | … | @@ -1313,6 +1704,7 @@ export class AppLifecycle extends EventEmitter { |
| 1313 | 1704 | let auxStore = null |
| 1314 | 1705 | let discovery = null |
| 1315 | 1706 | let onConnection = null |
| 1707 | let ownedResource = null |
| 1316 | 1708 | try { |
| 1317 | 1709 | if (typeof opts.createAuxStore === 'function') { |
| 1318 | 1710 | auxStore = await opts.createAuxStore() |
| … | … | @@ -1327,15 +1719,12 @@ export class AppLifecycle extends EventEmitter { |
| 1327 | 1719 | try { auxStore.replicate(conn) } catch (_) {} |
| 1328 | 1720 | } |
| 1329 | 1721 | if (typeof node.swarm.on === 'function') node.swarm.on('connection', onConnection) |
| 1330 | | core = auxStore.get({ key: b4a.from(shareBundleKey, 'hex') }) |
| 1331 | | await core.ready() |
| 1332 | | discovery = node.swarm.join(core.discoveryKey, { server: false, client: true }) |
| 1333 | | if (discovery && typeof discovery.flushed === 'function') { |
| 1334 | | await Promise.race([ |
| 1335 | | discovery.flushed().catch(() => {}), |
| 1336 | | new Promise(resolve => { const tmr = setTimeout(resolve, Math.min(timeoutMs, 5_000)); if (typeof tmr.unref === 'function') tmr.unref() }) |
| 1337 | | ]) |
| 1338 | | } |
| 1722 | // Exactly one core session + one topic join per read. This block was |
| 1723 | // duplicated verbatim by the da70b0f restore: the second acquisition |
| 1724 | // overwrote `core`/`discovery` with the first pair still open, so every |
| 1725 | // read leaked one core session and one PeerDiscovery join (neither is |
| 1726 | // reachable from the teardown resource, which only ever saw the second |
| 1727 | // pair) and paid the flush race twice. |
| 1339 | 1728 | core = auxStore.get({ key: b4a.from(shareBundleKey, 'hex') }) |
| 1340 | 1729 | await core.ready() |
| 1341 | 1730 | discovery = node.swarm.join(core.discoveryKey, { server: false, client: true }) |
| … | … | @@ -1355,7 +1744,7 @@ export class AppLifecycle extends EventEmitter { |
| 1355 | 1744 | // flight. |
| 1356 | 1745 | let resolveSettled |
| 1357 | 1746 | const operationSettled = new Promise(resolve => { resolveSettled = resolve }) |
| 1358 | | const ownedResource = { appKey, core, snapshotCore: null, tracker: null, auxStore, discovery, onConnection, operationSettled } |
| 1747 | ownedResource = { appKey, core, snapshotCore: null, tracker: null, auxStore, discovery, onConnection, operationSettled } |
| 1359 | 1748 | this._auxShareBundleResources.add(ownedResource) |
| 1360 | 1749 | try { |
| 1361 | 1750 | if (typeof core.update === 'function') { |
| … | … | @@ -1405,20 +1794,17 @@ export class AppLifecycle extends EventEmitter { |
| 1405 | 1794 | } catch (_) { |
| 1406 | 1795 | return null |
| 1407 | 1796 | } finally { |
| 1408 | | if (onConnection && typeof node.swarm.removeListener === 'function') { |
| 1409 | | try { node.swarm.removeListener('connection', onConnection) } catch (_) {} |
| 1410 | | } |
| 1411 | | if (discovery && typeof discovery.destroy === 'function') { |
| 1412 | | try { await discovery.destroy() } catch (_) {} |
| 1413 | | } else if (core) { |
| 1414 | | try { await node.swarm.leave(core.discoveryKey) } catch (_) {} |
| 1415 | | } |
| 1416 | | if (core) { |
| 1417 | | try { await core.close() } catch (_) {} |
| 1418 | | } |
| 1419 | | if (auxStore && auxStore !== node.store && typeof auxStore.close === 'function') { |
| 1420 | | try { await auxStore.close() } catch (_) {} |
| 1421 | | } |
| 1797 | // Single ordered teardown. Previously this block open-coded the release |
| 1798 | // and did NOT close the snapshot session or destroy the download |
| 1799 | // tracker — _releaseAuxShareBundleResource was the only code that did, |
| 1800 | // and nothing called it. Every snapshot-path share-bundle read therefore |
| 1801 | // leaked a core session and a download tracker. |
| 1802 | // |
| 1803 | // The synthetic fallback covers the early `return null` paths that bail |
| 1804 | // before the tracked resource exists but after cores/discovery are open. |
| 1805 | const resource = ownedResource || |
| 1806 | { appKey, core, snapshotCore: null, tracker: null, auxStore, discovery, onConnection } |
| 1807 | try { await this._releaseAuxShareBundleResource(resource) } catch (_) {} |
| 1422 | 1808 | } |
| 1423 | 1809 | } |
| 1424 | 1810 | |
| … | … | @@ -1506,6 +1892,15 @@ export class AppLifecycle extends EventEmitter { |
| 1506 | 1892 | if (!owner) { |
| 1507 | 1893 | const entry = node.appRegistry.get(appKeyHex) |
| 1508 | 1894 | if (!entry) return |
| 1895 | // A forget retirement ENDS in a durable delete, and that delete hits |
| 1896 | // memory first. If the durable write then fails we restore the entry — |
| 1897 | // but the restore is itself a registry write, so under an unavailable |
| 1898 | // physical authority it fails too and the drive vanishes from the |
| 1899 | // registry entirely. Refuse before the first mutation instead of |
| 1900 | // discovering it half-way through teardown. |
| 1901 | if (opts.forget !== false && typeof node.appRegistry.assertDurableWritesAvailable === 'function') { |
| 1902 | node.appRegistry.assertDurableWritesAvailable() |
| 1903 | } |
| 1509 | 1904 | entry.retiring = true |
| 1510 | 1905 | owner = { |
| 1511 | 1906 | appKey: appKeyHex, |
| … | … | @@ -1513,27 +1908,87 @@ export class AppLifecycle extends EventEmitter { |
| 1513 | 1908 | forget: opts.forget !== false, |
| 1514 | 1909 | bridgeRegistered: !!node.distributedDriveBridge, |
| 1515 | 1910 | rangesSettled: false, |
| 1516 | | driveSettled: false |
| 1911 | snapshotCoresSettled: false, |
| 1912 | discoverySettled: false, |
| 1913 | driveSettled: false, |
| 1914 | commitmentReleased: false, |
| 1915 | completed: false, |
| 1916 | inflight: null |
| 1517 | 1917 | } |
| 1518 | 1918 | this._retiringDrives.set(appKeyHex, owner) |
| 1519 | 1919 | } else { |
| 1520 | 1920 | if (opts.forget !== undefined) owner.forget = opts.forget !== false |
| 1521 | 1921 | } |
| 1522 | 1922 | |
| 1523 | | // Phase 1: Settle live resources (download ranges → bridge → drive). |
| 1524 | | // Ordered so the durable delete (Phase 2) cannot race with in-flight |
| 1525 | | // replication or bridge lookups. Only runs once — on retry these are |
| 1526 | | // already null. |
| 1923 | // Coalesce concurrent retirements of the same drive onto ONE owner. A |
| 1924 | // second caller (e.g. an eviction upgrading a shutdown to a forget) must |
| 1925 | // not re-close resources the first caller already has in flight — it |
| 1926 | // chains behind it instead. Its intent is applied above, so a forget |
| 1927 | // arriving mid-shutdown is still honoured by the in-flight pass. |
| 1928 | const previous = owner.inflight || Promise.resolve() |
| 1929 | const run = previous.catch(() => {}) |
| 1930 | .then(() => this._withDriveLease(appKeyHex, () => this._retireDrive(owner), { allowWhenStopping: true })) |
| 1931 | owner.inflight = run.catch(() => {}) |
| 1932 | return run |
| 1933 | } |
| 1934 | |
| 1935 | /** |
| 1936 | * Idempotent drive retirement. Every step records a settled flag on the |
| 1937 | * owner, so a rejection parks the owner in _retiringDrives with all |
| 1938 | * DOWNSTREAM owners intact and a retry resumes exactly where it stopped. |
| 1939 | */ |
| 1940 | async _retireDrive (owner) { |
| 1941 | if (owner.completed) return |
| 1942 | const node = this.node |
| 1943 | const appKeyHex = owner.appKey |
| 1944 | |
| 1945 | // Phase 1: Settle live resources, inverse of acquisition — stop pulling |
| 1946 | // bytes (ranges → proof snapshots), stop being findable (discovery), |
| 1947 | // stop bridging, then close the drive. Ordered so the durable delete |
| 1948 | // (Phase 2) cannot race with in-flight replication or bridge lookups. |
| 1949 | // A step that throws leaves everything after it untouched for the retry. |
| 1527 | 1950 | if (!owner.rangesSettled) { |
| 1528 | | if (Array.isArray(owner.entry.downloadRanges)) { |
| 1529 | | for (const dl of owner.entry.downloadRanges) { |
| 1530 | | try { if (dl && typeof dl.destroy === 'function') dl.destroy() } catch (_) {} |
| 1951 | const ranges = owner.entry.downloadRanges |
| 1952 | if (Array.isArray(ranges)) { |
| 1953 | while (ranges.length > 0) { |
| 1954 | const dl = ranges[0] |
| 1955 | if (dl && typeof dl.destroy === 'function') dl.destroy() |
| 1956 | ranges.shift() |
| 1531 | 1957 | } |
| 1532 | 1958 | owner.entry.downloadRanges = null |
| 1533 | 1959 | } |
| 1534 | 1960 | owner.rangesSettled = true |
| 1535 | 1961 | } |
| 1536 | 1962 | |
| 1963 | if (!owner.snapshotCoresSettled) { |
| 1964 | const cores = owner.entry.downloadSnapshotCores |
| 1965 | if (Array.isArray(cores)) { |
| 1966 | while (cores.length > 0) { |
| 1967 | const core = cores[0] |
| 1968 | if (core && typeof core.close === 'function') await core.close() |
| 1969 | cores.shift() |
| 1970 | } |
| 1971 | owner.entry.downloadSnapshotCores = null |
| 1972 | } |
| 1973 | owner.snapshotCoresSettled = true |
| 1974 | } |
| 1975 | |
| 1976 | // Destroy the exact PeerDiscovery handles this drive owns, rather than |
| 1977 | // leaving the topic by key: swarm.leave(discoveryKey) cannot distinguish |
| 1978 | // this drive's join from another subsystem's join on the same topic. A |
| 1979 | // handle that fails to destroy is retained by identity for the retry. |
| 1980 | if (!owner.discoverySettled) { |
| 1981 | const handles = owner.entry.discoveryHandles |
| 1982 | if (handles && typeof handles.delete === 'function') { |
| 1983 | for (const handle of [...handles]) { |
| 1984 | if (handle && typeof handle.destroy === 'function') await handle.destroy() |
| 1985 | handles.delete(handle) |
| 1986 | } |
| 1987 | owner.entry.discoveryHandles = null |
| 1988 | } |
| 1989 | owner.discoverySettled = true |
| 1990 | } |
| 1991 | |
| 1537 | 1992 | if (owner.bridgeRegistered && node.distributedDriveBridge) { |
| 1538 | 1993 | node.distributedDriveBridge.unregisterDrive(appKeyHex) |
| 1539 | 1994 | owner.bridgeRegistered = false |
| … | … | @@ -1574,10 +2029,13 @@ export class AppLifecycle extends EventEmitter { |
| 1574 | 2029 | // Phase 3: Complete — the durable delete succeeded, so leave the swarm |
| 1575 | 2030 | // and release the storage admission commitment. |
| 1576 | 2031 | try { if (node.swarm && typeof node.swarm.leave === 'function') await node.swarm.leave(owner.entry.discoveryKey) } catch (_) {} |
| 1577 | | if (owner.forget && node.storageAdmission && typeof node.storageAdmission.release === 'function') { |
| 2032 | if (owner.forget && !owner.commitmentReleased && |
| 2033 | node.storageAdmission && typeof node.storageAdmission.release === 'function') { |
| 1578 | 2034 | try { node.storageAdmission.release(`drive:${appKeyHex}`) } catch (_) {} |
| 2035 | owner.commitmentReleased = true |
| 1579 | 2036 | } |
| 1580 | 2037 | |
| 2038 | owner.completed = true |
| 1581 | 2039 | owner.entry.retiring = false |
| 1582 | 2040 | this._retiringDrives.delete(appKeyHex) |
| 1583 | 2041 | this._safeEmit('unseeded', { appKey: appKeyHex }) |