| … | … | @@ -3336,9 +3336,59 @@ function setupMirrorWriterCoordination ({ |
| 3336 | 3336 | // re-trigger after rebinding a tracker post-boot. The first call is |
| 3337 | 3337 | // gated by `announced`; subsequent re-announces (post-rebind) bypass |
| 3338 | 3338 | // the gate by passing `force: true`. |
| 3339 | // Failure handling here is load-bearing. `registerMirrorWriters` APPENDS to |
| 3340 | // the identity autobase, and every append calls Autobase._bump, which is |
| 3341 | // debounceify'd around _advance — so a failed announce that retries |
| 3342 | // immediately drives _advance in a pure-microtask cycle that never hands the |
| 3343 | // event loop back. |
| 3344 | // |
| 3345 | // Measured on stortron (mirror, 145 repos, swarm disabled): the heartbeat died |
| 3346 | // at t=45s with _advance at 1,684, then _advance ran past 800,000 while |
| 3347 | // _queueBump stayed flat at 427 — re-entry, not scheduling. Caller stacks |
| 3348 | // captured at _advance #200,000 and #800,000 were identical: |
| 3349 | // _doAnnounce -> registerMirrorWriters -> _appendBatch -> _bump -> _advance |
| 3350 | // The daemon never printed its banner, never accept()ed a connection, and left |
| 3351 | // exited git children unreaped (SIGCHLD also needs the macrotask phase). |
| 3352 | // |
| 3353 | // The trigger is `announce failed: Not writable` — a mirror whose writer key |
| 3354 | // the origin has not enrolled yet. That is the NORMAL state on first boot after |
| 3355 | // re-keying, and permanent when the swarm is off. Clearing `announced` in the |
| 3356 | // catch reopened the gate with no delay and no in-flight guard, so every retry |
| 3357 | // re-opened all 145 tracker autobases and appended again. |
| 3358 | // |
| 3359 | // Three guards, all required: |
| 3360 | // - `announcing` — an in-flight attempt is never re-entered |
| 3361 | // - backoff — a failed attempt is not retried immediately |
| 3362 | // - setTimeout — the retry is a MACROTASK, so the loop is handed back. |
| 3363 | // `await` alone cannot do this (see 0.1.180, 0.1.214). |
| 3364 | const ANNOUNCE_BACKOFF_MIN = 2000 |
| 3365 | const ANNOUNCE_BACKOFF_MAX = 60000 |
| 3366 | |
| 3339 | 3367 | let announced = false |
| 3368 | let announcing = false |
| 3369 | let announceFailedAt = 0 |
| 3370 | let announceBackoffMs = ANNOUNCE_BACKOFF_MIN |
| 3371 | let announceRetryTimer = null |
| 3372 | |
| 3373 | function scheduleAnnounceRetry () { |
| 3374 | if (announceRetryTimer) return |
| 3375 | announceRetryTimer = setTimeout(() => { |
| 3376 | announceRetryTimer = null |
| 3377 | _doAnnounce({ force: false }).catch(() => {}) |
| 3378 | }, announceBackoffMs) |
| 3379 | if (announceRetryTimer && typeof announceRetryTimer.unref === 'function') { |
| 3380 | announceRetryTimer.unref() |
| 3381 | } |
| 3382 | } |
| 3383 | |
| 3340 | 3384 | async function _doAnnounce ({ force = false } = {}) { |
| 3341 | 3385 | if (announced && !force) return |
| 3386 | if (announcing) return // an attempt is already in flight — never overlap |
| 3387 | if (announceFailedAt && (Date.now() - announceFailedAt) < announceBackoffMs) { |
| 3388 | scheduleAnnounceRetry() // still cooling down; ensure a retry is pending |
| 3389 | return |
| 3390 | } |
| 3391 | announcing = true |
| 3342 | 3392 | announced = true |
| 3343 | 3393 | try { |
| 3344 | 3394 | const keys = {} |
| … | … | @@ -3389,13 +3439,25 @@ function setupMirrorWriterCoordination ({ |
| 3389 | 3439 | keys.__repoeventlog__ = svc.bootstrapKeys.repoEventLog |
| 3390 | 3440 | } |
| 3391 | 3441 | await identityRegistry.registerMirrorWriters(identityHex, keys) |
| 3442 | // Success: clear the failure clock so a later re-announce starts fresh. |
| 3443 | announceFailedAt = 0 |
| 3444 | announceBackoffMs = ANNOUNCE_BACKOFF_MIN |
| 3392 | 3445 | // eslint-disable-next-line no-console |
| 3393 | 3446 | console.log('[mirror-writer] announced local keys to primary:', |
| 3394 | 3447 | Object.keys(keys).join(', '), force ? '(re-announce)' : '') |
| 3395 | 3448 | } catch (err) { |
| 3396 | 3449 | if (!force) announced = false |
| 3450 | // Back off before the next attempt, and make that attempt a MACROTASK. |
| 3451 | // Without this, "Not writable" retried at microtask speed and each retry |
| 3452 | // appended to the identity autobase — the wedge described above. |
| 3453 | announceFailedAt = Date.now() |
| 3397 | 3454 | // eslint-disable-next-line no-console |
| 3398 | | console.error('[mirror-writer] announce failed:', err.message) |
| 3455 | console.error('[mirror-writer] announce failed:', err.message, |
| 3456 | '— retrying in ' + Math.round(announceBackoffMs / 1000) + 's') |
| 3457 | announceBackoffMs = Math.min(announceBackoffMs * 2, ANNOUNCE_BACKOFF_MAX) |
| 3458 | scheduleAnnounceRetry() |
| 3459 | } finally { |
| 3460 | announcing = false |
| 3399 | 3461 | } |
| 3400 | 3462 | } |
| 3401 | 3463 | |