| … | … | @@ -152,6 +152,14 @@ export class Sidecar { |
| 152 | 152 | // ENOTEMPTYs mid-rm and leaves an empty (or partially-populated) |
| 153 | 153 | // install dir behind. |
| 154 | 154 | this._installInFlight = new Set() |
| 155 | // Per-app persistent-failure backoff. Keyed by app name → |
| 156 | // { token, count, nextRetryAt }. An autoupdate that fails the SAME way every |
| 157 | // tick (un-fetchable tree, a registry Corestore whose fd-lock can't be |
| 158 | // acquired, etc.) must NOT be retried on every autobase 'update' event — that |
| 159 | // zero-backoff tight loop bled the sidecar to 232k fds / 15.8 GB on stortron |
| 160 | // (2026-07-13). We back off exponentially per (name, token); a new token |
| 161 | // (version/tree bump) resets the counter for a fresh attempt. |
| 162 | this._installFailures = new Map() |
| 155 | 163 | // Overlay subsystem — per-repo file metadata index + cross-peer file |
| 156 | 164 | // serving. Initialised lazily during start() because it needs the |
| 157 | 165 | // identity (for the swarm topic). Stays null when enableSeed === false |
| … | … | @@ -698,6 +706,12 @@ export class Sidecar { |
| 698 | 706 | continue |
| 699 | 707 | } |
| 700 | 708 | |
| 709 | // Persistent-failure backoff. If this exact release (name, token) has been |
| 710 | // failing, don't hammer it on every autobase 'update' tick — wait out the |
| 711 | // exponential backoff. A new token (version/tree bump) falls through with a |
| 712 | // fresh counter. See _installFailures in the constructor. |
| 713 | if (this._isInInstallBackoff(name, token)) continue |
| 714 | |
| 701 | 715 | // Per-app install mutex — see _installInFlight comment in |
| 702 | 716 | // constructor. Skip if a previous tick is still installing. |
| 703 | 717 | if (this._installInFlight.has(name)) { |
| … | … | @@ -717,16 +731,42 @@ export class Sidecar { |
| 717 | 731 | this._pushReleaseReady({ name, udsPath }, best).catch(() => {}) |
| 718 | 732 | } |
| 719 | 733 | this._lastNotifiedVersion.set(name, token) |
| 734 | this._installFailures.delete(name) |
| 720 | 735 | } catch (err) { |
| 721 | | console.error('[pgh-sidecar] autoupdate failed for ' + name + ': ' + err.message) |
| 722 | | // Retry on the next tick; don't poison _lastNotifiedVersion so |
| 723 | | // a later fire gets another shot. |
| 736 | // Record the failure and back off. Don't poison _lastNotifiedVersion so |
| 737 | // a later fire (once the backoff elapses, or a new version appears) gets |
| 738 | // another shot — but no longer on EVERY tick (the leak that melted the |
| 739 | // sidecar). Backoff resets when the release token changes. |
| 740 | const prev = this._installFailures.get(name) |
| 741 | const count = (prev && prev.token === token ? prev.count : 0) + 1 |
| 742 | const backoffMs = this._installBackoffMs(count) |
| 743 | this._installFailures.set(name, { token, count, nextRetryAt: Date.now() + backoffMs }) |
| 744 | console.error('[pgh-sidecar] autoupdate failed for ' + name + ': ' + err.message + |
| 745 | ' (attempt ' + count + '; backing off ' + Math.round(backoffMs / 1000) + 's)') |
| 724 | 746 | } finally { |
| 725 | 747 | this._installInFlight.delete(name) |
| 726 | 748 | } |
| 727 | 749 | } |
| 728 | 750 | } |
| 729 | 751 | |
| 752 | // Exponential backoff for a persistently-failing autoupdate. Attempt 1 waits |
| 753 | // 30s, then doubles (1m, 2m, 4m, …) capped at 1h. Bounds the retry rate of an |
| 754 | // un-installable release so a stuck app can't drive an unbounded fd/socket |
| 755 | // leak (see _installFailures in the constructor). |
| 756 | _installBackoffMs (count) { |
| 757 | const base = 30000 |
| 758 | const cappedExp = Math.min(Math.max(count, 1) - 1, 7) // cap so no overflow |
| 759 | return Math.min(base * Math.pow(2, cappedExp), 3600000) |
| 760 | } |
| 761 | |
| 762 | // True while (name, token) is inside its post-failure backoff window. A token |
| 763 | // change (new version/tree) is NOT in backoff — a fresh release always gets an |
| 764 | // immediate attempt. Extracted for testability. |
| 765 | _isInInstallBackoff (name, token) { |
| 766 | const fail = this._installFailures.get(name) |
| 767 | return !!(fail && fail.token === token && Date.now() < fail.nextRetryAt) |
| 768 | } |
| 769 | |
| 730 | 770 | /** |
| 731 | 771 | * Union of running-registered apps (sidecar's in-memory map) plus |
| 732 | 772 | * installed-on-disk apps (have a .pgh-install.json). CLIs like pgh |