// A failed release pull must not throw the work away.
//
// reconcile() clears `pending` BEFORE pulling, then restored the repo list on
// exactly ONE failure reason ('a pull is already in progress'). Every other
// failure — notably the 15s `swarm-rpc: connection timeout` that fires when the
// mirror answers its own request for the origin — silently discarded the scoped
// repos and scheduled no retry.
//
// Nothing replays them: pear-git.js saveCursor(seq) runs immediately after
// notify(entry), and notify only QUEUES, so the release-event cursor is already
// past the dropped work — across restarts too.
//
// Measured on stortron 2026-08-16: 24 timeouts vs 46 successful pulls in ~40
// minutes, i.e. a third of reconciles dropped. Net drift was zero only because
// repos publish repeatedly and a later event covered the loss; a repo that
// publishes ONCE loses that release until its next publish.
import test from 'brittle'
import { mkdtemp, rm } from '#node:fs/promises'
import { writeFileSync } from '#node:fs'
import { tmpdir } from '#node:os'
import { join } from '#node:path'
import { startReleasePull, ReleasePuller } from '../lib/services/release-puller.js'
const tick = (ms) => new Promise((r) => setTimeout(r, ms))
function fakePuller (onPull) {
return { pullFrom: onPull, stats: { errors: [] } }
}
test('a connect failure re-queues the repos and retries', async (t) => {
const attempts = []
let failNext = true
const pull = startReleasePull({
puller: fakePuller(async () => ({ ok: true, stats: { fetched: 1, replaced: 0, skipped: 0, errors: [] } })),
originServerKey: 'a'.repeat(64),
debounceMs: 10,
connect: async () => {
attempts.push(Date.now())
if (failNext) {
failNext = false
throw new Error('swarm-rpc: connection timeout (2 peer(s) answered but failed verification)')
}
return { call: async () => ({ files: [] }), close: async () => {} }
},
log: () => {}
})
t.teardown(() => pull.stop())
pull.notify({ repo: 'ifsrenderer-linux' })
// First attempt fails. Pre-fix the repo was gone here and nothing retried.
await tick(400)
t.ok(attempts.length >= 2, 'the failed pull was retried rather than dropped')
t.is(pull.pending().length, 0, 'and the repo was drained once a retry succeeded')
})
test('the re-queued repo is the one that failed', async (t) => {
const seen = []
let fail = true
const pull = startReleasePull({
puller: fakePuller(async (client, opts) => {
seen.push([...(opts.repos || [])].sort())
return { ok: true, stats: { fetched: 0, replaced: 0, skipped: 0, errors: [] } }
}),
originServerKey: 'b'.repeat(64),
debounceMs: 10,
connect: async () => {
if (fail) { fail = false; throw new Error('swarm-rpc: connection timeout') }
return { call: async () => ({ files: [] }), close: async () => {} }
},
log: () => {}
})
t.teardown(() => pull.stop())
pull.notify({ repo: 'hyper-tps' })
await tick(400)
t.ok(seen.length >= 1, 'a pull eventually reached the puller')
t.alike(seen[0], ['hyper-tps'], 'carrying the repo whose event was dropped')
})
test('repeated failures back off instead of hot-looping', async (t) => {
const at = []
const pull = startReleasePull({
puller: fakePuller(async () => ({ ok: true, stats: { errors: [] } })),
originServerKey: 'c'.repeat(64),
debounceMs: 20,
connect: async () => {
at.push(Date.now())
throw new Error('swarm-rpc: connection timeout')
},
log: () => {}
})
t.teardown(() => pull.stop())
pull.notify({ repo: 'always-fails' })
await tick(700)
t.ok(at.length >= 3, 'it kept retrying')
// Fixed-interval retry at 20ms would be ~35 attempts in 700ms. Backoff must
// grow the gap, so the count stays far below that.
t.ok(at.length < 15, 'but backed off rather than retrying every debounce (' + at.length + ' attempts)')
const first = at[1] - at[0]
const last = at[at.length - 1] - at[at.length - 2]
t.ok(last > first, 'the gap between attempts grew (' + first + 'ms → ' + last + 'ms)')
})
test('a successful pull clears the backoff', async (t) => {
let fail = true
const at = []
const pull = startReleasePull({
puller: fakePuller(async () => ({ ok: true, stats: { errors: [] } })),
originServerKey: 'd'.repeat(64),
debounceMs: 10,
connect: async () => {
at.push(Date.now())
if (fail) throw new Error('swarm-rpc: connection timeout')
return { call: async () => ({ files: [] }), close: async () => {} }
},
log: () => {}
})
t.teardown(() => pull.stop())
pull.notify({ repo: 'r1' })
await tick(300)
fail = false
await tick(400)
const before = at.length
pull.notify({ repo: 'r2' })
await tick(200)
t.ok(at.length > before, 'a later event is served promptly, not stuck behind the grown backoff')
t.is(pull.pending().length, 0, 'and nothing is left queued')
})
test('per-pull deltas are reported, not lifetime totals', async (t) => {
// ReleasePuller.stats is cumulative by design (fetched++, skipped +=,
// errors.push, never reset). The summary printed those cumulative fields per
// repo, so the live log read `[ifsrenderer-linux] fetched 61, skipped 1994`
// when that pull had done far less — counts that only ever climbed, across
// different repos.
const puller = new ReleasePuller({
releaseDir: '/nonexistent',
overlay: { fetchFile: async () => null }
})
puller.stats.fetched = 60
puller.stats.skipped = 1900
puller.stats.errors.push('an old error from a previous pull')
const res = await puller.pullFrom({ call: async () => ({ files: [] }) }, { repos: ['x'] })
t.ok(res.delta, 'a per-pull delta is returned')
t.is(res.delta.fetched, 0, 'delta counts only this pull, not the lifetime 60')
t.is(res.delta.skipped, 0, 'same for skipped')
t.is(res.delta.errors.length, 0, 'and the stale error is not re-reported')
t.is(res.stats.fetched, 60, 'the cumulative total is still available')
})
// ── a pull that fetched SOME of a version is not a success ──────────────
//
// pullFrom returned { ok: true } whenever it completed, no matter how many
// individual files failed — the live log line is literally
// `fetched 1, replaced 0, skipped 120, errors 3` on the SUCCESS path. reconcile()
// only re-queues on `!res.ok` and resets `failures = 0` otherwise, so a version
// that lost the publish/serve race was stranded permanently: nothing retried it,
// and the release-event cursor had long since moved past.
//
// Observed on stortron: ifsrenderer-linux linux-port-14855c7 sat with one of four
// files present for ~12 hours, its channel still pinned to the previous release,
// until a pull was triggered by hand. The bytes were fetchable the whole time —
// only the retry was missing.
//
// The origin-side ordering fix (release-index-serve-order.test.js) closes the
// window; this closes the consequence, so a partial pull heals itself even when
// the far side is briefly unable to serve.
test('a pull that could not fetch every file reports itself partial', async (t) => {
const dir = await mkdtemp(join(tmpdir(), 'relpull-partial-'))
t.teardown(() => rm(dir, { recursive: true, force: true }))
const puller = new ReleasePuller({
releaseDir: dir,
overlay: { fetchFile: async () => null } // null = "no holder"
})
const res = await puller.pullFrom({
call: async () => ({
files: [{ path: 'demo/versions/1/app.bin', hash: 'f'.repeat(64), size: 10, repo: 'demo', version: '1' }]
})
})
t.ok(res.ok, 'the pull itself ran — this is not a transport failure')
t.ok(res.partial, 'but it is flagged partial, because a file it was asked for did not land')
})
test('a pull where everything landed is not partial', async (t) => {
// The control. Without it, `partial: true` on every pull would also satisfy
// the assertion above while making the retry loop spin forever.
const dir = await mkdtemp(join(tmpdir(), 'relpull-complete-'))
t.teardown(() => rm(dir, { recursive: true, force: true }))
const puller = new ReleasePuller({
releaseDir: dir,
overlay: {
fetchFile: async (hash, opts) => {
writeFileSync(opts.destPath, 'x'.repeat(10))
return { size: 10 }
}
}
})
const res = await puller.pullFrom({
call: async () => ({
files: [{ path: 'demo/versions/1/app.bin', hash: 'f'.repeat(64), size: 10, repo: 'demo', version: '1' }]
})
})
t.ok(res.ok, 'the pull ran')
t.absent(res.partial, 'and is not partial, because every file it was asked for landed')
})
test('a partial pull is retried rather than left stranded', async (t) => {
// The behaviour that was missing. Pre-fix the first (partial) pull reset
// `failures` to 0, drained `pending`, and nothing ever looked at that repo again.
const attempts = []
let landEverything = false
const pull = startReleasePull({
puller: fakePuller(async (client, opts) => {
attempts.push([...(opts.repos || [])])
return landEverything
? { ok: true, stats: { errors: [] }, delta: { errors: [] } }
: { ok: true, partial: true, stats: { errors: ['no holder for x'] }, delta: { errors: ['no holder for x'] } }
}),
originServerKey: 'e'.repeat(64),
debounceMs: 10,
connect: async () => ({ call: async () => ({ files: [] }), close: async () => {} }),
log: () => {}
})
t.teardown(() => pull.stop())
pull.notify({ repo: 'ifsrenderer-linux' })
await tick(300)
t.ok(attempts.length >= 2, 'the partial pull was retried (' + attempts.length + ' attempts)')
t.alike(attempts[1], ['ifsrenderer-linux'], 'carrying the same repo that came back incomplete')
landEverything = true
await tick(400)
t.is(pull.pending().length, 0, 'and once everything landed the repo was drained')
})
test('a new error is logged with its text, not just counted', async (t) => {
const lines = []
// A REAL dir: with a bogus releaseDir the fetch loop dies in mkdirSync and
// never reaches the "no holder" branch this test is about.
const dir = await mkdtemp(join(tmpdir(), 'relpull-'))
t.teardown(() => rm(dir, { recursive: true, force: true }))
const puller = new ReleasePuller({
releaseDir: dir,
// null with no onEnd call = fetchFile gave up during its peer wait, before
// any scheduler ran. That is NOT a no-holder: nothing about holders was
// established. See describeFetchFailure in services/release-puller.js.
overlay: { fetchFile: async () => null },
log: (l) => lines.push(l)
})
await puller.pullFrom({
call: async () => ({
files: [{ path: 'demo/versions/1/app.bin', hash: 'f'.repeat(64), size: 10, repo: 'demo', version: '1' }]
})
})
t.is(puller.stats.errors.length, 1, 'the failure was recorded')
// Assert the INTENT -- the error text reaches the log naming the artifact --
// rather than a literal phrase. This previously grepped for "no holder",
// which pinned an imprecise message as the contract: the puller called every
// falsy fetchFile a missing holder, including this case where it never got
// far enough to ask.
const logged = lines.filter((l) => l.includes('app.bin'))
t.is(logged.length, 1, 'and its TEXT reached the log, not only the count')
t.absent(/no holder/i.test(logged[0]),
'and does not claim a holder was missing when the peer wait expired first')
t.ok(logged[0].includes('app.bin'), 'naming the artifact that failed')
})