| … | … | @@ -10,7 +10,7 @@ |
| 10 | 10 | import '../lib/bare-globals.js' |
| 11 | 11 | import http from 'bare-http1' |
| 12 | 12 | import bareFetch from 'bare-fetch' |
| 13 | | import { readFile } from 'bare-fs/promises' |
| 13 | import { readFile, writeFile, mkdir, rename } from 'bare-fs/promises' |
| 14 | 14 | import fs from 'bare-fs' |
| 15 | 15 | import Hyperswarm from 'hyperswarm' |
| 16 | 16 | import Corestore from 'corestore' |
| … | … | @@ -18,6 +18,7 @@ import { DidPeer, DidPeerRouter } from 'atproto-did-peer' |
| 18 | 18 | import { openCorestore } from 'atproto-store/lib/corestore.js' |
| 19 | 19 | import { OverlayRepoStore } from 'atproto-store/lib/overlay-repo-store.js' |
| 20 | 20 | import { roomKey16, blobPath, sniffMime, isSafeCid } from '../lib/blob-serve.js' |
| 21 | import { verifyBlobCid } from '../lib/blob-refs.js' |
| 21 | 22 | import { parseHostedDids, createIsForeign, isIgnoredCollection, IGNORED_COLLECTIONS } from '../lib/hosted-dids.js' |
| 22 | 23 | import { join } from 'bare-path' |
| 23 | 24 | import { discoveryTopic } from 'atproto-identity/lib/discovery.js' |
| … | … | @@ -518,6 +519,27 @@ const delay = (ms) => new Promise((r) => setTimeout(r, ms)) |
| 518 | 519 | // The dedup key includes the DID's front/RPC topology, not just the reason — a line keyed on the reason |
| 519 | 520 | // alone is emitted once and then indistinguishable from a live failure, which is the trap #14 recorded |
| 520 | 521 | // for _logFetchMiss. Keyed this way it re-emits the moment connectivity changes. |
| 522 | // Keep a blob we had to fetch, so the next reader is served from disk and this node becomes a source. |
| 523 | // |
| 524 | // Writes into the SAME shared content-addressed dir hpp uses (blob-serve.js keeps that derivation in |
| 525 | // one place), so a co-located hpp sees the file too rather than a second copy appearing beside it. |
| 526 | // |
| 527 | // .part then rename: a reader hitting the path mid-write must never get a truncated image, and a |
| 528 | // truncated file under a content-addressed name would be served as fact forever. Same reason |
| 529 | // favela-android stages its OTA bundle that way. |
| 530 | // |
| 531 | // The CALLER verifies the bytes against the cid first. This function must never be handed unverified |
| 532 | // bytes — the filename is the hash, so writing the wrong body poisons the name permanently. |
| 533 | async function retainBlob (did, cid, bytes) { |
| 534 | const dest = blobPath(OVERLAY_ROOT, did, cid) |
| 535 | const dir = dest.slice(0, dest.lastIndexOf('/')) |
| 536 | await mkdir(dir, { recursive: true }) |
| 537 | const part = dest + '.part' |
| 538 | await writeFile(part, bytes) |
| 539 | await rename(part, dest) |
| 540 | console.log(JSON.stringify({ msg: 'blob retained from relay — this node can now serve it', did, cid: String(cid).slice(0, 20), bytes: bytes.length })) |
| 541 | } |
| 542 | |
| 521 | 543 | const _relayMiss = new Set() |
| 522 | 544 | function relayFronts (did) { |
| 523 | 545 | try { |
| … | … | @@ -1359,7 +1381,27 @@ const server = http.createServer(async (req, res) => { |
| 1359 | 1381 | // which never replicates, so waiting could never help. Fall back to asking the peer that HOLDS the |
| 1360 | 1382 | // blob, exactly as the keyless bridge does (peer-front.getBlob -> hpp's get-blob RPC). Works for |
| 1361 | 1383 | // both backends: the holder answers from its overlay files or its legacy store, whichever it has. |
| 1362 | | if (!bytes) bytes = await relayBlob(bdid, bcid) |
| 1384 | if (!bytes) { |
| 1385 | const relayed = await relayBlob(bdid, bcid) |
| 1386 | // ⛔ VERIFY BEFORE SERVING, not just before storing. The CID *is* the hash, so a front that |
| 1387 | // returns the wrong body cannot pass — and this used to be served straight through, which made |
| 1388 | // any peer answering get-blob able to put arbitrary content under someone else's avatar. |
| 1389 | if (relayed && await verifyBlobCid(relayed, bcid)) { |
| 1390 | bytes = relayed |
| 1391 | // ★ RETAIN. Content-addressed bytes we already hold have no reason to be fetched twice, and a |
| 1392 | // node that keeps them becomes a SOURCE — which is the whole point of a content-addressed |
| 1393 | // store and the thing this surface was missing. Measured 2026-08-24: tabertron served two of |
| 1394 | // e2jev's blobs from disk while stortron 404'd the same CIDs, because a keyless reader had |
| 1395 | // exactly two routes — local file, or ask the holder — and no way to keep what it was given. |
| 1396 | // With this, the first request while a holder is up is the last one that needs it up. |
| 1397 | // |
| 1398 | // Best-effort: a failed write must never fail the response. We have the bytes; the reader |
| 1399 | // gets them either way. |
| 1400 | retainBlob(bdid, bcid, relayed).catch(() => {}) |
| 1401 | } else if (relayed) { |
| 1402 | console.error(JSON.stringify({ msg: 'relayed blob failed content-address verification — refusing', did: bdid, cid: String(bcid).slice(0, 20) })) |
| 1403 | } |
| 1404 | } |
| 1363 | 1405 | if (!bytes) return json(res, 404, JSON.stringify({ error: 'BlobNotFound' })) |
| 1364 | 1406 | res.writeHead(200, { 'content-type': sniffMime(bytes), 'cache-control': 'public, max-age=31536000, immutable' }) |
| 1365 | 1407 | return res.end(bytes) |