| … | … | @@ -0,0 +1,92 @@ |
| 1 | // OverlayBlobStore — atproto blobs (avatars/images) stored as REAL content-addressed FILES on disk, |
| 2 | // deduped + distributed by hyper-overlay. The drop-in replacement for BlobStore's Hyperbee-of-bytes |
| 3 | // (which stored [len][header][BYTES] as bee values → an append-only balloon, the exact model the |
| 4 | // records redesign fled). Here the BYTES are files (FastCDC dedup, no corestore balloon); the bee |
| 5 | // holds only a TINY per-blob record { mimeType, size, oid } — metadata + the overlay content hash |
| 6 | // (oid) so a peer that has the metadata can fetch the bytes by hash cross-node. |
| 7 | // |
| 8 | // Same public API as BlobStore (drop-in): ready / put(bytes,mime)->{cid,mimeType,size} / |
| 9 | // get(cidStr)->{mimeType,size,bytes}|null / has / close. `cid` is the atproto blob CID |
| 10 | // (cidForBlobSync — raw codec); the overlay `oid` is blake2b (hypercore-crypto data()) — distinct, |
| 11 | // hence the mapping in the bee. |
| 12 | // |
| 13 | // overlay = a HyperOverlay (injected — this module never imports hyper-overlay, so it stays |
| 14 | // dual-runtime like OverlayRepoStore). blobRoot = dir for the canonical blob files. indexStore = a |
| 15 | // Corestore for the tiny metadata bee. |
| 16 | |
| 17 | import Corestore from 'corestore' |
| 18 | import Hyperbee from 'hyperbee' |
| 19 | import b4a from 'b4a' |
| 20 | import fs from '#node:fs' // dual-runtime: bare-fs under Bare (the daemon), builtin fs under Node (tests) |
| 21 | import { join } from '#node:path' |
| 22 | import { cidForBlobSync } from 'atproto-repo/lib/cid.js' |
| 23 | |
| 24 | export class OverlayBlobStore { |
| 25 | constructor (blobRoot, { overlay = null, indexStore, name = 'blob-index' } = {}) { |
| 26 | if (typeof blobRoot !== 'string') throw new Error('OverlayBlobStore: blobRoot dir required') |
| 27 | if (!indexStore) throw new Error('OverlayBlobStore: indexStore (Corestore) required') |
| 28 | this._blobRoot = blobRoot |
| 29 | this._overlay = overlay |
| 30 | this._ownStore = !(indexStore instanceof Corestore) |
| 31 | this._store = this._ownStore ? new Corestore(indexStore) : indexStore |
| 32 | this._name = name |
| 33 | this.bee = null |
| 34 | } |
| 35 | |
| 36 | async ready () { |
| 37 | fs.mkdirSync(this._blobRoot, { recursive: true }) |
| 38 | const core = this._store.get({ name: this._name }) |
| 39 | await core.ready() |
| 40 | // metadata only (tiny JSON) — NOT the blob bytes. root bee, alwaysDuplicate:false is safe here. |
| 41 | this.bee = new Hyperbee(core, { keyEncoding: 'utf-8', valueEncoding: 'binary', alwaysDuplicate: false }) |
| 42 | await this.bee.ready() |
| 43 | return this |
| 44 | } |
| 45 | |
| 46 | _pathFor (cidStr) { return join(this._blobRoot, cidStr) } |
| 47 | |
| 48 | // Store bytes as a content-addressed file + register with the overlay (dedup + servable by oid). |
| 49 | // Idempotent (content-addressed): the same bytes always map to the same cid/file. |
| 50 | async put (bytes, mimeType = 'application/octet-stream') { |
| 51 | const buf = b4a.isBuffer(bytes) ? bytes : b4a.from(bytes) |
| 52 | const cid = cidForBlobSync(buf) |
| 53 | const cidStr = cid.toString() |
| 54 | const path = this._pathFor(cidStr) |
| 55 | fs.writeFileSync(path, buf) |
| 56 | let oid = null |
| 57 | if (this._overlay) { |
| 58 | const r = await this._overlay.registerFile('/blobs/' + cidStr, path, { size: buf.length }) |
| 59 | oid = r ? r.contentHash : null |
| 60 | } |
| 61 | await this.bee.put(cidStr, b4a.from(JSON.stringify({ mimeType, size: buf.length, oid }), 'utf-8')) |
| 62 | return { cid, mimeType, size: buf.length } |
| 63 | } |
| 64 | |
| 65 | // Fetch by CID string; returns { mimeType, size, bytes } or null. Serves the local file if present; |
| 66 | // otherwise pulls the bytes from a connected peer by the overlay oid (content-hash verified). |
| 67 | async get (cidStr) { |
| 68 | const node = await this.bee.get(cidStr) |
| 69 | if (!node) return null |
| 70 | const meta = JSON.parse(b4a.toString(node.value, 'utf-8')) |
| 71 | const path = this._pathFor(cidStr) |
| 72 | let bytes = null |
| 73 | try { |
| 74 | bytes = fs.readFileSync(path) |
| 75 | } catch (e) { |
| 76 | if (!e || e.code !== 'ENOENT') throw e |
| 77 | if (this._overlay && meta.oid) { |
| 78 | const r = await this._overlay.fetchFile(meta.oid, { destPath: path }) |
| 79 | if (r) { try { bytes = fs.readFileSync(path) } catch { bytes = null } } |
| 80 | } |
| 81 | } |
| 82 | if (!bytes) return null |
| 83 | return { mimeType: meta.mimeType, size: meta.size, bytes } |
| 84 | } |
| 85 | |
| 86 | async has (cidStr) { return !!(await this.bee.get(cidStr)) } |
| 87 | |
| 88 | async close () { |
| 89 | try { if (this.bee) await this.bee.close() } catch {} |
| 90 | if (this._ownStore) { try { await this._store.close() } catch {} } |
| 91 | } |
| 92 | } |