// RecordStore — the bytes half of a feed or a document.
//
// One markdown file per record at records/<collection>/<name>.md. Overwrite in place on edit,
// unlink on delete, so on-disk size is O(current records) rather than O(operations ever). An
// optional hyper-overlay indexes and replicates the files by content hash (`oid`); without one this
// is a fully-functional local tree, which is what the tests use.
//
// This layer stores BYTES and nothing else. It has no log, no convergence, no notion of who wrote
// what and no tombstones — a delete here only drops bytes, because absence is ambiguous in a
// replicated system and the positive evidence belongs in the log above.
//
// `name` is deliberately not `rkey` and not `position`: the address question is open
// (hyper-social #61 §12 #1), and this layer must not prejudge it. Any single path segment works.
import b4a from 'b4a'
import fs from '#node:fs'
import { join, relative, dirname, isAbsolute } from '#node:path'
import { parse, serialize } from 'favela-markdown/lib/record.js'
const EXT = '.md'
// Byte order, so derivation over the record set is order-stable regardless of readdir order.
const byteOrder = (a, b) => b4a.compare(b4a.from(a), b4a.from(b))
// A collection or a name is one path segment. Reject anything that could escape records/.
const isSafeSeg = (s) => typeof s === 'string' && s.length > 0 && !s.includes('/') && s !== '.' && s !== '..'
export class RecordStore {
constructor (root, opts = {}) {
if (typeof root !== 'string') throw new Error('RecordStore: root dir path required')
this._root = root
this._recordsDir = join(root, 'records')
// The storage root this store sits under. ⛔ The overlay path is DERIVED from the disk path
// relative to it, rather than spelled out — the on-disk layout is `<base>/records/<feed>/
// records/<collection>/<name>.md`, with `records` doubled, and any hand-written path template
// is one refactor away from silently not resolving.
this._base = opts.base || dirname(dirname(root))
this._overlay = opts.overlay || null
// ⛔ A byte fetch is bounded. `ensureLocal` is awaited once per record by `log.js`'s scan, so an
// overlay call that never answers blocked every listing of a feed whose bytes nobody is
// serving — `timeline`, `search` and `posts-by` alike. There is already a name for that state:
// UNRESOLVED, "we hold the pointer and cannot read the bytes". A fetch that has not come back
// IS that, and saying so is the honest answer rather than waiting for one.
//
// ⚠ Generous rather than tight. A slow peer is not an absent one, and the cost of being wrong
// is a record that reads as unresolved for one pass and resolves on the next.
this._fetchTimeoutMs = opts.fetchTimeoutMs == null ? 5000 : opts.fetchTimeoutMs
}
_pathFor (collection, name) {
if (!isSafeSeg(collection) || !isSafeSeg(name)) {
throw new Error(`RecordStore: unsafe record key "${collection}/${name}"`)
}
return join(this._recordsDir, collection, name + EXT)
}
/**
* Where this record lives in the overlay's namespace.
*
* ⛔ It names the FEED. Every feed has its own RecordStore under `<storage>/records/<feed>/`, and
* this used to be `/records/<collection>/<name>` with no feed in it — so two feeds holding
* `post/p1` registered the SAME overlay path, the second overwriting the first's disk mapping.
*
* ⛔ And, worse, the path could not be resolved BACK to a file, which is what
* `seedFilePathsFromIndex` needs at boot to rebuild the serve map. The invariant this establishes
* is exactly that: `join(storage, overlayPath)` IS the file on disk, the same way `/blobs/<oid>`
* and `/avatars/<oid>` already were.
*/
_overlayPath (collection, name) { return '/' + relative(this._base, this._pathFor(collection, name)) }
// Write the record and return { oid, size }. `oid` is the overlay content hash a peer fetches the
// bytes by; null when there is no overlay. The serialized form is canonical
// (serialize(parse(x)) === x is a law of the codec), so the same record always yields the same oid.
async put (collection, name, record) {
const bytes = b4a.from(serialize(record), 'utf8')
const p = this._pathFor(collection, name)
fs.mkdirSync(join(this._recordsDir, collection), { recursive: true })
fs.writeFileSync(p, bytes)
let oid = null
if (this._overlay) {
try {
const r = await this._overlay.registerFile(this._overlayPath(collection, name), p, { size: bytes.length })
oid = r ? r.contentHash : null
} catch {}
}
return { oid, size: bytes.length }
}
// The stored bytes plus the parsed record. The bytes are what `put` wrote — never a re-encode.
async getRaw (collection, name) {
let bytes
try { bytes = fs.readFileSync(this._pathFor(collection, name)) } catch (e) {
if (e && e.code === 'ENOENT') return null
throw e
}
return { bytes, record: parse(b4a.toString(bytes, 'utf8')) }
}
async get (collection, name) {
const raw = await this.getRaw(collection, name)
return raw ? raw.record : null
}
// The content hash the file on disk CURRENTLY has, or null. Reads only — safe against a live store.
async localOid (collection, name) {
if (!this._overlay || typeof this._overlay.hashFile !== 'function') return null
let p
try { p = this._pathFor(collection, name) } catch { return null }
const r = await this._overlay.hashFile(p)
return r ? { oid: r.contentHash, size: r.size } : null
}
// Resolve a record whose pointer we hold, fetching the bytes by oid if we do not have them.
//
// ⛔ A local hit is VERIFIED against `oid` before it is returned. Skipping that check is what let
// two co-owners hold identical pointer sets, resolve them both happily, and publish different
// content. A superseded copy is never returned: stalling loudly beats advancing with content
// nobody agreed on.
//
// ⛔⛔ BUT A MISMATCH MUST FETCH, and for a long time it did not — it returned null right there,
// and the fetch below was reachable only when there was no local file at all. Posts never hit it,
// because a post is written under a fresh name and has nothing to be stale. The PROFILE is the
// one record in the model that is MUTABLE and overwritten in place at the same (collection,
// name) — so the moment its author edited it, every holder's copy went stale, the pointer named
// bytes they would never go and get, and the record read as unresolved FOR EVER. Not "until the
// next pass": nothing in the system would ever fetch it. That is why a profile change had never
// once reached a follower, and why the symptom was a profile that went BLANK rather than one that
// failed to update.
//
// "Do not return the stale copy" and "do not go and get the right one" are different rules, and
// only the first was ever intended.
async ensureLocal (collection, name, oid) {
const local = await this.get(collection, name)
if (local != null) {
if (!oid || !this._overlay || typeof this._overlay.hashFile !== 'function') return local
const h = await this.localOid(collection, name)
// Only a POSITIVE mismatch is staleness. If the bytes cannot be hashed we cannot judge them,
// and inventing a verdict would re-resolve every record on every pass.
if (!h || h.oid === oid) return local
// Stale — fall through to the fetch. The superseded bytes stay addressable by their own oid
// in the overlay; what is replaced here is only this name's view of which version is current.
}
if (!this._overlay || !oid) return null
const disk = this._pathFor(collection, name)
try {
fs.mkdirSync(join(this._recordsDir, collection), { recursive: true })
// ⚠ Raced, not cancelled — the overlay has no cancel and the fetch may still land, which is
// fine: the bytes end up on disk and the NEXT pass reads them locally for free.
const r = await Promise.race([
this._overlay.fetchFile(oid, { destPath: disk }),
new Promise((resolve) => setTimeout(() => resolve(null), this._fetchTimeoutMs))
])
if (!r) return null
} catch { return null }
// ⛔ VERIFY WHAT LANDED, because "a fetch returned" is not "the agreed bytes are on disk". Two
// ways it can be false: a peer answers with content that does not hash to what was asked for,
// and — the one that actually shipped, in files.js and avatars.js (0.0.46) — the fetch lands at
// its own `destPath` while the caller re-reads its own path and quietly returns the OLD bytes.
// Without this check, a stale record would refuse to resolve and then, one line later, resolve
// to exactly the stale content it just refused.
if (typeof this._overlay.hashFile === 'function') {
const after = await this.localOid(collection, name)
if (after && after.oid !== oid) return null
}
return this.get(collection, name)
}
/**
* Re-register every record file this store holds on disk.
*
* ⛔ The repair the boot seed cannot do. The seed rebuilds the serve map from INDEX entries, and
* an entry written under an older overlay-path shape resolves to a file that was never there — so
* it is skipped, correctly, and stays unservable for ever. Nothing else rewrites it: `put()` only
* registers what it writes, and these records were written long ago.
*
* ⚠ This HASHES each file, which the seed deliberately does not. That is the cold-start cost the
* overlay's own `reindexOverlay` is kept off the boot path to avoid, so the caller runs this
* without waiting on it.
*/
async reregisterLocal () {
if (!this._overlay || typeof this._overlay.registerFile !== 'function') return 0
let n = 0
let collections = []
try { collections = fs.readdirSync(this._recordsDir) } catch { return 0 }
for (const collection of collections) {
let names = []
try { names = fs.readdirSync(join(this._recordsDir, collection)) } catch { continue }
for (const file of names) {
if (!file.endsWith(EXT)) continue
const name = file.slice(0, -EXT.length)
try {
const p = this._pathFor(collection, name)
const size = fs.statSync(p).size
await this._overlay.registerFile(this._overlayPath(collection, name), p, { size })
n++
} catch { /* one unreadable file must not stop the rest */ }
}
}
return n
}
// Drop the BYTES. Not a deletion of the record — see the header.
async del (collection, name) {
try { fs.unlinkSync(this._pathFor(collection, name)) } catch (e) {
if (!e || e.code !== 'ENOENT') throw e
}
if (this._overlay) { try { await this._overlay.unregisterFile(this._overlayPath(collection, name)) } catch {} }
try { fs.rmdirSync(join(this._recordsDir, collection)) } catch {} // best-effort: drop an empty dir
}
// { name, record } for one collection, in byte order.
async * list (collection) {
const dir = join(this._recordsDir, collection)
let names
try { names = fs.readdirSync(dir) } catch (e) {
if (e && e.code === 'ENOENT') return
throw e
}
for (const n of names.filter((x) => x.endsWith(EXT)).sort(byteOrder)) {
const bytes = fs.readFileSync(join(dir, n))
yield { name: n.slice(0, -EXT.length), record: parse(b4a.toString(bytes, 'utf8')) }
}
}
// Every record as { key, bytes, record } in byte order across collections.
async all () {
const out = []
let collections
try { collections = fs.readdirSync(this._recordsDir) } catch (e) {
if (e && e.code === 'ENOENT') return out
throw e
}
for (const collection of collections) {
const dir = join(this._recordsDir, collection)
let st
try { st = fs.statSync(dir) } catch { continue }
if (!st.isDirectory()) continue
for (const n of fs.readdirSync(dir)) {
if (!n.endsWith(EXT)) continue
const bytes = fs.readFileSync(join(dir, n))
out.push({
key: `${collection}/${n.slice(0, -EXT.length)}`,
bytes,
record: parse(b4a.toString(bytes, 'utf8'))
})
}
}
out.sort((a, b) => byteOrder(a.key, b.key))
return out
}
}