/**
* OverlayManager — unified-FileIndex sidecar wrapper.
*
* One FileIndex (corestore namespace `overlay-files-unified`) holds entries
* for every repo's overlay-tracked files, keyed by `<reponame>/<relPath>`.
* A second Hyperbee (corestore namespace `overlay-repo-config`) holds the
* per-repo working-tree config:
* `repo:<name>` → { repoRoot, addedAt }
*
* Why unified — see issue #14. The sidecar's swarm/writer scope is already
* sidecar-wide, so per-repo FileIndexes don't deliver selective replication;
* unified gives drop-in compatibility with hyper-overlay's OverlayProtocolV2
* (which assumes one FileIndex), enables natural cross-repo chunk dedup,
* and is operationally cheaper (1 core + 1 config core vs N).
*
* External API takes `(reponame, relPath)` and is unchanged from previous
* versions — internals translate to overlay keys.
*
* Hash form for verification (`readFileByOid`) is `crypto.data(buf)`,
* the LEAF_TYPE-prefixed digest matching `hashChunk(file)` and
* `createStreamingHasher({ size })`. The pgh CLI's clean filter produces
* the same digest, so register-then-verify round-trips cleanly.
*
* Used by:
* - sidecar/lib/index.js (instantiates one + wires UDS RPCs)
* - sidecar/lib/overlay-protocol.js (calls readFileByOid for cross-peer
* fetch — slated for replacement by OverlayProtocolV2 in #14d)
*/
import fs from '#node:fs'
import path from '#node:path'
import crypto from 'hypercore-crypto'
import Hyperbee from 'hyperbee'
import { FileIndex } from 'hyper-overlay/file-index'
const REPO_PREFIX = 'repo:'
export class OverlayManager {
constructor (corestore, opts = {}) {
this._corestore = corestore
this._opts = opts
this._fileIndex = null
this._configBee = null
this._repoCache = new Map() // reponame → repoRoot (read-through)
this._opening = null
// Shared map: overlayPath ('<repo>/<relPath>') → absolute disk path.
// OverlayProtocolV2 reads this directly via opts.filePaths to satisfy
// file-request handlers without scanning baseDirs. Populated lazily
// on putFile and pre-seeded on attachSyncEngine().
this._filePaths = opts.filePaths || new Map()
// Optional SyncEngine reference. When present, putFile/deleteFile
// also call logPut/logDel so the engine's feed grows in lockstep
// with the FileIndex — required for OverlayProtocolV2 to broadcast
// local files via fileOffer on peer connection.
this._syncEngine = null
}
/** FileIndex accessor — opens it lazily. Used by sidecar to wire
* SyncEngine / TransferManager against the same underlying index. */
async getFileIndex () {
await this._ensureReady()
return this._fileIndex
}
/** Shared overlayPath→diskPath map. Pass to OverlayProtocolV2's
* opts.filePaths so the protocol sees live additions. */
get filePaths () { return this._filePaths }
/**
* Wire a SyncEngine to this manager. Subsequent putFile/deleteFile
* calls also write to its feed. Existing entries are NOT replayed —
* call `replayLogToFeed()` once if a fresh SyncEngine needs to catch
* up with a pre-populated FileIndex.
*/
attachSyncEngine (syncEngine) {
this._syncEngine = syncEngine
}
/**
* Pre-seed `filePaths` from existing FileIndex entries. Called once on
* sidecar startup after both OverlayManager and the protocol stack are
* instantiated, so peer connections immediately know where to find
* already-registered files.
*/
async populateFilePaths () {
await this._ensureReady()
const all = await this._fileIndex.listFiles()
for (const f of all) {
const split = this._splitOverlayPath(f.path)
if (!split) continue
const root = await this._getRepoRoot(split.reponame)
if (root) this._filePaths.set(f.path, path.join(root, split.path))
}
}
/**
* Resolve an overlay-path (`<reponame>/<relPath>`) to an absolute disk
* path using the per-repo `repoRoot`. Returns null if the repo isn't
* registered. Async — opens the manager if needed.
*/
async getDiskPath (overlayPath) {
await this._ensureReady()
const split = this._splitOverlayPath(overlayPath)
if (!split) return null
const root = await this._getRepoRoot(split.reponame)
if (!root) return null
return path.join(root, split.path)
}
/**
* Synchronous variant of getDiskPath using only the in-memory repoRoot
* cache. Returns null if the manager isn't open or the repo isn't in
* the cache. Use after `warmRepoCache()` so the cache is hot.
*
* Used by `MultiRepoOverlayProtocolV2._resolveFilePath` — the upstream
* call site is sync (called from `_onFileOffer` / `_offerLocalFiles`),
* so we trade off completeness (cold-cache misses skip the offer) for
* keeping the override synchronous.
*/
getDiskPathSync (overlayPath) {
const split = this._splitOverlayPath(overlayPath)
if (!split) return null
const root = this._repoCache.get(split.reponame)
if (!root) return null
return path.join(root, split.path)
}
/**
* Warm the in-memory repoRoot cache by iterating the config bee.
* Called once at startup so `getDiskPathSync` can resolve every
* registered repo without falling back to a bee read.
*/
async warmRepoCache () {
await this._ensureReady()
for await (const node of this._configBee.createReadStream({
gt: REPO_PREFIX,
lt: REPO_PREFIX + '\xff'
})) {
const reponame = node.key.slice(REPO_PREFIX.length)
const root = node.value && node.value.repoRoot
if (root) this._repoCache.set(reponame, root)
}
}
// ── Lazy open ───────────────────────────────────────────────
async _ensureReady () {
if (this._fileIndex && this._configBee) return
if (!this._opening) {
this._opening = (async () => {
const fileIndexNs = this._corestore.namespace('overlay-files-unified')
this._fileIndex = new FileIndex(fileIndexNs)
await this._fileIndex.ready()
const configNs = this._corestore.namespace('overlay-repo-config')
const configCore = configNs.get({ name: 'config' })
this._configBee = new Hyperbee(configCore, {
keyEncoding: 'utf-8',
valueEncoding: 'json'
})
await this._configBee.ready()
})()
}
await this._opening
}
// ── Repo / overlay-key helpers ──────────────────────────────
/** `(reponame, relPath)` → unified overlay key. */
_overlayPath (reponame, relPath) {
return reponame + '/' + relPath
}
/** Inverse of `_overlayPath` — returns `null` if the key has no separator. */
_splitOverlayPath (overlayPath) {
const slash = overlayPath.indexOf('/')
if (slash < 0) return null
return {
reponame: overlayPath.slice(0, slash),
path: overlayPath.slice(slash + 1)
}
}
async _getRepoRoot (reponame) {
if (this._repoCache.has(reponame)) return this._repoCache.get(reponame)
const v = await this._configBee.get(REPO_PREFIX + reponame)
const root = v && v.value ? v.value.repoRoot : null
if (root) this._repoCache.set(reponame, root)
return root
}
/** Translate FileIndex `{contentHash, ...}` → external `{oid, ...}`. */
_toExternal (entry) {
if (!entry) return null
const { contentHash, ...rest } = entry
return { ...rest, oid: contentHash }
}
// ── Repo lifecycle ──────────────────────────────────────────
/**
* Register or update a repo's working-tree root. Sidecar serves files
* from `<repoRoot>/<relPath>` when peers request them by oid.
*/
async initRepo (reponame, absPath) {
if (typeof reponame !== 'string' || !reponame) throw new Error('reponame required')
if (reponame.includes('/')) throw new Error('reponame must not contain "/"')
if (typeof absPath !== 'string' || !path.isAbsolute(absPath)) {
throw new Error('absPath must be absolute')
}
await this._ensureReady()
await this._configBee.put(REPO_PREFIX + reponame, {
repoRoot: absPath,
addedAt: Date.now()
})
this._repoCache.set(reponame, absPath)
return { reponame, repoRoot: absPath }
}
/**
* List all repos that have been registered (have a repoRoot in the
* config bee). Reads the config bee — does not require any FileIndex
* entry to exist for the repo.
*/
async listRepos () {
await this._ensureReady()
const out = []
for await (const node of this._configBee.createReadStream({
gt: REPO_PREFIX,
lt: REPO_PREFIX + '\xff'
})) {
out.push(node.key.slice(REPO_PREFIX.length))
}
return out
}
async getRepoInfo (reponame) {
await this._ensureReady()
const repoRoot = await this._getRepoRoot(reponame)
return { reponame, repoRoot }
}
// ── File metadata ───────────────────────────────────────────
/**
* Record a file in the unified index. `meta` shape:
* { oid, size, mtime, executable?, linkname?, chunkmap? }
* `chunkmap` is the optional `[{ hash, offset, length }]` array — when
* present, persisted under the FileIndex's `chunkmap:<overlayPath>` key.
*/
async putFile (reponame, relPath, meta) {
if (!relPath) throw new Error('relPath required')
if (!meta || !meta.oid) throw new Error('meta.oid required')
await this._ensureReady()
const overlayPath = this._overlayPath(reponame, relPath)
await this._fileIndex.putFile(overlayPath, {
contentHash: meta.oid,
size: meta.size,
mtime: meta.mtime,
hashing: false,
executable: !!meta.executable,
linkname: meta.linkname || null
})
if (Array.isArray(meta.chunkmap)) {
await this._fileIndex.putChunkMap(overlayPath, meta.chunkmap)
}
// Refresh the shared filePaths cache so OverlayProtocolV2 knows
// where to read this file from on incoming file-request.
const root = await this._getRepoRoot(reponame)
if (root) this._filePaths.set(overlayPath, path.join(root, relPath))
// Append to the SyncEngine feed so peers see this in fileOffer.
if (this._syncEngine) {
try {
await this._syncEngine.logPut(overlayPath, {
contentHash: meta.oid,
size: meta.size,
mtime: meta.mtime
})
} catch (err) {
// Feed append failures are non-fatal for the local FileIndex
// write. Log + continue — protocol-side replication will lag
// until the next put succeeds.
console.error('[overlay-manager] syncEngine.logPut failed:', err.message)
}
}
return { reponame, path: relPath, ...meta }
}
async getFile (reponame, relPath) {
await this._ensureReady()
const overlayPath = this._overlayPath(reponame, relPath)
const entry = await this._fileIndex.getFile(overlayPath)
if (!entry) return null
const out = { path: relPath, ...this._toExternal(entry) }
const cm = await this._fileIndex.getChunkMap(overlayPath)
if (cm) out.chunkmap = cm
return out
}
async deleteFile (reponame, relPath) {
await this._ensureReady()
const overlayPath = this._overlayPath(reponame, relPath)
// FileIndex.delFile clears both file:<key> and chunkmap:<key>.
await this._fileIndex.delFile(overlayPath)
this._filePaths.delete(overlayPath)
if (this._syncEngine) {
try { await this._syncEngine.logDel(overlayPath) }
catch (err) {
console.error('[overlay-manager] syncEngine.logDel failed:', err.message)
}
}
return { reponame, path: relPath }
}
async listFiles (reponame) {
await this._ensureReady()
const prefix = reponame + '/'
const files = await this._fileIndex.listFiles(prefix)
return files.map(f => ({
...this._toExternal(f),
// Strip the reponame prefix so callers see relPath only.
path: f.path.startsWith(prefix) ? f.path.slice(prefix.length) : f.path
}))
}
/**
* Find a file across the entire unified index by oid. Linear scan; if
* the index grows large we can add a content-hash secondary index.
*/
async findByOid (oid) {
await this._ensureReady()
const all = await this._fileIndex.listFiles()
for (const f of all) {
if (f.contentHash !== oid) continue
const split = this._splitOverlayPath(f.path)
if (!split) continue
const repoRoot = await this._getRepoRoot(split.reponame)
return {
reponame: split.reponame,
path: split.path,
...this._toExternal({ ...f, path: split.path }),
repoRoot
}
}
return null
}
// ── Local serving — read bytes for a known oid ──────────────
/**
* Read the bytes of a file by oid. Re-verifies the oid against on-disk
* content using `crypto.data` (LEAF_TYPE-prefixed) — same form the pgh
* CLI's `createStreamingHasher({ size })` produces, so verification
* succeeds for files registered via the clean filter.
*/
async readFileByOid (oid) {
const found = await this.findByOid(oid)
if (!found) return null
if (!found.repoRoot) return null
const abs = path.join(found.repoRoot, found.path)
let data
try {
data = fs.readFileSync(abs)
} catch {
return null
}
const actualOid = crypto.data(data).toString('hex')
if (actualOid !== oid) return null
return data
}
/**
* Helper: compute oid + size + mtime for a working-tree file.
* The pgh CLI's clean filter does its own streaming hash; this is kept
* for the legacy `overlay-hash-file` UDS RPC and any out-of-band callers.
*/
async hashWorkingTreeFile (reponame, relPath) {
await this._ensureReady()
const repoRoot = await this._getRepoRoot(reponame)
if (!repoRoot) throw new Error('repo not initialized: ' + reponame)
const abs = path.join(repoRoot, relPath)
const stat = fs.statSync(abs)
if (!stat.isFile()) throw new Error('not a file: ' + relPath)
const data = fs.readFileSync(abs)
const oid = crypto.data(data).toString('hex')
return {
oid,
size: data.length,
mtime: Math.floor(stat.mtimeMs),
executable: !!(stat.mode & 0o111)
}
}
// ── Aggregate counts ────────────────────────────────────────
async status () {
await this._ensureReady()
const repos = []
for await (const node of this._configBee.createReadStream({
gt: REPO_PREFIX,
lt: REPO_PREFIX + '\xff'
})) {
const reponame = node.key.slice(REPO_PREFIX.length)
const repoRoot = node.value && node.value.repoRoot
const prefix = reponame + '/'
const files = await this._fileIndex.listFiles(prefix)
const fileCount = files.filter(f => f.path.startsWith(prefix)).length
repos.push({ reponame, repoRoot, fileCount })
}
return { repos, totalRepos: repos.length }
}
async close () {
if (this._fileIndex) {
try { await this._fileIndex.close() } catch {}
this._fileIndex = null
}
if (this._configBee) {
try { await this._configBee.close() } catch {}
this._configBee = null
}
this._repoCache.clear()
this._opening = null
}
}