// Deep block reader - parse Hyperbee nodes from raw Hypercore blocks
// and decode Hyperdrive file entries with full blob mapping
import b4a from 'b4a'
// Protobuf varint decoder
function pbVarint (buf, offset) {
let result = 0
let shift = 0
let pos = offset
while (pos < buf.length) {
const byte = buf[pos]
result |= (byte & 0x7f) << shift
pos++
if ((byte & 0x80) === 0) break
shift += 7
}
return { value: result, end: pos }
}
// Protobuf bytes decoder (length-delimited)
function pbBytes (buf, offset) {
const { value: len, end: lenEnd } = pbVarint(buf, offset)
return { value: buf.subarray(lenEnd, lenEnd + len), end: lenEnd + len }
}
// Decode a Hyperbee Node protobuf { index, key, value }
export function decodeNode (buf) {
let offset = 0
const node = { index: null, key: null, value: null }
while (offset < buf.length) {
const { value: prefix, end: tagEnd } = pbVarint(buf, offset)
offset = tagEnd
const tag = prefix >> 3
const wireType = prefix & 7
if (tag === 1 && wireType === 2) {
const { value, end } = pbBytes(buf, offset)
node.index = decodeYoloIndex(value)
offset = end
} else if (tag === 2 && wireType === 2) {
const { value, end } = pbBytes(buf, offset)
node.key = value
offset = end
} else if (tag === 3 && wireType === 2) {
const { value, end } = pbBytes(buf, offset)
node.value = value
offset = end
} else {
// Skip unknown fields
if (wireType === 0) { pbVarint(buf, offset); offset = pbVarint(buf, offset).end }
else if (wireType === 2) { const { end } = pbBytes(buf, offset); offset = end }
else break
}
}
return node
}
// Decode YoloIndex { levels: [{ keys: [seq], children: [seq] }] }
function decodeYoloIndex (buf) {
const levels = []
let offset = 0
while (offset < buf.length) {
const { value: prefix, end: tagEnd } = pbVarint(buf, offset)
offset = tagEnd
const tag = prefix >> 3
if (tag === 1) {
const { value: levelBuf, end } = pbBytes(buf, offset)
offset = end
levels.push(decodeLevel(levelBuf))
} else {
break
}
}
return { levels }
}
function decodeLevel (buf) {
const keys = []
const children = []
let offset = 0
while (offset < buf.length) {
const { value: prefix, end: tagEnd } = pbVarint(buf, offset)
offset = tagEnd
const tag = prefix >> 3
if (tag === 1) {
// packed varints for keys
const { value: packedLen, end: lenEnd } = pbVarint(buf, offset)
offset = lenEnd
const packedEnd = offset + packedLen
while (offset < packedEnd) {
const { value, end } = pbVarint(buf, offset)
keys.push(value)
offset = end
}
} else if (tag === 2) {
// packed varints for children
const { value: packedLen, end: lenEnd } = pbVarint(buf, offset)
offset = lenEnd
const packedEnd = offset + packedLen
while (offset < packedEnd) {
const { value, end } = pbVarint(buf, offset)
children.push(value)
offset = end
}
} else {
break
}
}
return { keys, children }
}
// Decode Hyperbee Header protobuf { protocol: string, metadata?: { contentFeed? } }
export function decodeHeader (buf) {
let offset = 0
const header = { protocol: '', contentFeed: null }
while (offset < buf.length) {
const { value: prefix, end: tagEnd } = pbVarint(buf, offset)
offset = tagEnd
const tag = prefix >> 3
if (tag === 1) {
// protocol string
const { value, end } = pbBytes(buf, offset)
header.protocol = b4a.toString(value, 'utf8')
offset = end
} else if (tag === 2) {
// metadata submessage
const { value: metaBuf, end } = pbBytes(buf, offset)
offset = end
// Parse metadata: field 1 = contentFeed (bytes)
let mOff = 0
while (mOff < metaBuf.length) {
const { value: mPrefix, end: mTagEnd } = pbVarint(metaBuf, mOff)
mOff = mTagEnd
const mTag = mPrefix >> 3
if (mTag === 1) {
const { value, end } = pbBytes(metaBuf, mOff)
header.contentFeed = value
mOff = end
} else {
break
}
}
} else {
break
}
}
return header
}
// Decode a Hyperdrive file entry value
// Supports both JSON (compat/legacy) and compact-encoding formats
export function decodeDriveEntry (buf) {
if (!buf || buf.length === 0) return null
// JSON format starts with { (0x7b)
if (buf[0] === 0x7b) {
try {
const obj = JSON.parse(b4a.toString(buf, 'utf8'))
return {
executable: obj.executable || false,
linkname: obj.linkname || null,
blob: obj.blob || null,
metadata: obj.metadata || null
}
} catch {
return null
}
}
// Compact-encoding format
let pos = 0
const flags = buf[pos++]
const entry = {
executable: !!(flags & 1),
linkname: null,
blob: null,
metadata: null
}
if (flags & 2) {
const { value: len, end: lenEnd } = readCEVarint(buf, pos)
pos = lenEnd
entry.linkname = b4a.toString(buf.subarray(pos, pos + len), 'utf8')
pos += len
}
if (flags & 4) {
const { value: blockOffset, end: e1 } = readCEVarint(buf, pos)
pos = e1
const { value: blockLength, end: e2 } = readCEVarint(buf, pos)
pos = e2
const { value: byteOffset, end: e3 } = readCEVarint(buf, pos)
pos = e3
const { value: byteLength, end: e4 } = readCEVarint(buf, pos)
pos = e4
entry.blob = { blockOffset, blockLength, byteOffset, byteLength }
}
if (flags & 8) {
const { value: len, end: lenEnd } = readCEVarint(buf, pos)
pos = lenEnd
entry.metadata = buf.subarray(pos, pos + len)
pos += len
}
return entry
}
// Compact-encoding varint (unsigned)
function readCEVarint (buf, offset) {
let result = 0
let shift = 0
let pos = offset
while (pos < buf.length) {
const byte = buf[pos]
result |= (byte & 0x7f) << shift
pos++
if ((byte & 0x80) === 0) break
shift += 7
if (shift > 49) break // safety: max 7 bytes
}
return { value: result, end: pos }
}
// Strip SubEncoder prefix from a Hyperbee key
// SubEncoder 'files' adds: [prefix_byte, 0x00] at start and [0x00] at end
export function stripSubEncoder (key) {
if (!key || key.length < 3) return key
// SubEncoder format: prefix + SEP(0x00) + actual_key + SEP(0x00)
// The prefix is the sub-encoder name encoded
// For 'files': it's the string 'files' length-prefixed, then 0x00 separator
// Let's find the first 0x00 after position 0 and strip up to there + the trailing 0x00
const firstSep = key.indexOf(0x00)
if (firstSep < 0) return key
const lastByte = key[key.length - 1]
const end = lastByte === 0x00 ? key.length - 1 : key.length
return key.subarray(firstSep + 1, end)
}
// Walk all Hyperbee blocks in a core and extract key-value entries
export async function walkHyperbeeBlocks (core, opts = {}) {
const { maxBlocks = Infinity, waitForBlocks = false } = opts
const entries = []
let headerBlock = null
const limit = Math.min(core.length, maxBlocks)
for (let i = 0; i < limit; i++) {
try {
const block = await core.get(i, { wait: waitForBlocks })
if (!block) continue
if (i === 0) {
// Block 0 is the Hyperbee header
try {
headerBlock = decodeHeader(block)
} catch {}
continue
}
// All other blocks are Node protobuf messages
try {
const node = decodeNode(block)
if (node.key && node.key.length > 0) {
entries.push({
seq: i,
key: node.key,
value: node.value,
childCount: node.index?.levels?.[0]?.children?.length || 0,
keyCount: node.index?.levels?.[0]?.keys?.length || 0
})
}
} catch {}
} catch {
// Block not available locally
continue
}
}
return { header: headerBlock, entries }
}