const b4a = require('b4a')
const c = require('compact-encoding')
const crypto = require('hypercore-crypto')
const Hypercore = require('hypercore')
const messages = require('./messages.js')
const DEFAULT_MANIFEST_VERSION = 1
const INDEX_VERSION = 1
const MAX_TRACE_PER_VIEW = 256
const EMPTY = b4a.alloc(0)
const [NS_SIGNER_NAMESPACE] = crypto.namespace('autobase', 1)
class ViewCore {
constructor(name, core, base) {
this.name = name
this.core = null
this.batch = null
this.atomicBatch = null
this.moveTo = null
this.tracer = new Tracer()
this.shared = 0
this.ontruncate = (length) => {
this.shared = Math.min(this.shared, length)
}
this.migrated(base, core)
}
getCore() {
return this.atomicBatch || this.batch || this.core
}
migrated(base, core) {
// TODO: close old core if present, for now we just close when the autobase is closed indirectly
// atm its unsafe to do as moveTo has a bug due to a missing read lock in hc
this.core = core
if (this.name === '_system') {
const ff = base._queueFastForward.bind(base)
this.core.on('append', ff)
this.core.ready().then(ff, ff)
}
}
async matchesKey(key) {
if (!this.core.opened) await this.core.ready()
return b4a.equals(this.core.key, key)
}
async matchesNamespace(target) {
if (!this.core.opened) await this.core.ready()
if (this.core.manifest && this.core.manifest.signers.length > 0) {
const ns = this.core.manifest.signers[0].namespace
if (b4a.equals(ns, target)) return true
}
return false
}
getNextKey() {
return this.moveTo ? this.moveTo[0].key : null
}
async commit(atom, length, signature) {
// TODO: is this how its supposed be done atomic wise?
await this.core.commit(this._getAtomicBatch(atom), { length, signature })
}
async release() {
if (!this.atomicBatch) return
await this.atomicBatch.ready()
while (this.atomicBatch.state.sessions.length) {
await this.atomicBatch.state.sessions[0].close()
}
await this.atomicBatch.close()
this.atomicBatch = null
}
_getAtomicBatch(atom) {
if (this.atomicBatch === null) {
this.atomicBatch = this.batch.session({ atom, writable: true })
this.atomicBatch.on('truncate', this.ontruncate)
}
return this.atomicBatch
}
async catchup(atom, length) {
await this.release()
const batch = this._getAtomicBatch(atom)
await batch.ready()
await batch.state.catchup(length)
}
createSession(atom, valueEncoding) {
if (this.batch === null) {
this.batch = this.core.session({ name: 'batch', writable: true })
this.batch.on('truncate', this.ontruncate)
}
const s = atom
? this._getAtomicBatch(atom).session({
valueEncoding,
writable: true,
onseq: this.tracer.onseq.bind(this.tracer)
})
: this.batch.session({ valueEncoding, writable: false })
return s
}
}
class Tracer {
constructor() {
this.enabled = false
this.seen = new Set()
}
start() {
this.enabled = true
this.seen.clear()
}
end() {
this.enabled = false
return [...this.seen.values()]
}
onseq(seq, core) {
if (!this.enabled) return
if (this.seen.size >= MAX_TRACE_PER_VIEW) return
if (seq >= core.signedLength) return
this.seen.add(seq)
}
}
class AutoStore {
constructor(base, byName) {
this.base = base
this.store = base.store
this.byName = byName || new Map()
this.opened = []
this.atom = null
this.local = null
}
async close() {
if (this.local) await this.local.close()
for (const v of this.byName.values()) {
await v.release()
}
if (this.atom) return
for (const v of this.byName.values()) {
if (v.core) await v.core.close()
if (v.batch) await v.batch.close()
}
}
async export() {
const views = []
for (const [name, value] of this.byName) {
const core = await this.store.storage.export(value.core.discoveryKey, { batches: true })
views.push({ name, core })
}
return views
}
atomize() {
const store = new AutoStore(this.base, this.byName)
store.atom = this.store.storage.createAtom()
return store
}
async updateLocal() {
if (!this.local) return
await this.local.close()
this.local = null
}
flush() {
return this.atom ? this.atom.flush() : Promise.resolve()
}
get(opts, moreOpts) {
if (typeof opts === 'string') opts = { name: opts }
if (moreOpts) opts = { ...opts, ...moreOpts }
const { name, valueEncoding = null } = opts
if (!name) throw new Error('name is required')
return this.getViewByName(name).createSession(this.atom, valueEncoding)
}
getViewByName(name) {
let view = this.byName.get(name)
if (!view) {
const preload = this._preload(name)
const core = this.base.store.get({ preload })
view = new ViewCore(name, core, this.base)
this.byName.set(name, view)
}
if (this.opened.indexOf(view) === -1) {
this.opened.push(view)
}
return view
}
getLocal() {
if (this.local !== null) return this.local
this.local = this.base.local.session({
atom: this.atom,
valueEncoding: messages.OplogMessage,
encryption: this.base.getWriterEncryption(this.base.local.key),
active: false
})
return this.local
}
getViews() {
return [...this.byName.values()]
}
getViewCount() {
return this.byName.size
}
getSystemView() {
return this.getViewByName('_system')
}
getSystemCore() {
return this.getSystemView().core
}
getEncryption(name) {
if (!this.base.encryptionKey) return null
if (name === '_encryption') return null
return this.base.encryption.getViewEncryption(name)
}
async getIndexerManifests(entries) {
const manifests = []
for (const { key } of entries) {
const core = this.store.get({ key, active: false })
await core.ready()
if (!core.manifest) continue // danger but only for bootstrapping...
manifests.push(core.manifest)
await core.close()
}
return manifests
}
getViewCore(indexerManifests, name, prologue, manifestVersion, entropy, linked, manifestData) {
const manifest = this._createManifest(
indexerManifests,
name,
prologue,
manifestVersion,
entropy,
linked,
manifestData
)
return this.base.store.get({
manifest,
exclusive: false,
encryption: this.getEncryption(name)
})
}
listViews() {
const views = []
for (const [name, ref] of this.byName) {
const core = ref.batch || ref.core
views.push({ name, key: core.key, length: core.length, signedLength: core.signedLength })
}
return views
}
createAnchorCore(prologue, manifestData) {
const manifest = {
version: 2,
hash: 'blake2b',
prologue,
allowPatch: false,
quorum: 0,
signers: [],
userData: manifestData,
linked: null
}
const core = this.store.get({
manifest,
active: false
})
return core
}
async createView(
indexerManifests,
name,
prologue,
manifestVersion,
entropy,
linked,
manifestData
) {
const manifest = this._createManifest(
indexerManifests,
name,
prologue,
manifestVersion,
entropy,
linked,
manifestData
)
const core = this.store.get({
key: Hypercore.key(manifest),
manifest,
active: false
})
await core.ready()
const key = core.key
await core.close()
return key
}
async _preload(name) {
await Promise.resolve()
if (!this.base.opening) throw new Error('Autobase failed to open')
await this.base._preopen
const boot = (await this.base._getSystemInfo()) || {
key: this.getBootstrapSystemKey(),
indexers: [],
views: [],
entropy: null
}
const indexerManifests = await this.getIndexerManifests(boot.indexers)
// no system, everything is fresh
if (boot.indexers.length === 0) {
return this._freshCorePreload(indexerManifests, name, boot.entropy)
}
// asking for the system, just return it, easy
if (name === '_system') {
return {
key: boot.key,
exclusive: false,
encryption: this.getEncryption(name)
}
}
if (name === '_encryption') {
const core = this.getSystemCore()
await core.ready()
if (!core.manifest.linked) {
return this._freshCorePreload(indexerManifests, name, boot.entropy)
}
return {
key: core.manifest.linked[0],
exclusive: false,
encryption: null
}
}
// infer which view
const v = await this.findViewByName(indexerManifests, boot.views, name, boot.entropy)
// new view, fresh
if (v === null) {
return this._freshCorePreload(indexerManifests, name, boot.entropy)
}
return {
key: v.key,
exclusive: false,
encryption: this.getEncryption(name)
}
}
_freshCorePreload(indexerManifests, name, entropy) {
if (name === '_system') return this._freshSystemPreload(indexerManifests, entropy)
return {
manifest: this._createManifest(
indexerManifests,
name,
null,
DEFAULT_MANIFEST_VERSION,
entropy,
[],
null
),
exclusive: true,
encryption: this.getEncryption(name)
}
}
_freshSystemPreload(indexerManifests, entropy) {
return {
manifest: this._createSystemManifest(
indexerManifests,
null,
DEFAULT_MANIFEST_VERSION,
entropy,
null
),
exclusive: true,
encryption: this.getEncryption('_system')
}
}
_deriveStaticHash(name) {
// key doesnt matter...
return crypto.hash([this.base.key, b4a.from(name)])
}
_deriveNamespace(name, entropy) {
const encryptionId = crypto.hash(this.base.encryptionKey || EMPTY)
const version = c.encode(c.uint, INDEX_VERSION)
const bootstrap = this.base.key
return crypto.hash([
NS_SIGNER_NAMESPACE,
version,
bootstrap,
encryptionId,
entropy,
b4a.from(name)
])
}
async _getCoreManifest(key) {
const core = this.store.get({ key, active: false })
await core.ready()
const manifest = core.manifest
await core.close()
return manifest
}
getBootstrapSystemKey() {
return Hypercore.key(this._createSystemManifest([], null, DEFAULT_MANIFEST_VERSION, null))
}
async findViewByKey(key, indexers, manifestVersion, entropy) {
for (const v of this.byName.values()) {
if (await v.matchesKey(key)) return v
}
const manifest = await this._getCoreManifest(key)
const target = manifest && manifest.signers.length ? manifest.signers[0].namespace : null
if (target) {
for (const v of this.byName.values()) {
if (await v.matchesNamespace(target)) return v
}
}
const indexerManifests = await this.getIndexerManifests(indexers)
if (entropy === null) {
entropy = indexerManifests[0].signers[0].namespace
}
if (target) {
for (const v of this.byName.values()) {
const namespace = this._deriveNamespace(v.name, entropy)
if (b4a.equals(namespace, target)) return v
}
}
// prob the empty prologue, we dont have manifest for those in FF if len=0
for (const v of this.byName.values()) {
const manifestData = v.core ? v.core.manifest.userData : null
const manifest = this._createManifest(
indexerManifests,
v.name,
null,
manifestVersion,
entropy,
[],
manifestData
)
const manifestKey = Hypercore.key(manifest)
if (!b4a.equals(manifestKey, key)) continue
// we didnt have the core! this is because it is empty, ensure its on disk
const core = this.store.get({ key: manifestKey, manifest, active: false })
await core.ready()
await core.close()
return v
}
return null
}
async findViewByName(indexerManifests, views, name, entropy) {
if (indexerManifests.length === 0) return null
if (entropy === null) entropy = indexerManifests[0].signers[0].namespace
const namespace = this._deriveNamespace(name, entropy)
for (const v of views) {
const manifest = await this._getCoreManifest(v.key)
if (manifest.signers.length === 0) continue
const signer = manifest.signers[0]
if (b4a.equals(signer.namespace, namespace)) return v
}
return null
}
_createManifest(indexerManifests, name, prologue, version, entropy, linked, manifestData) {
if (!indexerManifests.length) {
prologue = {
hash: this._deriveStaticHash(name),
length: 0
}
} else if (prologue && prologue.length === 0) {
// just in case
prologue = null
}
const signers = []
if (indexerManifests.length && entropy === null) {
entropy = indexerManifests[0].signers[0].namespace
}
for (const manifest of indexerManifests) {
const signer = manifest.signers[0]
signers.push({
namespace: this._deriveNamespace(name, entropy),
signature: 'ed25519',
publicKey: signer.publicKey
})
}
return {
version,
hash: 'blake2b',
prologue,
allowPatch: true,
quorum: Math.min(signers.length, (signers.length >> 1) + 1),
signers,
userData: manifestData,
linked: version > 1 ? linked : null
}
}
_createSystemManifest(indexerManifests, prologue, version, entropy, manifestData) {
if (prologue !== null) throw new Error('Can only derive fresh system core')
const linked = []
if (version > DEFAULT_MANIFEST_VERSION) {
const encManifest = this._createManifest(
indexerManifests,
'_encryption',
null,
version,
entropy,
[],
manifestData
)
linked.push(Hypercore.key(encManifest))
}
return this._createManifest(
indexerManifests,
'_system',
null,
version,
entropy,
linked,
manifestData
)
}
}
module.exports = AutoStore