hyper-note / scripts / compact-autobases.js

#!/usr/bin/env node

/**
 * Compact hypernote Autobases — remove duplicate entries from sync loop bloat.
 *
 * Approach:
 * 1. Open each SharedNotebook and SharedRegistry Autobase
 * 2. Read the current view state (already deduplicated by Autobase linearization)
 * 3. Check how many entries the local writer has vs unique entries in view
 * 4. Report what can be compacted
 *
 * The actual compaction: delete the multiwriter config from the registry,
 * which causes the server to create fresh Autobases on next start.
 * The event-driven sync re-populates from local Hyperdrives (notes) and
 * identity registry (users).
 *
 * Usage:
 *   node scripts/compact-autobases.js <storage-path> [--apply]
 */

import Corestore from 'corestore'
import Hyperbee from 'hyperbee'
import Autobase from 'autobase'

const args = process.argv.slice(2)
const storagePath = args[0] || process.env.HOME + '/.config/hypernote/server-data'
const apply = args.includes('--apply')

console.log('=== Autobase Compaction ===')
console.log('Storage:', storagePath)
console.log('Mode:', apply ? 'APPLY (will modify data)' : 'DRY RUN (read-only)')
console.log('')

const store = new Corestore(storagePath)
await store.ready()

// Open the main registry to find multiwriter config
const registryCore = store.get({ name: 'registry' })
const registry = new Hyperbee(registryCore, { keyEncoding: 'utf-8', valueEncoding: 'json' })
await registry.ready()

const mwConfig = await registry.get('multiwriter-config')
if (!mwConfig || !mwConfig.value) {
  console.log('No multiwriter config found — nothing to compact.')
  await store.close()
  process.exit(0)
}

const bootstrapKey = mwConfig.value.registryBootstrapKey
console.log('Multiwriter config:', JSON.stringify(mwConfig.value, null, 2))
console.log('')

// Open SharedRegistry to inspect its state
const regStore = store.namespace('shared-registry')
const regBase = new Autobase(regStore, Buffer.from(bootstrapKey, 'hex'), {
  open: (store) => new Hyperbee(store.get('view'), {
    keyEncoding: 'utf-8',
    valueEncoding: 'json'
  }),
  apply: async (nodes, view, base) => {
    for (const node of nodes) {
      const op = node.value
      if (!op || !op.key) continue
      await view.put(op.key, op.value)
    }
  },
  valueEncoding: 'json'
})
await regBase.ready()
await regBase.update()

// Count view entries
let viewEntries = 0
let notebooks = []
for await (const entry of regBase.view.createReadStream()) {
  viewEntries++
  if (entry.key.startsWith('notebook:')) {
    const nb = entry.value
    if (nb && nb.slug) notebooks.push({ slug: nb.slug, autobaseKey: nb.autobaseKey, shared: nb.shared })
  }
}

const localWriterLen = regBase.local ? regBase.local.length : 0
console.log('SharedRegistry:')
console.log('  View entries (unique):', viewEntries)
console.log('  Local writer blocks:', localWriterLen)
console.log('  Bloat ratio:', localWriterLen > 0 ? (localWriterLen / Math.max(viewEntries, 1)).toFixed(1) + 'x' : 'N/A')
console.log('  Notebooks:', notebooks.map(n => n.slug).join(', '))
console.log('')

// Inspect each SharedNotebook
for (const nb of notebooks) {
  if (!nb.shared || !nb.autobaseKey) continue

  const nbStore = store.namespace('shared-notebook-' + nb.slug)
  try {
    const nbBase = new Autobase(nbStore, Buffer.from(nb.autobaseKey, 'hex'), {
      open: (store) => new Hyperbee(store.get('view'), {
        keyEncoding: 'utf-8',
        valueEncoding: 'json'
      }),
      apply: async (nodes, view, base) => {
        for (const node of nodes) {
          const op = node.value
          if (!op || !op.key) continue
          await view.put(op.key, op.value)
        }
      },
      valueEncoding: 'json'
    })
    await nbBase.ready()
    await nbBase.update()

    let nbViewEntries = 0
    for await (const entry of nbBase.view.createReadStream()) {
      nbViewEntries++
    }

    const nbLocalLen = nbBase.local ? nbBase.local.length : 0
    console.log(`SharedNotebook [${nb.slug}]:`)
    console.log('  View entries (unique):', nbViewEntries)
    console.log('  Local writer blocks:', nbLocalLen)
    console.log('  Bloat ratio:', nbLocalLen > 0 ? (nbLocalLen / Math.max(nbViewEntries, 1)).toFixed(1) + 'x' : 'N/A')

    await nbBase.close()
  } catch (err) {
    console.log(`SharedNotebook [${nb.slug}]: ERROR — ${err.message}`)
  }
}

console.log('')

if (!apply) {
  console.log('=== Dry run complete. Use --apply to compact. ===')
  console.log('')
  console.log('Compaction will:')
  console.log('  1. Delete multiwriter-config from registry')
  console.log('  2. Server creates fresh Autobases on next start')
  console.log('  3. Event-driven sync re-populates from local data')
  console.log('  4. Old Autobase cores become orphaned (reclaimable later)')
  await regBase.close()
  await store.close()
  process.exit(0)
}

// APPLY compaction
console.log('=== Applying compaction ===')
console.log('Deleting multiwriter-config from registry...')
await registry.del('multiwriter-config')
console.log('Done. Multiwriter config removed.')
console.log('')
console.log('On next server start:')
console.log('  - Fresh Autobases will be created')
console.log('  - Mirror sync will re-populate from local Hyperdrives')
console.log('  - Memory usage will drop significantly')
console.log('')
console.log('Old Autobase cores are still on disk but unused.')
console.log('Run hyper-inspector to verify storage savings after restart.')

await regBase.close()
await store.close()