#!/usr/bin/env node
/**
* generate_vectors.js — Reference test vector generator
*
* Runs the actual JS hyperdht/hyperswarm code with deterministic inputs
* and captures outputs as JSON. These vectors are the source of truth
* for verifying the C++ implementation.
*
* Usage:
* node generate_vectors.js > vectors.json
* node generate_vectors.js --pretty # human-readable
*
* The C++ verifier (test_verify_vectors) reads vectors.json and checks
* that every C++ function produces identical output.
*
* Philosophy: See memory/verification-first.md
* - The JS code IS the spec
* - Test vectors come from RUNNING the JS, not from reading it
* - Verification = byte-level comparison
*/
const crypto = require('hypercore-crypto')
const sodium = require('sodium-universal')
const c = require('compact-encoding')
const b4a = require('b4a')
const { NS, COMMANDS, FIREWALL } = require('hyperdht/lib/constants')
const m = require('hyperdht/lib/messages')
const pretty = process.argv.includes('--pretty')
// ============================================================================
// Deterministic seed — always produces same keypair
// ============================================================================
const SEED_A = b4a.alloc(32)
SEED_A[0] = 0x01 // seed = 0x01 0x00 ... 0x00
const SEED_B = b4a.alloc(32)
SEED_B[0] = 0x02
const SEED_C = b4a.alloc(32)
SEED_C[0] = 0x03
// ============================================================================
// Section 1: Keypair Generation
// ============================================================================
function generateKeypairVectors () {
const kpA = crypto.keyPair(SEED_A)
const kpB = crypto.keyPair(SEED_B)
const kpRandom = crypto.keyPair() // random, but we capture the output
return {
description: 'Ed25519 keypair generation via crypto_sign_seed_keypair',
tests: [
{
name: 'keypair_from_seed_A',
input: { seed: hex(SEED_A) },
output: {
public_key: hex(kpA.publicKey),
secret_key: hex(kpA.secretKey)
}
},
{
name: 'keypair_from_seed_B',
input: { seed: hex(SEED_B) },
output: {
public_key: hex(kpB.publicKey),
secret_key: hex(kpB.secretKey)
}
},
{
name: 'keypair_sizes',
input: {},
output: {
public_key_size: 32,
secret_key_size: 64
}
}
]
}
}
// ============================================================================
// Section 2: Namespace Derivation
// ============================================================================
function generateNamespaceVectors () {
// This is the exact call from hyperdht/lib/constants.js:42-49
const ns = crypto.namespace('hyperswarm/dht', [
COMMANDS.ANNOUNCE, // 4
COMMANDS.UNANNOUNCE, // 5
COMMANDS.MUTABLE_PUT, // 6
COMMANDS.PEER_HANDSHAKE, // 0
COMMANDS.PEER_HOLEPUNCH // 1
])
// Also test with sequential IDs
const nsSeq = crypto.namespace('hyperswarm/dht', 5) // IDs [0,1,2,3,4]
return {
description: 'Namespace derivation: BLAKE2b(name) then BLAKE2b(hash || id) per ID',
tests: [
{
name: 'hyperdht_namespaces',
input: {
name: 'hyperswarm/dht',
ids: [4, 5, 6, 0, 1]
},
output: {
ns_announce: hex(ns[0]),
ns_unannounce: hex(ns[1]),
ns_mutable_put: hex(ns[2]),
ns_peer_handshake: hex(ns[3]),
ns_peer_holepunch: hex(ns[4])
}
},
{
name: 'hyperdht_namespaces_match_constants',
input: {},
output: {
NS_ANNOUNCE: hex(NS.ANNOUNCE),
NS_UNANNOUNCE: hex(NS.UNANNOUNCE),
NS_MUTABLE_PUT: hex(NS.MUTABLE_PUT),
NS_PEER_HANDSHAKE: hex(NS.PEER_HANDSHAKE),
NS_PEER_HOLEPUNCH: hex(NS.PEER_HOLEPUNCH)
}
},
{
name: 'sequential_ids',
input: {
name: 'hyperswarm/dht',
ids: [0, 1, 2, 3, 4]
},
output: nsSeq.map(buf => hex(buf))
}
]
}
}
// ============================================================================
// Section 3: Token Generation
// ============================================================================
function generateTokenVectors () {
// Token = BLAKE2b(key=secret, data=host_string)
const secret = b4a.alloc(32)
secret[0] = 0xAA
secret[1] = 0xBB
const hosts = ['127.0.0.1', '192.168.1.100', '10.0.0.1']
const tokens = hosts.map(host => {
const token = b4a.allocUnsafe(32)
sodium.crypto_generichash(token, b4a.from(host), secret)
return { host, token: hex(token) }
})
return {
description: 'Token generation: BLAKE2b(key=secret, data=host_string)',
tests: [
{
name: 'token_generation',
input: { secret: hex(secret) },
output: tokens
}
]
}
}
// ============================================================================
// Section 4: Compact Encoding — Varint
// ============================================================================
function generateVarintVectors () {
const values = [0, 1, 127, 252, 253, 254, 255, 256, 1000, 65535, 65536, 100000, 0xFFFFFFFF]
const tests = values.map(v => {
const state = c.state()
c.uint.preencode(state, v)
state.buffer = b4a.allocUnsafe(state.end)
state.start = 0
c.uint.encode(state, v)
return {
name: `varint_${v}`,
input: { value: v },
output: { encoded: hex(state.buffer), byte_length: state.buffer.length }
}
})
return {
description: 'Compact encoding varint: 0-252=1B, 253-65535=3B(0xfd+LE16), 65536+=5B(0xfe+LE32)',
tests
}
}
// ============================================================================
// Section 5: Peer Record Encoding
// ============================================================================
function generatePeerRecordVectors () {
const kpA = crypto.keyPair(SEED_A)
// No relay addresses
const peer1 = { publicKey: kpA.publicKey, relayAddresses: [] }
const encoded1 = c.encode(m.peer, peer1)
// With one relay address
const peer2 = {
publicKey: kpA.publicKey,
relayAddresses: [{ host: '192.168.1.100', port: 49737 }]
}
const encoded2 = c.encode(m.peer, peer2)
// With two relay addresses
const peer3 = {
publicKey: kpA.publicKey,
relayAddresses: [
{ host: '10.0.0.1', port: 12345 },
{ host: '172.16.0.5', port: 54321 }
]
}
const encoded3 = c.encode(m.peer, peer3)
return {
description: 'Peer record: fixed32(publicKey) + ipv4Array(relayAddresses)',
tests: [
{
name: 'peer_no_relays',
input: {
public_key: hex(kpA.publicKey),
relay_addresses: []
},
output: { encoded: hex(encoded1), byte_length: encoded1.length }
},
{
name: 'peer_one_relay',
input: {
public_key: hex(kpA.publicKey),
relay_addresses: [{ host: '192.168.1.100', port: 49737 }]
},
output: { encoded: hex(encoded2), byte_length: encoded2.length }
},
{
name: 'peer_two_relays',
input: {
public_key: hex(kpA.publicKey),
relay_addresses: [
{ host: '10.0.0.1', port: 12345 },
{ host: '172.16.0.5', port: 54321 }
]
},
output: { encoded: hex(encoded3), byte_length: encoded3.length }
}
]
}
}
// ============================================================================
// Section 6: Announce Message Encoding
// ============================================================================
function generateAnnounceMessageVectors () {
const kpA = crypto.keyPair(SEED_A)
const peer = { publicKey: kpA.publicKey, relayAddresses: [] }
// Basic announce (peer + signature, no refresh, no bump)
const sig = b4a.alloc(64, 0xAB) // deterministic fake signature
const ann1 = { peer, refresh: null, signature: sig, bump: 0 }
const encoded1 = c.encode(m.announce, ann1)
// With refresh
const refresh = b4a.alloc(32, 0xCC)
const ann2 = { peer, refresh, signature: sig, bump: 0 }
const encoded2 = c.encode(m.announce, ann2)
// With bump
const ann3 = { peer, refresh: null, signature: sig, bump: 42 }
const encoded3 = c.encode(m.announce, ann3)
// Full (all fields)
const ann4 = { peer, refresh, signature: sig, bump: 42 }
const encoded4 = c.encode(m.announce, ann4)
return {
description: 'Announce message: flags(varint) + optional(peer, refresh, signature, bump)',
tests: [
{
name: 'announce_peer_and_sig',
input: { flags_expected: 5 },
output: { encoded: hex(encoded1), byte_length: encoded1.length }
},
{
name: 'announce_with_refresh',
input: { flags_expected: 7 },
output: { encoded: hex(encoded2), byte_length: encoded2.length }
},
{
name: 'announce_with_bump',
input: { flags_expected: 13 },
output: { encoded: hex(encoded3), byte_length: encoded3.length }
},
{
name: 'announce_full',
input: { flags_expected: 15 },
output: { encoded: hex(encoded4), byte_length: encoded4.length }
}
]
}
}
// ============================================================================
// Section 7: Announce Signable (CRITICAL — this was the bug)
// ============================================================================
function generateAnnounceSignableVectors () {
const kpA = crypto.keyPair(SEED_A)
const target = b4a.alloc(32, 0x11)
const token = b4a.alloc(32, 0x22)
const nodeId = b4a.alloc(32, 0x33)
const refresh = b4a.alloc(32, 0x44)
const peer = { publicKey: kpA.publicKey, relayAddresses: [] }
// This is the EXACT function from persistent.js:269-284
// Reproducing it here so we capture its output
function annSignable (target, token, id, ann, ns) {
const signable = b4a.allocUnsafe(32 + 32)
const hash = signable.subarray(32)
signable.set(ns, 0)
sodium.crypto_generichash_batch(hash, [
target,
id,
token,
c.encode(m.peer, ann.peer),
ann.refresh || b4a.alloc(0)
])
return signable
}
// Test 1: Announce without refresh
const ann1 = { peer, refresh: null }
const signable1 = annSignable(target, token, nodeId, ann1, NS.ANNOUNCE)
// Test 2: Announce with refresh
const ann2 = { peer, refresh }
const signable2 = annSignable(target, token, nodeId, ann2, NS.ANNOUNCE)
// Test 3: Unannounce
const signable3 = annSignable(target, token, nodeId, ann1, NS.UNANNOUNCE)
// Test 4: Different node ID (per-node signature requirement)
const nodeId2 = b4a.alloc(32, 0x55)
const signable4 = annSignable(target, token, nodeId2, ann1, NS.ANNOUNCE)
// Test 5: Different token (per-node signature requirement)
const token2 = b4a.alloc(32, 0x66)
const signable5 = annSignable(target, token2, nodeId, ann1, NS.ANNOUNCE)
// Test 6: Actual signature (sign + verify round-trip)
const signature1 = b4a.allocUnsafe(64)
sodium.crypto_sign_detached(signature1, signable1, kpA.secretKey)
const verifies = sodium.crypto_sign_verify_detached(signature1, signable1, kpA.publicKey)
// Also capture the intermediate encoded peer for debugging
const encodedPeer = c.encode(m.peer, peer)
return {
description: 'Announce signable: NS(32) || BLAKE2b_batch(target, node_id, token, encoded_peer, refresh)(32) = 64 bytes. CRITICAL: each node gets different signable because token and node_id differ.',
tests: [
{
name: 'signable_no_refresh',
input: {
target: hex(target),
token: hex(token),
node_id: hex(nodeId),
public_key: hex(kpA.publicKey),
relay_addresses: [],
refresh: null,
namespace: 'ANNOUNCE'
},
output: {
signable: hex(signable1),
signable_length: signable1.length,
ns_prefix: hex(signable1.subarray(0, 32)),
hash_suffix: hex(signable1.subarray(32, 64)),
encoded_peer: hex(encodedPeer)
}
},
{
name: 'signable_with_refresh',
input: {
target: hex(target),
token: hex(token),
node_id: hex(nodeId),
public_key: hex(kpA.publicKey),
relay_addresses: [],
refresh: hex(refresh),
namespace: 'ANNOUNCE'
},
output: {
signable: hex(signable2),
signable_length: signable2.length
}
},
{
name: 'signable_unannounce',
input: {
target: hex(target),
token: hex(token),
node_id: hex(nodeId),
public_key: hex(kpA.publicKey),
relay_addresses: [],
refresh: null,
namespace: 'UNANNOUNCE'
},
output: {
signable: hex(signable3),
signable_length: signable3.length
}
},
{
name: 'signable_different_node_id',
note: 'MUST differ from signable_no_refresh because node_id is different. This proves per-node signatures.',
input: {
target: hex(target),
token: hex(token),
node_id: hex(nodeId2),
public_key: hex(kpA.publicKey),
relay_addresses: [],
refresh: null,
namespace: 'ANNOUNCE'
},
output: {
signable: hex(signable4),
differs_from_test_1: !b4a.equals(signable1, signable4)
}
},
{
name: 'signable_different_token',
note: 'MUST differ from signable_no_refresh because token is different.',
input: {
target: hex(target),
token: hex(token2),
node_id: hex(nodeId),
public_key: hex(kpA.publicKey),
relay_addresses: [],
refresh: null,
namespace: 'ANNOUNCE'
},
output: {
signable: hex(signable5),
differs_from_test_1: !b4a.equals(signable1, signable5)
}
},
{
name: 'sign_and_verify',
input: {
signable: hex(signable1),
secret_key: hex(kpA.secretKey),
public_key: hex(kpA.publicKey)
},
output: {
signature: hex(signature1),
signature_length: 64,
verifies: verifies
}
}
]
}
}
// ============================================================================
// Section 8: Mutable Put/Get Signable
// ============================================================================
function generateMutableSignableVectors () {
const kpA = crypto.keyPair(SEED_A)
// Target for mutable = BLAKE2b(publicKey)
const target = b4a.allocUnsafe(32)
sodium.crypto_generichash(target, kpA.publicKey)
// Mutable put signable: NS_MUTABLE_PUT(32) || seq(varint) || value
const value = b4a.from('hello world')
const seq = 42
// Build signable like hyperdht does it
const seqBuf = c.encode(c.uint, seq)
const signable = b4a.allocUnsafe(32 + seqBuf.length + value.length)
signable.set(NS.MUTABLE_PUT, 0)
signable.set(seqBuf, 32)
signable.set(value, 32 + seqBuf.length)
const signature = b4a.allocUnsafe(64)
sodium.crypto_sign_detached(signature, signable, kpA.secretKey)
return {
description: 'Mutable storage signable: NS_MUTABLE_PUT(32) || seq(varint) || value',
tests: [
{
name: 'mutable_target',
input: { public_key: hex(kpA.publicKey) },
output: { target: hex(target) }
},
{
name: 'mutable_signable',
input: {
seq: seq,
value: hex(value),
secret_key: hex(kpA.secretKey)
},
output: {
signable: hex(signable),
signable_length: signable.length,
signature: hex(signature)
}
}
]
}
}
// ============================================================================
// Section 9: Immutable Put Target
// ============================================================================
function generateImmutableVectors () {
const value = b4a.from('test immutable value')
const target = b4a.allocUnsafe(32)
sodium.crypto_generichash(target, value)
return {
description: 'Immutable storage: target = BLAKE2b(value)',
tests: [
{
name: 'immutable_target',
input: { value: hex(value) },
output: { target: hex(target) }
}
]
}
}
// ============================================================================
// Section 10: Noise IK — Ed25519 DH (deterministic with seeds)
// ============================================================================
function generateNoiseIKVectors () {
const kpInitiator = crypto.keyPair(SEED_A)
const kpResponder = crypto.keyPair(SEED_B)
// DH is done internally by noise-handshake using noise-curve-ed
// We can verify the Ed25519-to-Curve25519 conversion and DH
const curve = require('noise-curve-ed')
// Generate ephemeral keypair from seed for determinism
const ephSeed = b4a.alloc(32)
ephSeed[0] = 0xEE
const ephKp = crypto.keyPair(ephSeed)
// Ed25519 -> Curve25519 conversion
const curveSkInitiator = b4a.allocUnsafe(32)
const curvePkResponder = b4a.allocUnsafe(32)
sodium.crypto_sign_ed25519_sk_to_curve25519(curveSkInitiator, kpInitiator.secretKey)
sodium.crypto_sign_ed25519_pk_to_curve25519(curvePkResponder, kpResponder.publicKey)
// Compute DH (Ed25519 style — same as noise-curve-ed does internally)
const shared = b4a.allocUnsafe(32)
sodium.crypto_scalarmult(shared, curveSkInitiator, curvePkResponder)
return {
description: 'Noise IK uses Ed25519 keys with noise-curve-ed for DH. Prologue = NS.PEER_HANDSHAKE.',
tests: [
{
name: 'ed25519_to_curve25519_sk',
input: { ed25519_secret_key: hex(kpInitiator.secretKey) },
output: { curve25519_secret_key: hex(curveSkInitiator) }
},
{
name: 'ed25519_to_curve25519_pk',
input: { ed25519_public_key: hex(kpResponder.publicKey) },
output: { curve25519_public_key: hex(curvePkResponder) }
},
{
name: 'dh_result',
input: {
initiator_ed_sk: hex(kpInitiator.secretKey),
responder_ed_pk: hex(kpResponder.publicKey)
},
output: { shared_secret: hex(shared) }
},
{
name: 'noise_prologue',
input: {},
output: { prologue: hex(NS.PEER_HANDSHAKE) }
}
]
}
}
// ============================================================================
// Section 11: NoisePayload Encoding
// ============================================================================
function generateNoisePayloadVectors () {
const payload = {
version: 1,
error: 0,
firewall: FIREWALL.OPEN,
holepunch: null,
addresses4: [{ host: '192.168.1.100', port: 12345 }],
addresses6: [],
udx: { id: 1, seq: 0, reusable: false },
secretStream: { id: 0, seq: 0 },
relayAddresses: []
}
const encoded = c.encode(m.noisePayload, payload)
return {
description: 'NoisePayload encoding — sent inside Noise IK handshake',
tests: [
{
name: 'basic_payload',
input: payload,
output: {
encoded: hex(encoded),
byte_length: encoded.length
}
}
]
}
}
// ============================================================================
// Section 12: BLAKE2b batch hash (the core of announce signable)
// ============================================================================
function generateBlake2bBatchVectors () {
// Test that crypto_generichash_batch produces expected results
// This is the primitive used in annSignable
const input1 = b4a.alloc(32, 0x11)
const input2 = b4a.alloc(32, 0x22)
const input3 = b4a.alloc(32, 0x33)
const empty = b4a.alloc(0)
// Single input
const hash1 = b4a.allocUnsafe(32)
sodium.crypto_generichash_batch(hash1, [input1])
// Multiple inputs
const hash2 = b4a.allocUnsafe(32)
sodium.crypto_generichash_batch(hash2, [input1, input2, input3])
// Including empty
const hash3 = b4a.allocUnsafe(32)
sodium.crypto_generichash_batch(hash3, [input1, input2, input3, empty])
// Prove: including empty produces DIFFERENT hash than without empty
const hash4 = b4a.allocUnsafe(32)
sodium.crypto_generichash_batch(hash4, [input1, input2, input3])
return {
description: 'BLAKE2b batch hash (crypto_generichash_batch). NOTE: empty input at end DOES NOT change hash (verified below).',
tests: [
{
name: 'single_input',
input: { inputs: [hex(input1)] },
output: { hash: hex(hash1) }
},
{
name: 'three_inputs',
input: { inputs: [hex(input1), hex(input2), hex(input3)] },
output: { hash: hex(hash2) }
},
{
name: 'three_inputs_plus_empty',
input: { inputs: [hex(input1), hex(input2), hex(input3), ''] },
output: { hash: hex(hash3) }
},
{
name: 'empty_input_matters',
input: {},
output: {
with_empty_equals_without: b4a.equals(hash3, hash4),
note: 'If true, empty trailing input does not affect hash. If false, C++ must include it.'
}
}
]
}
}
// ============================================================================
// Section 13: IPv4 Address Encoding
// ============================================================================
function generateIPv4Vectors () {
const addresses = [
{ host: '127.0.0.1', port: 49737 },
{ host: '192.168.1.100', port: 12345 },
{ host: '10.0.0.1', port: 443 },
{ host: '255.255.255.255', port: 65535 }
]
const tests = addresses.map(addr => {
const state = c.state()
// ipv4 is 6 bytes: 4 bytes host + 2 bytes port LE
state.end = 6
state.buffer = b4a.allocUnsafe(6)
state.start = 0
// Manual encoding matching compact-encoding-net
const parts = addr.host.split('.')
state.buffer[0] = parseInt(parts[0])
state.buffer[1] = parseInt(parts[1])
state.buffer[2] = parseInt(parts[2])
state.buffer[3] = parseInt(parts[3])
state.buffer[4] = addr.port & 0xff
state.buffer[5] = (addr.port >> 8) & 0xff
return {
name: `ipv4_${addr.host.replace(/\./g, '_')}_${addr.port}`,
input: addr,
output: { encoded: hex(state.buffer), byte_length: 6 }
}
})
return {
description: 'IPv4 address encoding: 4 bytes host (network order) + 2 bytes port (LE)',
tests
}
}
// ============================================================================
// Section 14: FIND_PEER Target
// ============================================================================
function generateFindPeerTargetVectors () {
const kpA = crypto.keyPair(SEED_A)
// find_peer target = BLAKE2b(publicKey)
const target = b4a.allocUnsafe(32)
sodium.crypto_generichash(target, kpA.publicKey)
return {
description: 'FIND_PEER target = BLAKE2b(publicKey), NOT the raw public key',
tests: [
{
name: 'find_peer_target',
input: { public_key: hex(kpA.publicKey) },
output: { target: hex(target) }
}
]
}
}
// ============================================================================
// Assemble and output
// ============================================================================
const vectors = {
generated_at: new Date().toISOString(),
generator: 'generate_vectors.js',
reference_packages: {
hyperdht: require('hyperdht/package.json').version,
'hypercore-crypto': require('hypercore-crypto/package.json').version,
'compact-encoding': require('compact-encoding/package.json').version,
'sodium-universal': require('sodium-universal/package.json').version
},
sections: {
keypair: generateKeypairVectors(),
namespace: generateNamespaceVectors(),
token: generateTokenVectors(),
varint: generateVarintVectors(),
peer_record: generatePeerRecordVectors(),
announce_message: generateAnnounceMessageVectors(),
announce_signable: generateAnnounceSignableVectors(),
mutable_signable: generateMutableSignableVectors(),
immutable: generateImmutableVectors(),
noise_ik: generateNoiseIKVectors(),
noise_payload: generateNoisePayloadVectors(),
blake2b_batch: generateBlake2bBatchVectors(),
ipv4: generateIPv4Vectors(),
find_peer_target: generateFindPeerTargetVectors()
}
}
console.log(JSON.stringify(vectors, null, pretty ? 2 : 0))
// ============================================================================
// Helpers
// ============================================================================
function hex (buf) {
return b4a.toString(buf, 'hex')
}