const gen = require('generate-object-property')
const s = require('generate-string')
const { BitwiseNumericTypes, getBitwiseSize } = require('./types.js')
module.exports = function generateSchema(hyperschema, { esm = false, filename } = {}) {
const structs = []
const structsByName = new Map()
const deferred = []
const dedup = new Map()
const externals = new Map()
for (let i = 0; i < hyperschema.schema.length; i++) {
const fqn = hyperschema.typesByPosition.get(i)
const type = hyperschema.resolve(fqn)
if (type.isStruct) {
structs.push(type)
}
if (type.isArray) {
structs.push(type)
}
if (type.isRecord) {
structs.push(type)
}
if (type.isEnum) {
structs.push(type)
}
if (type.isExternal) {
structs.push(type)
}
if (type.isVersioned) {
structs.push(type)
}
}
for (let i = 0; i < structs.length; i++) {
const struct = structs[i]
const id = 'encoding' + i
const result = { encoder: null, id }
structsByName.set(struct.fqn, result)
}
for (let i = 0; i < structs.length; i++) {
const struct = structs[i]
const result = structsByName.get(struct.fqn)
if (struct.isExternal) {
const v = requireExternal(struct, filename)
result.encoder = `const ${result.id} = ${gen(v, struct.external)}\n`
continue
}
result.encoder = struct.isEnum
? generateEnum(result.id, struct)
: struct.isVersioned
? generateVersioned(result.id, struct)
: generateStruct(result.id, struct, deferred, false)
if (struct.isInlined) {
result.encoder += '\n' + generateStruct(result.id, struct, deferred, true)
}
}
let str = ''
str += '// This file is autogenerated by the hyperschema compiler\n'
str += `// Schema Version: ${hyperschema.version}\n`
str += '/* eslint-disable camelcase */\n'
str += '/* eslint-disable quotes */\n'
str += '/* eslint-disable space-before-function-paren */\n'
str += '\n'
if (esm) {
str += `import { c } from 'hyperschema/runtime'\n`
} else {
str += `const { c } = require('hyperschema/runtime')\n`
}
for (const [req, v] of externals) {
if (esm) {
str += `import * as ${v} from ${s(req)}\n`
} else {
str += `const ${v} = require(${s(req)})\n`
}
}
str += '\n'
str += `const VERSION = ${hyperschema.version}\n`
str += '\n'
str += '// eslint-disable-next-line no-unused-vars\n'
str += 'let version = VERSION\n'
str += '\n'
for (let i = 0; i < structs.length; i++) {
const struct = structs[i]
const { encoder } = structsByName.get(struct.fqn)
str += encoder
str += '\n'
}
if (deferred.length) {
str += deferred.join('')
str += '\n'
}
str += 'function setVersion(v) {\n'
str += ' version = v\n'
str += '}\n'
str += '\n'
str += 'function encode(name, value, v = VERSION) {\n'
str += ' version = v\n'
str += ' return c.encode(getEncoding(name), value)\n'
str += '}\n'
str += '\n'
str += 'function decode(name, buffer, v = VERSION) {\n'
str += ' version = v\n'
str += ' return c.decode(getEncoding(name), buffer)\n'
str += '}\n'
str += '\n'
str += 'function getEnum(name) {\n'
str += ' switch (name) {\n'
for (let i = 0; i < structs.length; i++) {
const struct = structs[i]
if (struct.enum) {
str += ` case ${s(struct.fqn)}:\n`
str += ` return ${structsByName.get(struct.fqn).id}_enum\n`
}
}
str += ' default:\n'
str += " throw new Error('Enum not found ' + name)\n"
str += ' }\n'
str += '}\n'
str += '\n'
str += 'function getEncoding(name) {\n'
str += ' switch (name) {\n'
for (let i = 0; i < structs.length; i++) {
const struct = structs[i]
str += ` case ${s(struct.fqn)}:\n`
str += ` return ${structsByName.get(struct.fqn).id}\n`
}
str += ' default:\n'
str += " throw new Error('Encoder not found ' + name)\n"
str += ' }\n'
str += '}\n'
str += '\n'
str += 'function getStruct(name, v = VERSION) {\n'
str += ' const enc = getEncoding(name)\n'
str += ' return {\n'
str += ' preencode(state, m) {\n'
str += ' version = v\n'
str += ' enc.preencode(state, m)\n'
str += ' },\n'
str += ' encode(state, m) {\n'
str += ' version = v\n'
str += ' enc.encode(state, m)\n'
str += ' },\n'
str += ' decode(state) {\n'
str += ' version = v\n'
str += ' return enc.decode(state)\n'
str += ' }\n'
str += ' }\n'
str += '}\n'
str += '\n'
str += 'const resolveStruct = getStruct // compat\n'
str += '\n'
if (esm) {
str +=
'export { resolveStruct, getStruct, getEnum, getEncoding, encode, decode, setVersion, version }\n'
} else {
str +=
'module.exports = {\n resolveStruct,\n getStruct,\n getEnum,\n getEncoding,\n encode,\n decode,\n setVersion,\n version\n}\n'
}
return str
function requireExternal(struct, filename) {
if (!filename) throw new Error('Must provide filename if using external types')
const req = struct.require(filename)
let v = externals.get(req)
if (v) return v
v = 'external' + externals.size
externals.set(req, v)
return v
}
function generateEnum(id, struct) {
let str = ''
str += `const ${id}_enum = {\n`
for (let i = 0; i < struct.enum.length; i++) {
const e = struct.enum[i]
const value = struct.strings ? s(e.key) : i + struct.offset
str += ` ${gen.property(e.key)}: ${value}${i < struct.enum.length - 1 ? ',' : ''}\n`
}
str += '}\n\n'
const preencode = generateEncode({ preencode: true })
const encode = generateEncode()
const decode = generateDecode()
str += `// ${struct.fqn} enum\n`
str += `const ${id} = {\n`
str += ' preencode (state, m) {\n'
str += `${preencode}`
str += ' },\n'
str += ' encode (state, m) {\n'
str += `${encode}`
str += ' },\n'
str += ' decode (state) {\n'
str += `${decode}`
str += ' }\n'
str += '}\n'
return str
function generateEncode({ preencode = false } = {}) {
const max = struct.enum.length + struct.offset - 1
const encode = preencode ? 'preencode' : 'encode'
if (preencode && max <= 0xfc) {
return ` state.end++ // max enum is ${max} so always one byte\n`
}
if (!struct.strings) {
let str = ''
str += ` if (m > ${max}) throw new Error('Unknown enum')\n`
str += ` c.uint.${encode}(state, m)\n`
return str
}
let str = ' switch (m) {\n'
for (let i = 0; i < struct.enum.length; i++) {
str += ` case ${s(struct.enum[i].key)}:\n`
str += ` c.uint.${encode}(state, ${i + struct.offset})\n`
str += ' break\n'
}
str += ' default:\n'
str += " throw new Error('Unknown enum')\n"
str += ' }\n'
return str
}
function generateDecode() {
if (!struct.strings) return ' return c.uint.decode(state)\n'
let str = ' switch (c.uint.decode(state)) {\n'
for (let i = 0; i < struct.enum.length; i++) {
str += ` case ${i + struct.offset}:\n`
str += ` return ${s(struct.enum[i].key)}\n`
}
str += ' default: return null\n'
str += ' }\n'
return str
}
}
function getGeneratedTypeEncoder(type) {
if (type.isPrimitive) {
return getPrimitiveEncoder(type)
}
if (type.isAlias) {
return getGeneratedTypeEncoder(type.type)
}
return structsByName.get(type.fqn).id
}
function dedupConst(name, def, deferred = false) {
if (dedup.has(def)) return `const ${name} = ${dedup.get(def).name}\n`
dedup.set(def, { deferred, name })
return `const ${name} = ${def}\n`
}
function getGeneratedFieldEncoder(field) {
const typeStr = getGeneratedTypeEncoder(field.type)
if (field.array) {
if (field.framed) {
return `c.array(c.frame(${typeStr}))`
}
return `c.array(${typeStr})`
}
if (field.record) {
const keyType = getGeneratedTypeEncoder(field.key)
const valueType = getGeneratedTypeEncoder(field.value)
return `c.record(${keyType}, ${valueType})`
}
if (field.framed) {
return `c.frame(${typeStr})`
}
return typeStr
}
function generateVersioned(id, struct) {
let str = ''
let enc = ''
str += `// ${struct.fqn}\n`
str += ''
const preencode = generateEncode({ preencode: true })
const encode = generateEncode()
const decode = generateDecode()
enc += '{\n'
enc += ' preencode(state, m) {\n'
enc += `${preencode.trimRight()}\n`
enc += ' },\n'
enc += ' encode(state, m) {\n'
enc += `${encode.trimRight()}\n`
enc += ' },\n'
enc += ' decode(state) {\n'
enc += `${decode.trimRight()}\n`
enc += ' }\n'
enc += '}'
str += dedupConst(id, enc)
return str
function generateEncode({ preencode = false } = {}) {
const fn = preencode ? 'preencode' : 'encode'
let str = ''
str += ` c.uint.${fn}(state, m.version)\n`
str += ' switch (m.version) {\n'
let version = 0
for (let i = 0; i < struct.versions.length; i++) {
const def = struct.versions[i]
while (version <= def.version) str += ` case ${version++}:\n`
str += ` ${getEncoder(def.type)}.${fn}(state, m)\n`
str += ' break\n'
}
str += ' default:\n'
str += " throw new Error('Unsupported version')\n"
str += ' }'
return str
}
function generateDecode() {
let str = ''
str += ' const start = state.start\n'
str += ' const v = c.uint.decode(state)\n'
str += ' state.start = start\n'
str += ' switch (v) {'
let version = 0
for (let i = 0; i < struct.versions.length; i++) {
const def = struct.versions[i]
while (version <= def.version) str += `\n case ${version++}:`
str += ' {\n'
str += ` const decoded = ${getEncoder(def.type)}.decode(state)\n`
if (def.map) {
const v = requireExternal(struct, filename)
str += ` const map = ${gen(v, def.map)}\n`
str += ' return map(decoded)\n'
} else {
str += ' return decoded\n'
}
str += ' }'
}
str += '\n default:\n'
str += " throw new Error('Unsupported version')\n"
str += ' }'
return str
}
function getEncoder(type) {
if (type.isAlias) return getEncoder(type.type)
if (type.isPrimitive) return getPrimitiveEncoder(type)
return structsByName.get(type.fqn).id
}
}
function generateStruct(id, struct, deferred, inlinable) {
const fieldTypes = new Map()
if (inlinable) id += '_inline'
const fields = struct.isArray ? [] : struct.resolvedFields
const flagType = struct.compact ? getFittingType(struct.maxFlag) : 'uint'
let str = ''
let enc = ''
let keyEnc = ''
if (struct.isArray) {
enc = getGeneratedTypeEncoder(struct.type)
} else if (struct.isRecord) {
keyEnc = getGeneratedTypeEncoder(struct.key)
enc = getGeneratedTypeEncoder(struct.value)
} else {
for (let i = 0; i < struct.fields.length; i++) {
const field = struct.fields[i]
if (!field.framed && !field.array) continue
const fieldId = `${id}_${i}`
fieldTypes.set(field.name, fieldId)
const fieldEnc = getGeneratedFieldEncoder(field)
const isDeferred = !!dedup.get(fieldEnc)?.deferred
// if recursive, defer the field type
if (structsByName.get(field.type.fqn)?.encoder === null || isDeferred) {
let d = ''
d += `// ${struct.fqn}.${field.name}, deferred due to recusive use\n`
d += dedupConst(fieldId, fieldEnc, true)
deferred.push(d)
} else {
str += `// ${struct.fqn}.${field.name}\n`
str += dedupConst(fieldId, fieldEnc, false)
}
}
if (str) {
str += '\n'
}
const preencode = generateEncode({ preencode: true })
const encode = generateEncode()
const decode = generateDecode()
enc += '{\n'
enc += ` preencode(state, m) {\n`
enc += `${preencode.trimRight()}\n`
enc += ' },\n'
enc += ` encode(state, m) {\n`
enc += `${encode.trimRight()}\n`
enc += ' },\n'
enc += ` decode(state${inlinable ? ', inlining' : ''}) {\n`
enc += `${decode.trimRight()}\n`
enc += ' }\n'
enc += '}'
}
str += `// ${struct.fqn}${inlinable ? ' (inline)' : ''}\n`
if (struct.isArray) {
if (struct.framed) {
str += dedupConst(id, `c.array(c.frame(${enc}))`)
} else {
str += dedupConst(id, `c.array(${enc})`)
}
} else if (struct.isRecord) {
str += dedupConst(id, `c.record(${keyEnc}, ${enc})`)
} else {
str += dedupConst(id, enc)
}
return str
function generateFlags(indent, flag, parent = 'm') {
const first = !flag
const entries = (flag ? flag.fields : fields).filter((f) => !f.field.external)
const mutable = first && entries.some((f) => f.fields.length > 0)
const mut = first ? (mutable ? 'let' : 'const') : ''
const flags = entries
.filter((f) => !f.field.required)
.map((f) => generateBitFlag(f.field, parent, f.flag))
let str = combineFlags(indent, mut, first ? '=' : '|=', flags)
for (const f of entries) {
if (!f.fields.length) continue
const name = gen(parent, f.field.name)
str += `${indent}if (${name}) {\n`
str += generateFlags(`${indent} `, f, name)
str += `${indent}}\n`
}
return str
}
function generateEncode({ preencode = false } = {}) {
const fn = preencode ? 'preencode' : 'encode'
let str = ''
let fastFlags = false
const flags = generateFlags(' ')
const smallUint = struct.maxFlag < 128
if (flags !== ' const flags = 0\n') {
if ((smallUint || struct.compact) && preencode) {
fastFlags = !struct.compact
} else if (!inlinable) {
str += flags
str += '\n'
}
}
let bitfield = false
let optional = false
for (let i = 0; i < struct.fields.length; i++) {
const field = struct.fields[i]
if (field.external) continue
if (i === struct.flagsPosition && !inlinable) {
if (fastFlags) {
str += ` state.end++ // max flag is ${struct.maxFlag} so always one byte`
} else if (struct.compact && preencode) {
if (getByteSize(struct.maxFlag) === 1) {
str += ' state.end++ // flags are fixed size'
} else {
str += ` state.end += ${getByteSize(struct.maxFlag)} // flags are fixed size`
}
} else {
str += ` c.${flagType}.${fn}(state, flags)`
}
bitfield = true
}
if (field.type.bool || field.type.bitwise) continue
if (bitfield === true) {
str += '\n'
bitfield = false
}
if (!field.required && !optional) {
str += '\n'
optional = true
}
let prefix = ' '
if (!field.required) {
prefix += ` ${vPrefix(field.version, gen('m', field.name))}`
}
const suffix = field.inline ? '_inline' : ''
const encoder = `${getFieldEncoder(field)}${suffix}.${fn}(state, ${gen('m', field.name)})`
str += `${prefix} ${encoder}`
if (i !== struct.fields.length - 1) {
let onlyBools = true
for (let j = i + 1; j < struct.fields.length; j++) {
const field = struct.fields[j]
if (!field) break
if (!field.type.bool && !field.type.bitwise) {
onlyBools = false
break
}
}
if (!onlyBools || !bitfield) str += '\n'
}
}
for (let i = 0; i < str.length; i++) {
if (str[i] !== '\n') {
str = str.slice(i)
break
}
}
return str
}
function generateDecode() {
let str = ''
let seen = 0
if (struct.expectsVersion) {
str += ' const v = c.uint.decode(state)\n'
}
const pre = new Map()
for (let i = 0; i < fields.length; i++) {
const field = fields[i].field
if (!field.required || field.type.bool || i === struct.flagsPosition || field.inline) {
break
}
seen++
if (field.external) continue
pre.set(field, 'r' + i)
str += ` const r${i} = ${getFieldEncoder(field)}.decode(state)\n`
}
if (seen === struct.flagsPosition) {
if (inlinable) {
str += ' const flags = inlining\n\n'
} else if (alwaysOptional(struct)) {
str += ` const flags = c.${flagType}.decode(state)\n\n`
} else {
str += ` const flags = state.start < state.end ? c.${flagType}.decode(state) : 0\n\n`
}
} else if (pre.size) {
str += '\n'
}
str += ' return {\n'
if (struct.expectsVersion) str += ' version: v,\n'
for (let i = 0; i < fields.length; i++) {
const f = fields[i]
const field = f.field
const end = i < fields.length - 1 ? ',\n' : '\n'
const flag = vInlinePrefix(field.version, `(flags & ${field.flag}) !== 0`)
const prop = gen.property(field.name)
let shift = ''
if (field.inline) {
shift = `, ${shiftRight('flags', f.shift, false, field.struct)}`
}
const suffix = field.inline ? '_inline' : ''
const decoder = `${getFieldEncoder(field)}${suffix}.decode(state${shift})`
if (field.external) {
str += ` ${prop}: ${field.getDefaultValue()}${end}`
continue
}
if (pre.has(field)) {
str += ` ${prop}: ${pre.get(field)}${end}`
continue
}
if (field.type.bool) {
str += ` ${prop}: ${flag}${end}`
continue
}
if (field.type.bitwise) {
const mask = 2 ** field.type.bitwise - 1
const prefix = vInlinePrefix(field.version)
const shift = Math.log2(field.flag)
const value = `${shiftRight('flags', shift, true, field.struct)} & ${mask}`
const v = prefix ? `${prefix} ? ${value} : 0` : `${value}`
str += ` ${prop}: ${v}${end}`
continue
}
if (field.required) {
str += ` ${prop}: ${decoder}${end}`
continue
}
str += ` ${prop}: ${flag} ? ${decoder} : ${field.getDefaultValue()}${end}`
}
str += ' }'
return str
}
function vPrefix(v, condition) {
if (v === 1) return condition ? `if (${condition})` : ''
return condition ? `if (version >= ${v} && ${condition})` : `if (version >= ${v}) `
}
function vInlinePrefix(v, condition) {
if (v === 1) return condition || ''
return condition ? `(version >= ${v} && ${condition})` : `version >= ${v}`
}
function generateBitFlag(field, parent = 'm', flag = field.flag) {
const id = gen(parent, field.name)
if (field.type.bitwise) {
const mask = 2 ** field.type.bitwise - 1
const prefix = vInlinePrefix(field.version)
const shift = Math.log2(flag)
const value = shiftLeft(`(${id} & ${mask})`, shift, !prefix, field.struct)
return prefix ? `(${prefix} ? ${value} : 0)` : `${value}`
}
return `(${vInlinePrefix(field.version, id)} ? ${flag} : 0)`
}
function combineFlags(indent, mut, op, flags) {
if (mut) mut += ' '
let s = `${indent}${mut}flags ${op}`
if (flags.length === 0) flags.push('(0)')
if (flags.length === 1) {
flags = flags.slice(0)
flags[0] = flags[0].slice(1, -1)
}
const inline = flags.join(' | ')
if (s.length + inline.length + 1 <= 100) return s + ' ' + inline + '\n'
if (inline.length + indent.length + 2 <= 100) return s + '\n ' + indent + inline + '\n'
s += '\n'
for (let i = 0; i < flags.length; i++) {
s += indent + ' ' + flags[i] + (i < flags.length - 1 ? ' |\n' : '\n')
}
return s
}
function getFieldEncoder(field) {
if (fieldTypes.has(field.name)) return fieldTypes.get(field.name)
return getEncoder(field.type)
}
function getEncoder(type) {
if (type.isAlias) return getEncoder(type.type)
if (type.isPrimitive) return getPrimitiveEncoder(type)
return structsByName.get(type.fqn).id
}
}
}
function alwaysOptional(struct) {
for (let i = 0; i < struct.fields.length; i++) {
const f = struct.fields[i]
if (f.required) continue
for (let j = 0; j < i; j++) {
const p = struct.fields[j]
if (p.version !== f.version) return false
}
return true
}
return false
}
function getPrimitiveEncoder(type) {
return `c.${type.name}`
}
function shiftRight(v, n, paren, struct) {
if (n === 0) return v
const start = paren ? '(' : ''
const end = paren ? ')' : ''
if (struct.maxFlag <= 2 ** 31) {
return `${start}${v} >>> ${n}${end}`
}
return `Math.floor(${v} / ${2 ** n})`
}
function shiftLeft(v, n, paren, struct) {
if (n === 0) return v
const start = paren ? '(' : ''
const end = paren ? ')' : ''
if (struct.maxFlag <= 2 ** 31) {
return `${start}${v} << ${n}${end}`
}
return `${start}${v} * ${2 ** n}${end}`
}
function getByteSize(bit) {
if (bit < 2 ** 8) return 1
if (bit < 2 ** 16) return 2
if (bit < 2 ** 24) return 3
if (bit < 2 ** 32) return 4
if (bit < 2 ** 40) return 5
if (bit < 2 ** 48) return 6
if (bit < 2 ** 56) return 7
return 8
}
function getFittingType(bit) {
return 'uint' + getByteSize(bit) * 8
}