// Files embedded in records — bytes in the overlay, an oid in the markdown.
//
// ⛔ Nothing new is invented here: hyper-overlay already holds pointers to files on disk and
// replicates them by content hash. This is `avatars.js` generalised past images.
//
// ⛔ PATHS cross the control socket, never bytes. Both ends are on one machine and `registerFile`
// takes a path, so handing a file over as base64 was an encode/decode round trip to give a process
// a file it wrote straight back to disk.
import test from 'brittle'
import Corestore from 'corestore'
import fs from '#node:fs'
import os from '#node:os'
import { join } from '#node:path'
import b4a from 'b4a'
import { Files, sniff, isInertText, looksDelimited } from '../lib/files.js'
const tmp = () => fs.mkdtempSync(join(os.tmpdir(), 'favela-files-'))
const at = (bytes, name) => {
const p = join(tmp(), name)
fs.writeFileSync(p, bytes)
return p
}
const PNG = b4a.concat([b4a.from([0x89]), b4a.from('PNG\r\n\x1a\n'), b4a.alloc(64)])
const PDF = b4a.concat([b4a.from('%PDF-1.7\n'), b4a.alloc(64)])
const MP4 = b4a.concat([b4a.alloc(4), b4a.from('ftypisom'), b4a.alloc(64)])
const WEBM = b4a.concat([b4a.from([0x1a, 0x45, 0xdf, 0xa3]), b4a.alloc(64)])
async function overlayIn (dir) {
const store = new Corestore(tmp())
const { HyperOverlay } = await import('hyper-overlay')
const o = new HyperOverlay(store.namespace('overlay'), {
namespace: 'favela-records', destDir: join(dir, '.fetch')
})
await o.ready()
return { o, close: async () => { await o.close(); await store.close() } }
}
test('★ a binary type is a FACT — magic bytes, never a filename', (t) => {
t.is(sniff(PNG), 'image/png')
t.is(sniff(b4a.concat([b4a.from([0xff, 0xd8, 0xff]), b4a.alloc(32)])), 'image/jpeg')
t.is(sniff(b4a.concat([b4a.from('GIF89a'), b4a.alloc(32)])), 'image/gif')
t.is(sniff(b4a.concat([b4a.from('RIFF'), b4a.alloc(4), b4a.from('WEBP'), b4a.alloc(32)])), 'image/webp')
t.is(sniff(PDF), 'application/pdf')
t.is(sniff(MP4), 'video/mp4', 'ISO-BMFF puts ftyp at offset 4, after the box length')
t.is(sniff(WEBM), 'video/webm')
})
test('⛔ SVG is refused, consistently with avatars', (t) => {
// An SVG is a document that can carry script and fetch remote resources. pgv3 serves SVG under a
// locked-down CSP, which is right for a forge showing a repo's own files and wrong for arbitrary
// bytes a stranger put in a post.
t.is(sniff(b4a.from('<svg onload=alert(1)>xxxxxxxxxxxx')), null)
t.is(sniff(b4a.from('<?xml version="1.0"?><svg/>xxxxx')), null)
t.is(sniff(b4a.from('<!doctype html><p>hi</p>xxxxxxxx')), null, 'nor html')
})
test('text is a fact; CSV is a GUESS, and both are inert', (t) => {
t.is(sniff(b4a.from('name,age\nalice,30\n')), 'text/csv', 'delimited')
t.is(sniff(b4a.from('just some prose about nothing')), 'text/plain', 'not delimited')
// ⚠ The guess is safe to be wrong about — the only thing that would make text dangerous is
// serving it as html, which nothing here does.
t.ok(looksDelimited(b4a.from('a\tb\tc\n')), 'tabs count too')
t.absent(looksDelimited(b4a.from('no separators here\n')))
})
test('⛔ binary that is not a known type is refused, not guessed', (t) => {
t.is(sniff(b4a.alloc(64)), null, 'NUL bytes are not text')
t.is(sniff(b4a.from([0x00, 0x01, 0x02, 0xff, 0xfe, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0b])), null)
t.is(sniff(b4a.alloc(3)), null, 'too short to judge is not a yes')
t.absent(isInertText(b4a.from([0x00, 0x41, 0x42, 0x43])), 'a NUL means binary')
})
test('★ a file goes in by PATH and comes back as a PATH', async (t) => {
const dir = tmp()
const { o, close } = await overlayIn(dir)
t.teardown(close)
const f = new Files(dir, o)
const r = await f.put(at(PDF, 'report.pdf'))
t.ok(/^[0-9a-f]{8,}$/i.test(r.oid), 'a content hash')
t.is(r.mimeType, 'application/pdf')
const got = await f.get(r.oid)
t.is(got.mimeType, 'application/pdf', 're-sniffed on the way out, never remembered')
t.is(got.size, PDF.length)
t.ok(b4a.equals(fs.readFileSync(got.path), PDF), 'and the bytes are intact')
// ⛔ No `bytes` field at all — a surface streams the file, it does not receive a copy.
t.is(got.bytes, undefined)
})
test('identical files collapse to one — the dedup the overlay gives for free', async (t) => {
const dir = tmp()
const { o, close } = await overlayIn(dir)
t.teardown(close)
const f = new Files(dir, o)
const a = await f.put(at(PNG, 'one.png'))
const b = await f.put(at(PNG, 'two-different-name.png'))
t.is(b.oid, a.oid, 'content addressing — the name was never part of the identity')
})
test('★ there is NO size cap, because the overlay chunks', async (t) => {
const dir = tmp()
const { o, close } = await overlayIn(dir)
t.teardown(close)
const f = new Files(dir, o)
// ⛔ An earlier draft capped this at 8 MiB and declared video unworkable, reasoning that a file
// rides the control socket as one base64 line. That limit was invented by copying the avatar
// path: the daemon takes a PATH, so nothing crosses the socket at all.
const big = b4a.concat([MP4, b4a.alloc(12 * 1024 * 1024)])
const r = await f.put(at(big, 'clip.mp4'))
t.is(r.mimeType, 'video/mp4')
t.is(r.size, big.length, '12 MiB, stored without complaint')
})
test('a file this node does not hold reads as absent, not as an error', async (t) => {
const dir = tmp()
const { o, close } = await overlayIn(dir)
t.teardown(close)
const f = new Files(dir, o)
t.is(await f.get('ab'.repeat(16)), null, 'not replicated yet is missing, not broken')
t.is(await f.get('not-an-oid'), null)
t.is(await f.get(null), null)
await t.exception(f.put('/no/such/file'), /no file at/)
await t.exception(f.put(at(b4a.alloc(0), 'empty')), /empty/)
})