// Files — bytes in the overlay, an oid in the record.
//
// ⛔ There is NO blob store, and nothing new is being invented here. hyper-overlay already holds
// pointers to files on disk and replicates them by content hash; `avatars.js` has done exactly this
// since profiles shipped. This is that same three-step generalised past images:
//
// hashFile(tmp) -> oid -> rename to <oid> -> registerFile('/blobs/' + oid, path)
//
// A record then names the oid — `` — and a peer that does not hold the
// bytes calls `fetchFile(oid)`. All of that is P2P; nothing about a file travels over HTTP.
//
// ⛔ No levels, no ladder, no manifest. SOCIAL-MODEL §6.6 specifies a mip-map pyramid, but that
// exists so a BRIDGE can pick a size that fits another network's ceiling. Nothing here bridges, so
// a pyramid would be complexity bought for a problem this stack does not have. If favela ever
// bridges media, §6.6 is waiting and this pointer-free design does not block it.
//
// ── Typing: what is a FACT and what is a GUESS ────────────────────────────────────────────────
//
// A record names an oid. There is no filename anywhere, so pgv3's `mimeFor(filename)` cannot
// transfer — extensions do not exist here. Types come from the bytes:
//
// binary a magic-byte match is a FACT. png, jpeg, gif, webp, mp4, webm, pdf.
// text "is this valid UTF-8 with no control bytes" is a FACT; "is this CSV" is a GUESS.
//
// ⚠ So text is accepted as text/plain on the fact, and upgraded to text/csv on the guess. The guess
// is safe to be wrong about: both are inert, and the only thing that would make a text file
// dangerous is serving it as html, which this never does.
import fs from '#node:fs'
import { join } from '#node:path'
import b4a from 'b4a'
// ⛔ There is NO size cap, because there is nothing to cap. hyper-overlay chunks with FastCDC and
// addresses by content hash — a large file is the case it is built for. An earlier draft of this
// file capped at 8 MiB and declared video unworkable, reasoning that a file rides the RPC as one
// base64 line. That was a limit I invented by copying the avatar path and then justified: the
// daemon and its surface share a machine by construction (it is a unix socket), and `registerFile`
// takes a PATH. Bytes never need to cross the socket at all.
const ascii = (b, i, s) => b4a.toString(b.subarray(i, i + s.length)) === s
/**
* The type of these bytes, or null to refuse them.
*
* ⛔ SVG is refused, deliberately and consistently with `avatars.js`. An SVG is a document that can
* carry script and fetch remote resources; png, jpeg, gif and webp cannot express either. pgv3
* takes the other route — it serves SVG under `Content-Security-Policy: default-src 'none'`
* (`lib/web/router.js:1028`) — and that is a reasonable position for a code forge showing a repo's
* own files. It is the wrong one for arbitrary bytes a stranger put in a post.
*/
export function sniff (bytes) {
if (!bytes || bytes.length < 12) return null
const b = bytes
// ── facts ───────────────────────────────────────────────────────────────────────────────────
if (b[0] === 0x89 && ascii(b, 1, 'PNG')) return 'image/png'
if (b[0] === 0xff && b[1] === 0xd8 && b[2] === 0xff) return 'image/jpeg'
if (ascii(b, 0, 'GIF8')) return 'image/gif'
if (ascii(b, 0, 'RIFF') && ascii(b, 8, 'WEBP')) return 'image/webp'
if (ascii(b, 0, '%PDF-')) return 'application/pdf'
// EBML — webm and mkv share it; only webm is claimed, since that is what a browser plays.
if (b[0] === 0x1a && b[1] === 0x45 && b[2] === 0xdf && b[3] === 0xa3) return 'video/webm'
// ISO-BMFF: the box length occupies 0..3 and the type follows, so `ftyp` sits at offset 4.
if (ascii(b, 4, 'ftyp')) return 'video/mp4'
// ── text: a fact, then a guess ──────────────────────────────────────────────────────────────
if (!isInertText(b)) return null
return looksDelimited(b) ? 'text/csv' : 'text/plain'
}
/**
* Valid UTF-8, and no control bytes that would mean this is really binary.
*
* ⛔ Also refuses anything opening as markup. Serving it as text/plain is already inert and the
* surface sets `nosniff`, so this is a third line rather than the only one — but an SVG is text,
* and "we refuse SVG" should not depend on one header being remembered.
*/
export function isInertText (b) {
if (!b || !b.length) return false
let s
try { s = b4a.toString(b, 'utf8') } catch { return false }
// A lone replacement char means the decode failed on bytes that were not text.
if (s.includes('�')) return false
// Tab, newline and carriage return are the only control characters text needs.
for (let i = 0; i < s.length; i++) {
const c = s.charCodeAt(i)
if (c < 0x20 && c !== 0x09 && c !== 0x0a && c !== 0x0d) return false
}
if (/^\s*(<\?xml|<svg|<!doctype|<html)/i.test(s.slice(0, 200))) return false
return true
}
/** Does the first line look like delimited fields? A GUESS, and labelled as one. */
export function looksDelimited (b) {
const first = b4a.toString(b.subarray(0, 4096), 'utf8').split('\n')[0]
if (!first) return false
const commas = (first.match(/,/g) || []).length
const tabs = (first.match(/\t/g) || []).length
return commas >= 1 || tabs >= 1
}
export class Files {
constructor (storage, overlay) {
this.dir = join(storage, 'blobs')
this.overlay = overlay
}
_path (oid) { return join(this.dir, oid) }
/**
* Take a file that already exists on disk and put it in the overlay: `{ oid, mimeType, size }`.
*
* ⛔ A PATH, not bytes. The caller is on this machine — the control socket is a unix socket — and
* `hashFile` wants a path anyway, so handing over bytes would mean encoding a file to cross a
* socket to a process that then writes it straight back to disk.
*
* The source is COPIED, not moved: it belongs to the caller and may be a file they still need.
* ⚠ It is read to sniff the type, which for a large file means reading it twice — once here and
* once by `hashFile`. Fixing that means sniffing from a partial read; not done, and not pretended.
*/
async put (srcPath) {
if (typeof srcPath !== 'string' || !srcPath) throw new Error('a file needs a path')
let head
try { head = fs.readFileSync(srcPath) } catch { throw new Error('no file at ' + srcPath) }
if (!head.length) throw new Error('that file is empty')
const mimeType = sniff(head)
if (!mimeType) {
throw new Error('favela does not recognise those bytes as an image, a video, a PDF or text — ' +
'the check is on the bytes rather than the file name, and SVG is refused on purpose')
}
fs.mkdirSync(this.dir, { recursive: true })
const h = await this.overlay.hashFile(srcPath)
const oid = h && h.contentHash
if (!oid) throw new Error('the overlay could not hash the file')
// Named by its hash. Identical files collapse to one across every record and every author this
// node holds — the dedup the overlay gives us for free.
const dest = this._path(oid)
try {
fs.copyFileSync(srcPath, dest)
} catch (err) {
throw new Error('could not store the file: ' + err.message)
}
await this.overlay.registerFile('/blobs/' + oid, dest, { size: head.length })
return { oid, mimeType, size: head.length }
}
/**
* Make sure the file is on local disk and say where it is: `{ path, mimeType, size }`, or null.
*
* ⛔ A PATH, not bytes — for the same reason `put` takes one. A surface serving this to a browser
* streams the file; it does not receive a copy of it through a socket.
*
* ⛔ Returns null rather than throwing. A file that has not replicated yet is a missing picture,
* not a broken page — the same distinction the record log makes between absent and unresolved.
*
* ⛔ The type is re-sniffed from the bytes every time, never remembered. These bytes may have
* arrived from another peer, and what a stranger's record claims about them is not evidence.
*/
async get (oid) {
if (typeof oid !== 'string' || !/^[0-9a-f]{8,128}$/i.test(oid)) return null
const p = this._path(oid)
const describe = (at = p) => {
try {
const fd = fs.openSync(at, 'r')
const head = b4a.alloc(4096)
let n = 0
try { n = fs.readSync(fd, head, 0, head.length, 0) } finally { fs.closeSync(fd) }
const mimeType = sniff(head.subarray(0, n))
if (!mimeType) return null
return { path: at, mimeType, size: fs.statSync(at).size }
} catch {
return null
}
}
const local = describe()
if (local) return local
try {
fs.mkdirSync(this.dir, { recursive: true })
const r = await this.overlay.fetchFile(oid, { destPath: p })
if (!r) return null
// ⛔ DESCRIBE WHERE IT ACTUALLY IS. On a LOCAL hit the overlay returns the path it already
// holds the bytes at — which may be the OTHER store's directory, since Files and Avatars
// fetch the same content hash from the same overlay and differ only in where they cache.
// Re-reading our own expected path meant a file the overlay had was reported missing: on
// ashatron, `file-get` for its own avatar answered {found:false} while the bytes sat in
// avatars/<oid>, and every consumer downstream faithfully rendered no picture.
return describe(r.destPath || p)
} catch {
return null
}
}
}
export default Files