// strip(source) -> { text, facets } — spec/markdown-strip.md
//
// The strip's output is what bsky sees and what a CID commits to, so every rule here cites the
// spec section it implements and nothing is left to "whatever the parser does" (§1). The conformance
// vectors are the oracle; this file exists to reproduce them byte-for-byte.
//
// Structure, and why:
//
// 1. CRLF normalisation happens on the INPUT, before parsing (§5.1).
// 2. Each block renders into its OWN buffer so an empty block can be omitted along with its
// separator (§4.1) — you cannot decide that after concatenating.
// 3. Facets are recorded as CHARACTER ranges while building, then mapped through normalisation,
// then converted to UTF-8 BYTE offsets last (§6.4: offsets are computed against the FINAL
// text, never an intermediate). Doing it in this order is what makes the multibyte vectors
// pass; computing byte offsets early and patching them later is the defect §8 exists to catch.
// 4. Normalisation is expressed as deletions with an index map, so a facet's range can be
// carried across it exactly rather than re-derived by searching the text.
//
// Extensions: micromark supplies the four that change block/inline STRUCTURE (GFM strikethrough,
// GFM tables, footnotes, directives). Wiki-links and hashtags are handled here because the spec
// gives them as lexical rules over text (§3.6, §3.7) — a normative regex in the hashtag case —
// so there is no structural ambiguity to get wrong.
import { isUserUri } from './uri.js'
import { fromMarkdown } from 'mdast-util-from-markdown'
import { gfmStrikethrough } from 'micromark-extension-gfm-strikethrough'
import { gfmStrikethroughFromMarkdown } from 'mdast-util-gfm-strikethrough'
import { gfmTable } from 'micromark-extension-gfm-table'
import { gfmTableFromMarkdown } from 'mdast-util-gfm-table'
import { gfmFootnote } from 'micromark-extension-gfm-footnote'
import { gfmFootnoteFromMarkdown } from 'mdast-util-gfm-footnote'
import { directive } from 'micromark-extension-directive'
import { directiveFromMarkdown } from 'mdast-util-directive'
const MICROMARK = {
// ⛔ singleTilde:false — micromark defaults it TRUE, which would consume `~x~` as strikethrough.
// §2.1 pins the extension as `~~x~~`, and the dialect is CLOSED: syntax outside it is text, not
// markup. Caught by the `single-tilde-is-literal` vector, which exists because of that rule.
extensions: [gfmStrikethrough({ singleTilde: false }), gfmTable(), gfmFootnote(), directive()],
mdastExtensions: [
gfmStrikethroughFromMarkdown(),
gfmTableFromMarkdown(),
gfmFootnoteFromMarkdown(),
directiveFromMarkdown()
]
}
// §3.3 — NORMATIVE. This is favela-web's shipped URL_RE verbatim; changing it changes records.
// The trailing class is what keeps "see https://x.com." from swallowing the full stop.
const URL_RE = /\bhttps?:\/\/[^\s<]+[^\s<.,:;!?)\]}'"]/g
// §3.6 — [[Page]] and [[Page|label]].
const WIKILINK_RE = /\[\[([^\][|]+)(?:\|([^\][]*))?\]\]/g
// §3.7 — '#' at start-of-text or after whitespace, then 1-64 of [A-Za-z0-9_] plus any non-ASCII
// letter. A '#' inside a word, or one followed only by digits, is literal text.
const HASHTAG_RE = /(^|\s)(#([A-Za-z0-9_À-]{1,64}))/g
// §3.2 — only http/https may carry a link facet. Mention schemes are handled separately (§3.4);
// anything else emits its label as plain text with no facet.
const isHttp = (uri) => /^https?:\/\//i.test(String(uri || ''))
// §3.4 — a mention names a USER. `favela://<identity-root>` is the canonical form; `did:` is the
// LEGACY form kept because §8 is forward-only and nothing is rewritten. strip() does NOT resolve
// either: it is a codec with no registry access, and resolution to a bsky DID belongs at
// PROJECTION time (identity.md §5). So the facet carries the target exactly as written.
// ⛔ NARROWED, and this is the copy that matters most: strip() produces the FACETS a bridge sends.
// The old test was bare `^(favela://|did:)`, so a record URI — `favela://<root>/<collection>/<name>`,
// which favela-surface emits for a quote — became a MENTION facet, and every bridge then tried to
// resolve a record address as if it were a person. A mention is the USER form: no path, or a `did:`.
//
// ⚠ This was a second, divergent copy of dialect.js's predicate. Two definitions of one normative
// rule is how they drift; it delegates now.
const isMention = (uri) => {
const s = String(uri || '')
return /^did:/i.test(s) || isUserUri(s)
}
// §6 — the NATIVE facet feature: { type, value }, the same compact form the vectors are written in.
//
// This used to be app.bsky.richtext.facet shape ($type / features[] / index{}), which put atproto's
// data format in the core library's OUTPUT — the inherited-shape problem one level below the one
// design.md §1 caught, and flagged as such by markdown-strip.md §3.4. It is gone (MVP.md §4
// decision 3): a mention's target is a favela:// URI, and it travelled in a field named `did`.
// project(rec, 'bsky') is the one place that knows atproto's names; nothing else may.
const linkFeature = (uri) => ({ type: 'link', value: uri })
const mentionFeature = (target) => ({ type: 'mention', value: target })
const tagFeature = (tag) => ({ type: 'tag', value: tag })
// A block's rendered text plus the facets found inside it, at character offsets relative to
// this block. Kept per-block so §4.1 can drop an empty block and its separator together.
class Buf {
constructor () {
this.text = ''
this.marks = [] // { start, end, feature } — character offsets into this.text
}
get length () { return this.text.length }
add (s) { this.text += s }
mark (start, end, feature) {
if (end > start) this.marks.push({ start, end, feature })
}
// Append another buffer's text and marks, shifting the marks by the current length.
append (other) {
const base = this.text.length
this.text += other.text
for (const m of other.marks) {
this.marks.push({ start: m.start + base, end: m.end + base, feature: m.feature })
}
}
}
// ── inline ────────────────────────────────────────────────────────────────────
// §3.6 / §3.7 / §3.3 — lexical passes over a plain text run, in one left-to-right sweep so the
// three can never produce overlapping facets (§6.2 would otherwise have to arbitrate).
function addTextRun (buf, raw) {
// §3.8 — a SOFT line break arrives as "\n" inside a text node and becomes a single space.
// A HARD break is a separate 'break' node (below) and stays a newline, so the distinction is
// made here by node type rather than by trying to remember which newlines were which after
// the text is assembled. One character in, one character out, so no offset shifts.
const soft = raw.replace(/\n/g, ' ')
// Wiki-links next: they rewrite the text, and §3.6 emits no facet, so doing them before the
// facet-bearing scans keeps every offset below correct without any later adjustment.
const s = soft.replace(WIKILINK_RE, (_m, page, label) => (label !== undefined ? label : page))
const found = []
for (const m of s.matchAll(URL_RE)) {
found.push({ start: m.index, end: m.index + m[0].length, feature: linkFeature(m[0]) })
}
for (const m of s.matchAll(HASHTAG_RE)) {
// m[1] is the leading boundary (start-of-text or the whitespace); the facet covers '#tag'
// INCLUDING the hash (§3.7: "the hash is kept in the text"), while the value drops it.
const start = m.index + m[1].length
found.push({ start, end: start + m[2].length, feature: tagFeature(m[3]) })
}
found.sort((a, b) => a.start - b.start || b.end - a.end)
const base = buf.length
let taken = -1
for (const f of found) {
if (f.start < taken) continue // overlapping match: the earlier/longer one already won (§6.2)
buf.mark(base + f.start, base + f.end, f.feature)
taken = f.end
}
buf.add(s)
}
function inline (nodes, buf) {
for (const node of nodes || []) {
switch (node.type) {
case 'text':
addTextRun(buf, node.value)
break
// §3.1 — markers removed, inner text kept verbatim, NO facet (atproto has no bold).
case 'emphasis':
case 'strong':
case 'delete':
inline(node.children, buf)
break
// §3.1 — a code span is text, not markup. Its value is literal, so no lexical scan:
// a URL inside backticks was written as code and must not become a link.
case 'inlineCode':
buf.add(node.value)
break
case 'link': {
const uri = node.url
const start = buf.length
const inner = new Buf()
inline(node.children, inner)
if (isMention(uri)) {
// §3.4 — a mention target names the user, not a network identifier. The label is kept.
buf.append(inner)
buf.mark(start, buf.length, mentionFeature(uri))
} else if (isHttp(uri)) {
if (inner.length === 0) {
// §3.2 — an empty label emits the URL as text, with a facet over it.
buf.add(uri)
buf.mark(start, buf.length, linkFeature(uri))
} else {
buf.append(inner)
buf.mark(start, buf.length, linkFeature(uri))
}
} else {
// §3.2 ⛔ — non-http (javascript:, etc.) emits the label as PLAIN TEXT, no facet.
// inner.marks are dropped deliberately: a refused link must not smuggle one out.
buf.add(inner.text)
}
break
}
// §3.5 — images produce no text and no facet. The embed pipeline consumes them separately.
case 'image':
case 'imageReference':
break
// §3.9 — a footnote reference is removed entirely.
case 'footnoteReference':
break
// §3.8 — a hard break is one newline. (A soft break arrives as "\n" inside a text node
// and is turned into a space by the block-level collapse below.)
case 'break':
buf.add('\n')
break
// §3.10 — raw HTML is emitted as its literal source text, unparsed, with no facet.
case 'html':
buf.add(node.value)
break
case 'linkReference':
// An unresolved reference has no target, so there is nothing to point a facet at:
// it degrades to its label, like a wiki-link (§3.6).
inline(node.children, buf)
break
default:
if (node.children) inline(node.children, buf)
else if (typeof node.value === 'string') buf.add(node.value)
}
}
}
function paragraphText (children) {
const buf = new Buf()
inline(children, buf)
return buf
}
// ── blocks ────────────────────────────────────────────────────────────────────
function listBlock (node, depth) {
const buf = new Buf()
const indent = ' '.repeat(depth) // §4.5 — nested lists indent two spaces per level
let n = typeof node.start === 'number' ? node.start : 1
let first = true
for (const item of node.children || []) {
if (item.type !== 'listItem') continue
// §4.5 — ordered items use the RENDERED sequence, not the literal source numbers.
const marker = node.ordered ? n + '. ' : '• '
if (node.ordered) n++
const inner = new Buf()
let firstChild = true
for (const child of item.children || []) {
const sub = child.type === 'list' ? listBlock(child, depth + 1) : blockBuf(child, depth)
if (sub.length === 0) continue
if (!firstChild) inner.add('\n')
inner.append(sub)
firstChild = false
}
if (inner.length === 0) continue
if (!first) buf.add('\n')
buf.add(indent + marker)
buf.append(inner)
first = false
}
return buf
}
function tableBlock (node) {
const buf = new Buf()
let firstRow = true
for (const row of node.children || []) {
if (row.type !== 'tableRow') continue
if (!firstRow) buf.add('\n')
let firstCell = true
for (const cell of row.children || []) {
if (!firstCell) buf.add(' | ') // §4.6 — cells joined with ' | ', outer pipes dropped
buf.append(paragraphText(cell.children))
firstCell = false
}
firstRow = false
}
return buf
}
// Render one block node into its own buffer. Returns an empty buffer for anything §4 omits.
function blockBuf (node, depth = 0) {
switch (node.type) {
case 'paragraph':
return paragraphText(node.children)
// §4.2 — ATX and setext both reduce to the heading text; no extra spacing, no case change.
case 'heading':
return paragraphText(node.children)
// §4.3 — contents verbatim: no fence, no info string, no re-indent. Not scanned for
// hashtags or URLs, for the same reason as an inline code span.
case 'code': {
const buf = new Buf()
buf.add(String(node.value || '').replace(/\n+$/, ''))
return buf
}
// §4.4 — the marker is removed and nested quotes flatten. No '> ' prefix is invented.
case 'blockquote': {
const buf = new Buf()
let first = true
for (const child of node.children || []) {
const sub = blockBuf(child, depth)
if (sub.length === 0) continue
if (!first) buf.add('\n\n')
buf.append(sub)
first = false
}
return buf
}
case 'list':
return listBlock(node, depth)
case 'table':
return tableBlock(node)
// §4.7 — a thematic break is omitted entirely, including its separator.
case 'thematicBreak':
return new Buf()
// §3.9 — the definition block is dropped along with its reference.
case 'footnoteDefinition':
return new Buf()
// §4.8 — a directive contributes its CONTENTS; the fences and the name are removed.
case 'containerDirective':
case 'leafDirective':
case 'textDirective': {
const buf = new Buf()
let first = true
for (const child of node.children || []) {
const sub = blockBuf(child, depth)
if (sub.length === 0) continue
if (!first) buf.add('\n\n')
buf.append(sub)
first = false
}
return buf
}
// §3.10 — block HTML is its literal source text.
case 'html': {
const buf = new Buf()
buf.add(String(node.value || ''))
return buf
}
default: {
if (node.children) {
const buf = new Buf()
let first = true
for (const child of node.children) {
const sub = blockBuf(child, depth)
if (sub.length === 0) continue
if (!first) buf.add('\n\n')
buf.append(sub)
first = false
}
return buf
}
const buf = new Buf()
if (typeof node.value === 'string') buf.add(node.value)
return buf
}
}
}
// ── normalisation (§5) ────────────────────────────────────────────────────────
// Every §5 rule is a DELETION, so normalising is a filter that also yields old->new index map.
// Carrying marks across the map is exact; re-finding them in the output would not be.
function normalise (text) {
const keep = []
const map = new Array(text.length + 1).fill(-1)
// §5.2 trailing whitespace per line, §5.3 3+ newlines -> 2, §5.4 outer trim.
const lines = []
let lineStart = 0
for (let i = 0; i <= text.length; i++) {
if (i === text.length || text[i] === '\n') {
lines.push([lineStart, i])
lineStart = i + 1
}
}
let newlineRun = 0
let emitted = false
for (let li = 0; li < lines.length; li++) {
const [s, e] = lines[li]
let end = e
while (end > s && (text[end - 1] === ' ' || text[end - 1] === '\t')) end-- // §5.2
const blank = end === s
if (blank) { newlineRun++; continue }
if (emitted) {
// §5.3 — a run of blank lines collapses so that at most two newlines survive.
keep.push(-1)
if (newlineRun > 0) keep.push(-1)
}
newlineRun = 0
for (let i = s; i < end; i++) keep.push(i)
emitted = true
}
let out = ''
for (const idx of keep) {
if (idx === -1) { out += '\n'; continue }
map[idx] = out.length
out += text[idx]
}
map[text.length] = out.length
// ⛔ TWO maps, because a deleted character means OPPOSITE things at each end of a range.
//
// Only KEPT characters were mapped above; every character §5 deleted — the newlines a block
// separator is emitted from, trailing whitespace, collapsed blank lines — still has an index a
// facet range may land on. A facet ending exactly at a block boundary has its `end` on such a
// character. Leaving it unmapped DROPPED the facet; mapping it forward to the next kept character
// made the facet SWALLOW the separator (byteEnd 8 where 6 was correct). Both were wrong, in
// opposite directions, which is the tell that one map cannot serve both ends.
//
// startMap a deleted start moves FORWARD to the next kept character — the range begins at
// real content rather than in a gap.
// endMap a deleted end stays BEHIND, just past the last kept character before it — the
// range ends where its content ended, not where the next one begins.
//
// Caught by `facet-at-end-of-block`: no vector had a facet at the end of a non-final block, and
// the fuzzer only generates single-block input. The CLI found it on its first real file.
const startMap = new Array(text.length + 1)
const endMap = new Array(text.length + 1)
startMap[text.length] = out.length
for (let i = text.length - 1; i >= 0; i--) {
startMap[i] = map[i] >= 0 ? map[i] : startMap[i + 1]
}
endMap[0] = 0
for (let i = 1; i <= text.length; i++) {
endMap[i] = map[i - 1] >= 0 ? map[i - 1] + 1 : endMap[i - 1]
}
// §5.4 — leading whitespace of the first kept line is not removed by the per-line pass above
// (only trailing is), so trim the assembled result and shift both maps to match.
const lead = out.length - out.replace(/^\s+/, '').length
if (lead > 0) {
out = out.slice(lead)
for (let i = 0; i < startMap.length; i++) {
startMap[i] = Math.max(0, startMap[i] - lead)
endMap[i] = Math.max(0, endMap[i] - lead)
}
}
const trimmed = out.replace(/\s+$/, '')
if (trimmed.length !== out.length) {
for (let i = 0; i < startMap.length; i++) {
if (startMap[i] > trimmed.length) startMap[i] = trimmed.length
if (endMap[i] > trimmed.length) endMap[i] = trimmed.length
}
out = trimmed
}
return { text: out, startMap, endMap }
}
const byteLen = (s) => {
let n = 0
for (let i = 0; i < s.length; i++) {
const c = s.codePointAt(i)
if (c > 0xffff) i++
n += c < 0x80 ? 1 : c < 0x800 ? 2 : c < 0x10000 ? 3 : 4
}
return n
}
// ── the strip ─────────────────────────────────────────────────────────────────
export function strip (source) {
// §2 — total: never throws. Anything unparseable degrades to its literal text.
let src
try {
src = String(source == null ? '' : source).replace(/\r\n?/g, '\n') // §5.1, on the INPUT
} catch {
return { text: '', facets: [] }
}
let tree
try {
tree = fromMarkdown(src, MICROMARK)
} catch {
return { text: src, facets: [] }
}
// §4.1 — blocks joined with exactly "\n\n"; a block producing empty text is omitted along
// with its separator, which is why each renders into its own buffer first.
const doc = new Buf()
let first = true
try {
for (const node of tree.children || []) {
const b = blockBuf(node, 0)
if (b.length === 0) continue
if (!first) doc.add('\n\n')
doc.append(b)
first = false
}
} catch {
return { text: '', facets: [] }
}
const { text, startMap, endMap } = normalise(doc.text)
// §6.4 — offsets are computed against the FINAL text, and only now converted to bytes.
const facets = []
for (const m of doc.marks) {
const s = startMap[m.start]
const e = endMap[m.end]
if (s === undefined || e === undefined || s < 0 || e < 0 || e <= s) continue // §6.3
// §6.5 — exactly one feature per facet. Flat, so a multi-feature facet is not expressible
// rather than being forbidden by a rule someone has to remember.
facets.push({
byteStart: byteLen(text.slice(0, s)),
byteEnd: byteLen(text.slice(0, e)),
type: m.feature.type,
value: m.feature.value
})
}
// §6.1 sorted by byteStart; §6.2 on a tie the longer wins, then mention > link > tag.
const rank = (f) => (f.type === 'mention' ? 0 : f.type === 'link' ? 1 : 2)
facets.sort((a, b) =>
a.byteStart - b.byteStart ||
b.byteEnd - a.byteEnd ||
rank(a) - rank(b))
// §6.2 — no overlaps: the winner of the sort above keeps its range, later overlaps are dropped.
const out = []
let last = -1
for (const f of facets) {
if (f.byteStart < last) continue
out.push(f)
last = f.byteEnd
}
return { text, facets: out }
}
export default strip