5926195 · favela-surface-android

Changed files (9) +155 -12

lib/bootstrap.js +30 -0

…	…	@@ -0,0 +1,30 @@
	1	// The worklet bundle's entry. Registers any OTA-staged native addons, then starts the node.
	2	//
	3	// ⛔ THIS IS WHAT MAKES NATIVE UPDATES DELIVERABLE WITHOUT A SIDELOAD. A `linked:` bundle NAMES the
	4	// sonames the APK must supply, so for as long as only the APK could satisfy them, any native change
	5	// meant a 154 MB sideload. Bare's Addon class removes that: `new Addon(fileURL)` dlopens an ABSOLUTE
	6	// PATH, and Addon.cache — keyed by url.href and consulted by the `linked:` branch of Addon.resolve —
	7	// lets a staged file answer to the soname the bundle asks for.
	8	//
	9	// ⛔ AND IT LIVES IN THE BUNDLE, NOT THE APK. A preloader that shipped only in the shell would put the
	10	// capability behind an APK release, which is the cost this exists to remove. It does not need to be
	11	// there: a module inside a bundle knows where its bundle is (import.meta.url is
	12	// `file:///…/<hash>.bundle/lib/bootstrap.js`), so it finds the addons staged beside it unaided. That
	13	// means THIS file reaches a device over the OTA that already works, on the APK already installed.
	14	//
	15	// ⛔ THE ONLY STATIC IMPORT THAT COSTS AN ADDON IS bare-fs, through preload-addons.js. Everything
	16	// imported here runs BEFORE the staged addons are registered, so it must already be satisfiable by the
	17	// APK on the device. Keeping that surface at one soname is what decides how many future releases can
	18	// carry native code without a new APK.
	19	import { addonsDirFor } from './host/loader-core.js'
	20	import { preloadAddons } from './host/preload-addons.js'
	21
	22	const dir = addonsDirFor(import.meta.url)
	23	const n = preloadAddons({ dir, say: (m) => console.log('[bootstrap] ' + m) })
	24	console.log('[bootstrap] ' + (dir === null ? 'not running from a bundle — no staged addons' : 'preloaded ' + n + ' staged addon(s) from ' + dir))
	25
	26	// ⛔ DYNAMIC, and that is the entire point. A static import is hoisted and evaluated before the code
	27	// above, so every addon the node needs would be resolved — and dlopened by soname against the APK —
	28	// before a single staged one was registered. Importing here, after preloadAddons has returned, is the
	29	// ordering the whole mechanism depends on.
	30	await import('./node.js')

lib/host/choose-bundle.js +1 -1

…	…	@@ -112,7 +112,7 @@ export function chooseBundle ({ current = null, pending = null, failed = [], ass
112	112	//
113	113	// Null and never '' — a loader has to tell "this release has no addons" (every release published so
114	114	// far, and a marker written before addons existed) from "the addons did not arrive".
115		addons: Array.isArray(current.addons) && current.addons.length > 0 ? current.treeHash + '.addons' : null,
	115	addons: Array.isArray(current.addons) && current.addons.length > 0 ? current.file + '.addons' : null,
116	116	markPending: true,
117	117	condemn,
118	118	reason: 'staged bundle available'

lib/host/loader-core.js +36 -0

…	…	@@ -45,3 +45,39 @@ export function runLoader ({ argv, preload, load, say = () => {} }) {
45	45	say('preloaded ' + n + ' staged addon(s); starting ' + bundle)
46	46	return load(bundle)
47	47	}
	48
	49	// ⛔ THE PIECE THAT REMOVES THE SIDELOAD. A preloader that only lives in the APK means gaining this
	50	// capability costs a new APK — which is the very cost the work exists to remove. It does not have to:
	51	// `new Addon(fileURL)` dlopens an absolute path and needs no filesystem module, and a module inside a
	52	// bundle already knows where its bundle is. Measured on a real packed bundle:
	53	//
	54	// import.meta.url = file:///…/L-boot/probe.bundle/selfloc-probe.mjs
	55	//
	56	// So a STAGED bundle can locate the addons staged beside it with no help from the shell, which means
	57	// the preloader ships as an ordinary JS release over the OTA that already works — to the APK that is
	58	// already on the device.
	59	//
	60	// ⛔ `.bundle` MUST BE A WHOLE SEGMENT that ends the bundle file. Substring matching would take
	61	// `/x/my.bundles/lib/a.js` or `/x/a.bundle.old/…` for a bundle and derive an addon directory that does
	62	// not exist — a silent no-op, which is the worst kind of wrong here.
	63	export function bundlePathFromUrl (metaUrl) {
	64	if (typeof metaUrl !== 'string' \|\| metaUrl === '') return null
	65	const path = metaUrl.startsWith('file://') ? metaUrl.slice('file://'.length) : metaUrl
	66	const parts = path.split('/')
	67	for (let i = 0; i < parts.length - 1; i++) {
	68	if (parts[i].endsWith('.bundle')) return parts.slice(0, i + 1).join('/')
	69	}
	70	return null
	71	}
	72
	73	/**
	74	* Where this module's own bundle keeps its staged addons, or null when not running from a bundle.
	75	*
	76	* ⛔ NAMED OFF THE BUNDLE FILE, not the tree hash, precisely so it is DERIVABLE. A name keyed on the
	77	* tree hash would have to be handed in by the shell, and a shell old enough to predate this cannot hand
	78	* in anything — which would put the capability back behind an APK release.
	79	*/
	80	export function addonsDirFor (metaUrl) {
	81	const bundle = bundlePathFromUrl(metaUrl)
	82	return bundle === null ? null : bundle + '.addons'
	83	}

lib/host/ota.js +8 -4

…	…	@@ -253,7 +253,11 @@ export function bundleFetcher ({ base = DEFAULT_BASE, repo = DEFAULT_REPO, doFet
253	253	*/
254	254	export function bundleStore ({ dir }) {
255	255	const markerPath = (treeHash) => path.join(dir, treeHash + '.json')
256		const addonsDir = (treeHash) => path.join(dir, treeHash + '.addons')
	256	// ⛔ NAMED OFF THE BUNDLE FILE, NOT THE TREE HASH. The staged bundle derives this itself from
	257	// import.meta.url (host/loader-core.js addonsDirFor), which is what lets the preloader ship as an
	258	// ordinary JS release to the APK already on the device. A tree-hash name would have to be handed in
	259	// by the shell, and a shell old enough to predate this cannot hand in anything.
	260	const addonsDir = (file) => path.join(dir, file + '.addons')
257	261	// The release currently being installed. pgh-runtime's installTree does not pass the release down to
258	262	// the store, but it emits `updating` with it immediately beforehand — so the caller notes it here and
259	263	// materialize can stamp the staged marker with a comparable build time. Without that the shell has no
…	…	@@ -284,7 +288,7 @@ export function bundleStore ({ dir }) {
284	288	// up, and the level where it cannot be fixed remotely.
285	289	const addons = Array.isArray(m.addons) ? m.addons : []
286	290	for (const name of addons) {
287		try { if (fs.statSync(path.join(addonsDir(treeHash), name)).size === 0) return false } catch { return false }
	291	try { if (fs.statSync(path.join(addonsDir(m.file), name)).size === 0) return false } catch { return false }
288	292	}
289	293	return true
290	294	},
…	…	@@ -307,7 +311,7 @@ export function bundleStore ({ dir }) {
307	311	addonsFor (treeHash) {
308	312	const m = readMarker(treeHash)
309	313	if (!m \|\| !Array.isArray(m.addons) \|\| m.addons.length === 0) return null
310		return addonsDir(treeHash)
	314	return addonsDir(m.file)
311	315	},
312	316
313	317	async materialize (rootHash, entries, { fetchContent, fetchTree } = {}) {
…	…	@@ -364,7 +368,7 @@ export function bundleStore ({ dir }) {
364	368	if (typeof fetchTree !== 'function') {
365	369	throw new Error('release carries ' + ADDONS_NAME + '/ but no fetchTree to read it with')
366	370	}
367		const adir = addonsDir(rootHash)
	371	const adir = addonsDir(want.childHash + '.bundle')
368	372	fs.mkdirSync(adir, { recursive: true })
369	373	const sub = await fetchTree(addonDirEntry.childHash)
370	374	for (const e of (sub \|\| [])) {

lib/host/preload-addons.js +12 -4

…	…	@@ -25,9 +25,12 @@
25	25	// ⚠ Addon.cache / Addon.load / Addon.resolve are all marked @deprecated upstream while bare-module still
26	26	// calls them on every `require.addon`. If they are ever removed, the replacement is the same idea one
27	27	// layer down — construct the Addon and hand it to whatever resolves addons then.
	28	// ⛔ ONE ADDON DEPENDENCY, DELIBERATELY. This runs BEFORE the staged addons are registered, so every
	29	// addon it imports must already be satisfiable by the APK on the device. bare-fs is foundational and
	30	// its soname changes only when we bump it; bare-path and bare-url were dropped for string joins and a
	31	// hand-built file: URL precisely to keep this surface at one. The fewer sonames the preloader needs,
	32	// the more releases can carry new native code without a new APK.
28	33	import fs from 'bare-fs'
29		import path from 'bare-path'
30		import { pathToFileURL } from 'bare-url'
31	34
32	35	// ⛔ THE FILENAME IS THE WHOLE CONTRACT. Each staged file is named after the soname it satisfies, so the
33	36	// directory listing IS the mapping — no manifest to fall out of step with the bytes, and no parsing of
…	…	@@ -67,12 +70,17 @@ export function preloadAddons ({ dir, cache = Bare.Addon.cache, Addon = Bare.Add
67	70	// `.so` and nothing else. A `.part` is an interrupted write — the store renames into place, so a
68	71	// leftover one means a run that died, and dlopening a truncated ELF is a crash rather than an error.
69	72	if (!name.endsWith('.so')) continue
70		const file = path.join(dir, name)
	73	// ⛔ FIRST REGISTRATION WINS. Two preloaders can run in one process — the APK's loader before the
	74	// bundle starts, and the bundle's own bootstrap once it does — and both read the same directory.
	75	// A second `new Addon(sameUrl)` would dlopen the same file and re-run its init. The cache is keyed
	76	// by href, so this check is exact.
	77	if (cache[addonHref(name)]) continue
	78	const file = dir + '/' + name
71	79	try {
72	80	// ⛔ ONE addon failing must not take the boot with it. The bundle still cannot run without it, but
73	81	// this is not the place that discovers that — the failure surfaces at the require that needs it,
74	82	// where the message names the module rather than the loader.
75		cache[addonHref(name)] = new Addon(pathToFileURL(file))
	83	cache[addonHref(name)] = new Addon(new URL('file://' + file))
76	84	n++
77	85	} catch (err) {
78	86	say('preload: ' + name + ' failed to load — ' + ((err && err.message) \|\| err))

package.json +1 -1

…	…	@@ -5,7 +5,7 @@
5	5	"type": "module",
6	6	"private": true,
7	7	"scripts": {
8		"pack:worklet": "bare-pack --preset android --defer bare-abstract-socket --out android/app/src/main/assets/worklet.bundle lib/node.js",
	8	"pack:worklet": "bare-pack --preset android --defer bare-abstract-socket --out android/app/src/main/assets/worklet.bundle lib/bootstrap.js",
9	9	"test": "brittle-bare test/*.test.js",
10	10	"stage:addons": "node scripts/stage-addons.mjs",
11	11	"gen:tree": "bare scripts/gen-tree-manifest.mjs",

test/lifecycle.test.js +2 -2

…	…	@@ -187,7 +187,7 @@ test('a staged bundle carrying addons says where they are', (t) => {
187	187	const staged = { treeHash: 'a'.repeat(64), file: 'abc.bundle', builtAt: 2000, addons: ['libbare-fs.4.8.1.so'] }
188	188	const r = chooseBundle({ current: staged, assetBuiltAt: 1000 })
189	189	t.is(r.use, 'staged')
190		t.is(r.addons, 'a'.repeat(64) + '.addons', 'named off the TREE hash, as bundleStore writes it')
	190	t.is(r.addons, 'abc.bundle.addons', 'named off the BUNDLE FILE, so a staged bundle can derive it itself')
191	191	})
192	192
193	193	test('a staged bundle with no addons reports none', (t) => {
…	…	@@ -235,7 +235,7 @@ test('a staged bundle whose demands only the STAGED addons satisfy runs', (t) =>
235	235	}
236	236	const r = chooseBundle({ current, assetBuiltAt: 1000, apkAddons: APK })
237	237	t.is(r.use, 'staged', 'the new addon came with the release')
238		t.is(r.addons, 'b'.repeat(64) + '.addons', 'and its directory is reported')
	238	t.is(r.addons, 'x.bundle.addons', 'and its directory is reported')
239	239	})
240	240
241	241	test('a demand in NEITHER source is refused, and says which', (t) => {

test/loader.test.js +46 -0

…	…	@@ -73,3 +73,49 @@ test('a bundle that throws is NOT caught — the watchdog must see it', (t) => {
73	73	load: () => { throw new Error('bundle is corrupt') }
74	74	}), /bundle is corrupt/)
75	75	})
	76
	77	// ── the in-bundle bootstrap ───────────────────────────────────────────────────────────────────────
	78	//
	79	// ⛔ THE APK LOADER IS NOT ENOUGH, AND THAT WAS THE WHOLE POINT. A preloader that only ships inside the
	80	// APK means gaining the capability costs a sideload — exactly the cost this work exists to remove. It
	81	// has to be possible to ship the preloader ITSELF as an ordinary JS release over the OTA that already
	82	// works today.
	83	//
	84	// It is, because `new Addon(fileURL)` needs no filesystem module to dlopen a path, and a module inside
	85	// a bundle knows where its bundle is: import.meta.url is `file:///…/<hash>.bundle/lib/bootstrap.js`.
	86	// Measured on a real packed bundle, not assumed. So a staged bundle can find the addons staged beside
	87	// it with no help from the shell — which is what makes this deliverable to the APK already on a device.
	88	import { bundlePathFromUrl, addonsDirFor } from '../lib/host/loader-core.js'
	89
	90	test('a module inside a bundle can find its own bundle file', (t) => {
	91	t.is(
	92	bundlePathFromUrl('file:///data/user/0/app/files/bundles/abc123.bundle/lib/bootstrap.js'),
	93	'/data/user/0/app/files/bundles/abc123.bundle'
	94	)
	95	t.is(
	96	bundlePathFromUrl('file:///d/b/x.bundle/node_modules/deep/nested/mod.js'),
	97	'/d/b/x.bundle',
	98	'however deep the module sits'
	99	)
	100	})
	101
	102	test('running from SOURCE is not a bundle, and says so', (t) => {
	103	// The test suite itself, and any laptop run. Null rather than a guess: preloading from a directory
	104	// invented out of a source path would be a silent no-op at best.
	105	t.is(bundlePathFromUrl('file:///repo/favela-surface-android/lib/bootstrap.js'), null)
	106	t.is(bundlePathFromUrl(''), null)
	107	t.is(bundlePathFromUrl(null), null)
	108	})
	109
	110	test('a path with .bundle inside a DIRECTORY name is not mistaken for the bundle', (t) => {
	111	t.is(bundlePathFromUrl('file:///x/my.bundles/lib/a.js'), null, 'my.bundles is not a bundle')
	112	t.is(bundlePathFromUrl('file:///x/a.bundle.old/lib/a.js'), null, 'a.bundle.old is not a bundle')
	113	})
	114
	115	test('the addon directory is derived from the bundle FILE, so nothing has to be told', (t) => {
	116	t.is(
	117	addonsDirFor('file:///data/user/0/app/files/bundles/abc123.bundle/lib/bootstrap.js'),
	118	'/data/user/0/app/files/bundles/abc123.bundle.addons'
	119	)
	120	t.is(addonsDirFor('file:///repo/lib/bootstrap.js'), null, 'no bundle, no addons')
	121	})

test/preload-addons.test.js +19 -0

…	…	@@ -124,3 +124,22 @@ test('REAL: a genuine addon loads from an arbitrary staged path', (t) => {
124	124	t.ok(addon, 'registered under the soname a bundle would ask for')
125	125	t.ok(addon.exports && typeof addon.exports === 'object', 'and it really initialised — it has exports')
126	126	})
	127
	128	test('an addon already registered is NOT loaded twice', (t) => {
	129	// ⛔ TWO PRELOADERS CAN RUN IN ONE PROCESS. The APK's loader registers staged addons before starting
	130	// the bundle, and the bundle's own bootstrap then does the same from the same directory — an APK new
	131	// enough to have the loader running a bundle new enough to bootstrap itself. A second
	132	// `new Addon(sameUrl)` would dlopen the same file and re-run its init, so the first registration wins
	133	// and the second is skipped. Addon.cache is keyed by href, which makes that check exact.
	134	const dir = tmpdir('twice')
	135	fs.writeFileSync(path.join(dir, 'libbare-fs.4.8.1.so'), 'x')
	136
	137	let constructed = 0
	138	const Addon = function (url) { constructed++; this.url = url }
	139	const cache = Object.create(null)
	140
	141	t.is(preloadAddons({ dir, cache, Addon }), 1, 'first pass registers it')
	142	t.is(constructed, 1)
	143	t.is(preloadAddons({ dir, cache, Addon }), 0, 'second pass registers nothing new')
	144	t.is(constructed, 1, 'and never constructs a second Addon for the same soname')
	145	})

favela-surface-android commits / 5926195

Changed files (9) +155 -12