53ad1eb · stremio-vsembed

Changed files (3) +271 -8

.gitignore +1 -0

…	…	@@ -14,3 +14,4 @@ node_helper/node_modules/
14	14	vidcore_reputation.json
15	15	source_breaker.json
16	16	mp4-cache/
	17	library_verified.json

addon.py +118 -6

…	…	@@ -595,7 +595,7 @@ TV_GENRES = [
595	595
596	596	MANIFEST = {
597	597	"id": "org.tabertron.vsembed",
598		"version": "0.13.2",
	598	"version": "0.13.3",
599	599	"name": "Tabertron VsEmbed",
600	600	"description": "Self-hosted HLS extractor with TMDB browse + search, played via vidcore",
601	601	"types": ["movie", "series"],
…	…	@@ -1399,11 +1399,15 @@ def _library_meta(kind: str, ident: str, season: int = 0,
1399	1399	year = (show.get("first_air_date") or "")[:4]
1400	1400	ep = _tmdb_get(f"tv/{tmdb_id}/season/{season}/episode/{episode}")
1401	1401	episode_title = ep.get("name") or ""
	1402	# Minutes, and free: this is the response we already had to fetch
	1403	# for the episode title, and it is cached with the rest of the meta.
	1404	runtime = ep.get("runtime")
1402	1405	else:
1403	1406	film = _tmdb_get(f"movie/{tmdb_id}")
1404	1407	title = film.get("title") or ""
1405	1408	year = (film.get("release_date") or "")[:4]
1406	1409	episode_title = ""
	1410	runtime = film.get("runtime")
1407	1411	except Exception as exc:
1408	1412	# Deliberately NOT cached — see docstring.
1409	1413	log.warning("library meta failed for %s: %s", key, exc)
…	…	@@ -1413,11 +1417,103 @@ def _library_meta(kind: str, ident: str, season: int = 0,
1413	1417	_library_meta_cache[key] = None
1414	1418	return None
1415	1419	meta = {"tmdb_id": tmdb_id, "title": title,
1416		"year": year, "episode_title": episode_title}
	1420	"year": year, "episode_title": episode_title, "runtime": runtime}
1417	1421	_library_meta_cache[key] = meta
1418	1422	return meta
1419	1423
1420	1424
	1425	# Whether a copy already on disk can be trusted, measured against TMDB's
	1426	# `runtime`. A LOOSER floor than REMUX_MIN_COMPLETENESS, deliberately, because
	1427	# it compares a different kind of number: the remux check compares an output
	1428	# against the EXTINF sum of the playlist it came from -- a measurement, which
	1429	# agreed to 0.003% on real data -- whereas `runtime` is editorial metadata. The
	1430	# fourteen known-good copies in the library measured 92-101% of it. A floor
	1431	# tight enough for the former would reject nearly all of the latter.
	1432	LIBRARY_MIN_COMPLETENESS = 0.85
	1433
	1434	# Copies whose length was measured against their own source at the moment they
	1435	# were written. That is strictly better evidence than TMDB's runtime, so it
	1436	# WINS over it -- and it has to, or a title whose upstream genuinely only holds
	1437	# a short version would loop for ever: the runtime gate rejects the copy, the
	1438	# re-ingest fetches the same short source, the playlist check passes it, and the
	1439	# runtime gate rejects it again on the next play.
	1440	LIBRARY_VERIFIED_FILE = os.path.join(os.path.dirname(os.path.abspath(__file__)),
	1441	"library_verified.json")
	1442
	1443	_duration_cache: dict[tuple, Optional[float]] = {}
	1444
	1445
	1446	def _load_verified() -> dict:
	1447	"""Never raises: like the job queue, this is evidence, not load-bearing."""
	1448	try:
	1449	with open(LIBRARY_VERIFIED_FILE) as f:
	1450	data = json.load(f)
	1451	return data if isinstance(data, dict) else {}
	1452	except Exception:
	1453	return {}
	1454
	1455
	1456	def _record_verified(path: str, seconds: float, expected: float) -> None:
	1457	try:
	1458	store = _load_verified()
	1459	store[os.path.abspath(path)] = {"seconds": seconds, "expected": expected,
	1460	"ts": time.time()}
	1461	with open(LIBRARY_VERIFIED_FILE, "w") as f:
	1462	json.dump(store, f)
	1463	except Exception as exc:
	1464	log.warning("verified store save failed: %s", exc)
	1465
	1466
	1467	def _cached_media_duration(path: str) -> Optional[float]:
	1468	"""_media_duration, keyed on the file's identity rather than its name.
	1469
	1470	Every play of a local copy asks this, so it must not spawn an ffprobe each
	1471	time. Size and mtime are in the key so a re-ingest invalidates it for free.
	1472	"""
	1473	try:
	1474	st = os.stat(path)
	1475	except OSError:
	1476	return None
	1477	key = (os.path.abspath(path), st.st_size, int(st.st_mtime))
	1478	if key not in _duration_cache:
	1479	_duration_cache[key] = _media_duration(path)
	1480	return _duration_cache[key]
	1481
	1482
	1483	def _copy_is_complete(path: str, meta: Optional[dict]) -> bool:
	1484	"""Is the file on disk a whole copy, or the 21% of an episode we published?
	1485
	1486	Both _library_has and _playable_path go through here, and they MUST agree:
	1487	if only the serving side rejected a partial, the copy would stop being
	1488	served but the ingest would still report "already held" and never replace
	1489	it. Rejecting it in both is what makes the repair automatic.
	1490
	1491	Everything unknown resolves to True. This is the opposite asymmetry from
	1492	_remux_to_mp4, and for a reason: there we had just written the file and knew
	1493	exactly how long it should be, so an unreadable result was a broken file.
	1494	Here the file has been serving fine and the only doubt comes from a third
	1495	party's metadata, so a check that cannot run must not condemn it.
	1496	"""
	1497	if os.path.abspath(path) in _load_verified():
	1498	return True
	1499	runtime = (meta or {}).get("runtime")
	1500	try:
	1501	expected = float(runtime) * 60
	1502	except (TypeError, ValueError):
	1503	return True # TMDB has no runtime for this title
	1504	if expected <= 0:
	1505	return True
	1506	actual = _cached_media_duration(path)
	1507	if actual is None:
	1508	return True # unprobeable, but it was here first
	1509	if actual >= expected * LIBRARY_MIN_COMPLETENESS:
	1510	return True
	1511	log.warning("library: %s holds %.0fs of an expected %.0fs (%.0f%%) — "
	1512	"not treating it as a copy",
	1513	os.path.basename(path), actual, expected, actual / expected * 100)
	1514	return False
	1515
	1516
1421	1517	def _library_has(kind: str, ident: str, season: int = 0,
1422	1518	episode: int = 0) -> bool:
1423	1519	"""True if the library already holds this episode, at ANY height.
…	…	@@ -1425,6 +1521,8 @@ def _library_has(kind: str, ident: str, season: int = 0,
1425	1521	Height is deliberately absent from the library path: the library holds one
1426	1522	copy per episode. A later request at a different quality is a no-op rather
1427	1523	than a second file, and nothing is ever deleted to make room for an upgrade.
	1524
	1525	A partial copy does not count, so an ingest is free to replace one.
1428	1526	"""
1429	1527	meta = _library_meta(kind, ident, season, episode)
1430	1528	if not meta:
…	…	@@ -1432,9 +1530,11 @@ def _library_has(kind: str, ident: str, season: int = 0,
1432	1530	dest = _library_path(kind, meta["title"], meta["year"], meta["tmdb_id"],
1433	1531	season, episode, meta["episode_title"])
1434	1532	try:
1435		return os.path.getsize(dest) > 0
	1533	if os.path.getsize(dest) <= 0:
	1534	return False
1436	1535	except OSError:
1437	1536	return False
	1537	return _copy_is_complete(dest, meta)
1438	1538
1439	1539
1440	1540	# How much of the source a remux must actually contain before it is published.
…	…	@@ -1548,6 +1648,11 @@ def _remux_to_mp4(variant_url: str, out_path: str) -> None:
1548	1648	f"{expected:.0f}s ({(actual or 0) / expected * 100:.0f}%) — "
1549	1649	f"not published")
1550	1650	log.info("remux verified: %.0fs of %.0fs", actual, expected)
	1651	# Recorded against the FINAL path, and only for a copy measured against
	1652	# its own source. _copy_is_complete trusts this over TMDB's runtime,
	1653	# which is what stops a genuinely-short upstream from being re-fetched
	1654	# on every play for ever.
	1655	_record_verified(out_path, actual, expected)
1551	1656	os.replace(tmp, out_path)
1552	1657
1553	1658
…	…	@@ -1639,21 +1744,28 @@ def _playable_path(kind: str, ident: str, season: int = 0, episode: int = 0,
1639	1744
1640	1745	Falls back cleanly when TMDB is unreachable — _library_meta returns None and
1641	1746	the legacy path still answers.
	1747
	1748	"Complete" is enforced here rather than at the row, because Jacktook caches
	1749	the source list for 24h: a row can be a day stale, but this is on the path
	1750	of every play. It is the only place that can refuse a partial copy no matter
	1751	which cached row asked for it.
1642	1752	"""
1643	1753	meta = _library_meta(kind, ident, season, episode)
1644	1754	if meta:
1645	1755	dest = _library_path(kind, meta["title"], meta["year"], meta["tmdb_id"],
1646	1756	season, episode, meta["episode_title"])
1647	1757	try:
1648		if os.path.getsize(dest) > 0:
	1758	if os.path.getsize(dest) > 0 and _copy_is_complete(dest, meta):
1649	1759	return dest
1650	1760	except OSError:
1651	1761	pass
1652	1762	legacy = _mp4_path(kind, ident, season, episode, height)
1653	1763	try:
1654		return legacy if os.path.getsize(legacy) > 0 else None
	1764	if os.path.getsize(legacy) > 0 and _copy_is_complete(legacy, meta):
	1765	return legacy
1655	1766	except OSError:
1656		return None
	1767	pass
	1768	return None
1657	1769
1658	1770
1659	1771	def _ensure_ingest(kind: str, ident: str, season: int = 0, episode: int = 0,

test_library_ingest.py +152 -2

…	…	@@ -30,6 +30,12 @@ import addon
30	30	@pytest.fixture(autouse=True)
31	31	def tmp_library(tmp_path, monkeypatch):
32	32	monkeypatch.setattr(addon, "LIBRARY_ROOT", str(tmp_path))
	33	# Redirected for every test, not just the ones that care: the verified store
	34	# lives beside addon.py in the deployed checkout, and a test that wrote to
	35	# the real one would quietly mark production files as trustworthy.
	36	monkeypatch.setattr(addon, "LIBRARY_VERIFIED_FILE",
	37	os.path.join(str(tmp_path), "library_verified.json"))
	38	monkeypatch.setattr(addon, "_duration_cache", {})
33	39	yield
34	40
35	41
…	…	@@ -187,8 +193,8 @@ def test_series_metadata_comes_from_the_tmdb_id(tmdb):
187	193	tmdb.responses["tv/138502"] = {"name": "X-Men '97", "first_air_date": "2024-03-20"}
188	194	tmdb.responses["tv/138502/season/2/episode/7"] = {"name": "Bright Eyes"}
189	195	m = addon._library_meta("series", "tmdb:138502", 2, 7)
190		assert m == {"tmdb_id": "138502", "title": "X-Men '97",
191		"year": "2024", "episode_title": "Bright Eyes"}
	196	assert m == {"tmdb_id": "138502", "title": "X-Men '97", "year": "2024",
	197	"episode_title": "Bright Eyes", "runtime": None}
192	198
193	199
194	200	def test_movie_metadata_comes_from_the_tmdb_id(tmdb):
…	…	@@ -765,3 +771,147 @@ def test_ingest_resolves_at_run_time_not_at_queue_time(ingest, monkeypatch):
765	771	lambda a, *k: calls.append(a) or "http://cdn/m.m3u8")
766	772	addon._run_ingest("series", "tmdb:1", 1, 1, 720)
767	773	assert calls, "the ingest must resolve for itself"
	774
	775
	776	# --- a partial copy already on disk is not a copy ---------------------------
	777	#
	778	# 0.13.2 stopped NEW partials being published. It could do nothing about the
	779	# three already in the library, and nothing about the way they were being
	780	# served: _local_redirect answers before the resolve, so Strange New Worlds
	781	# S04E04 was handed to the player from disk at 10.6 minutes while a complete
	782	# 50-minute upstream sat there unused.
	783	#
	784	# The gate goes in _playable_path and _library_has rather than in the row that
	785	# offers the copy, because Jacktook caches its source list for 24h -- a row can
	786	# be a day stale, but these are on the path of every single play.
	787	#
	788	# Both, and not just the serving side: if only _playable_path refused, the file
	789	# would stop being served and _library_has would still report "already held", so
	790	# the ingest that should replace it would decline to run. Refusing in both is
	791	# what makes the repair automatic.
	792
	793	@pytest.fixture
	794	def on_disk(tmdb, monkeypatch):
	795	"""A library copy of a 50-minute episode whose measured length is settable.
	796
	797	Returns (path, seconds) where seconds["value"] is what ffprobe would say.
	798	"""
	799	tmdb.responses["tv/1"] = {"name": "Show", "first_air_date": "2020-01-01"}
	800	tmdb.responses["tv/1/season/1/episode/1"] = {"name": "Pilot", "runtime": 50}
	801	meta = addon._library_meta("series", "tmdb:1", 1, 1)
	802	path = addon._library_path("series", meta["title"], meta["year"],
	803	meta["tmdb_id"], 1, 1, meta["episode_title"])
	804	os.makedirs(os.path.dirname(path), exist_ok=True)
	805	with open(path, "wb") as f:
	806	f.write(b"x" * 1024)
	807	seconds = {"value": 3000.0}
	808	monkeypatch.setattr(addon, "_media_duration", lambda p: seconds["value"])
	809	return path, seconds
	810
	811
	812	def test_metadata_carries_the_tmdb_runtime(tmdb):
	813	"""Free: it rides on the episode lookup the title already needed."""
	814	tmdb.responses["tv/1"] = {"name": "Show", "first_air_date": "2020-01-01"}
	815	tmdb.responses["tv/1/season/1/episode/1"] = {"name": "Pilot", "runtime": 51}
	816	assert addon._library_meta("series", "tmdb:1", 1, 1)["runtime"] == 51
	817
	818
	819	def test_a_partial_copy_is_not_served(on_disk):
	820	"""The SNW S04E04 shape: 10.6 minutes of a 51-minute episode on disk."""
	821	path, seconds = on_disk
	822	seconds["value"] = 636.7
	823	assert addon._playable_path("series", "tmdb:1", 1, 1) is None
	824
	825
	826	def test_a_partial_copy_does_not_block_its_own_replacement(on_disk):
	827	"""The half that makes the repair automatic rather than manual."""
	828	path, seconds = on_disk
	829	seconds["value"] = 636.7
	830	assert addon._library_has("series", "tmdb:1", 1, 1) is False
	831
	832
	833	def test_a_complete_copy_is_still_served(on_disk):
	834	path, seconds = on_disk
	835	assert addon._playable_path("series", "tmdb:1", 1, 1) == path
	836	assert addon._library_has("series", "tmdb:1", 1, 1) is True
	837
	838
	839	def test_editorial_drift_does_not_condemn_a_good_copy(on_disk):
	840	"""TMDB's runtime is not a measurement. The fourteen known-good copies in
	841	the library measured 92-101% of it, so the floor has to sit below that --
	842	which is why this is 0.85 and the remux check, comparing against a real
	843	EXTINF sum, is 0.99."""
	844	path, seconds = on_disk
	845	seconds["value"] = 3000.0 * 0.92
	846	assert addon._playable_path("series", "tmdb:1", 1, 1) == path
	847
	848
	849	def test_a_copy_verified_against_its_source_outranks_tmdb(on_disk):
	850	"""The loop-breaker. If upstream genuinely holds only a short version, the
	851	remux check passes it (it matched its own playlist) and a runtime gate would
	852	then reject it on every play, re-fetching the same short file for ever."""
	853	path, seconds = on_disk
	854	seconds["value"] = 636.7
	855	addon._record_verified(path, 636.7, 636.7)
	856	assert addon._playable_path("series", "tmdb:1", 1, 1) == path
	857	assert addon._library_has("series", "tmdb:1", 1, 1) is True
	858
	859
	860	def test_an_unknown_runtime_leaves_the_copy_alone(on_disk, tmdb, monkeypatch):
	861	"""TMDB does not always carry a runtime. Unknown must mean "leave it",
	862	never "condemn it" -- the opposite asymmetry to the remux check, because
	863	there we had just written the file and knew what it should be."""
	864	path, seconds = on_disk
	865	monkeypatch.setattr(addon, "_library_meta_cache", {})
	866	tmdb.responses["tv/1/season/1/episode/1"] = {"name": "Pilot"}
	867	seconds["value"] = 1.0
	868	assert addon._playable_path("series", "tmdb:1", 1, 1) == path
	869
	870
	871	def test_an_unprobeable_copy_is_left_alone(on_disk, monkeypatch):
	872	path, _ = on_disk
	873	monkeypatch.setattr(addon, "_media_duration", lambda p: None)
	874	assert addon._playable_path("series", "tmdb:1", 1, 1) == path
	875
	876
	877	def test_the_duration_is_probed_once_per_file(on_disk, monkeypatch):
	878	"""Every play of a local copy asks. An ffprobe per request would be a new
	879	cost on the one path that was fast."""
	880	path, _ = on_disk
	881	calls = []
	882	monkeypatch.setattr(addon, "_media_duration",
	883	lambda p: (calls.append(p), 3000.0)[1])
	884	addon._playable_path("series", "tmdb:1", 1, 1)
	885	addon._playable_path("series", "tmdb:1", 1, 1)
	886	assert len(calls) == 1
	887
	888
	889	def test_a_rewritten_file_is_probed_again(on_disk, monkeypatch):
	890	"""The cache key carries size and mtime, so a re-ingest invalidates it
	891	without anyone having to remember to."""
	892	path, _ = on_disk
	893	calls = []
	894	monkeypatch.setattr(addon, "_media_duration",
	895	lambda p: (calls.append(p), 3000.0)[1])
	896	addon._playable_path("series", "tmdb:1", 1, 1)
	897	with open(path, "wb") as f:
	898	f.write(b"y" * 4096)
	899	addon._playable_path("series", "tmdb:1", 1, 1)
	900	assert len(calls) == 2
	901
	902
	903	def test_a_verified_remux_records_what_it_measured(fake_ffmpeg, tmp_path):
	904	dest = os.path.join(str(tmp_path), "out.mp4")
	905	fake_ffmpeg.playlist_seconds = 3009.6
	906	fake_ffmpeg.output_seconds = 3009.5
	907	addon._remux_to_mp4("http://cdn/x.m3u8", dest)
	908	assert os.path.abspath(dest) in addon._load_verified()
	909
	910
	911	def test_a_refused_remux_records_nothing(fake_ffmpeg, tmp_path):
	912	dest = os.path.join(str(tmp_path), "out.mp4")
	913	fake_ffmpeg.playlist_seconds = 3009.6
	914	fake_ffmpeg.output_seconds = 636.7
	915	with pytest.raises(RuntimeError):
	916	addon._remux_to_mp4("http://cdn/x.m3u8", dest)
	917	assert addon._load_verified() == {}

stremio-vsembed commits / 53ad1eb

Changed files (3) +271 -8