| … | … | @@ -12,6 +12,8 @@ import random |
| 12 | 12 | import re |
| 13 | 13 | import shutil |
| 14 | 14 | import subprocess |
| 15 | import tempfile | |
| 16 | from concurrent.futures import ThreadPoolExecutor | |
| 15 | 17 | import threading |
| 16 | 18 | import time |
| 17 | 19 | from dataclasses import dataclass, field |
| … | … | @@ -597,7 +599,7 @@ TV_GENRES = [ |
| 597 | 599 | |
| 598 | 600 | MANIFEST = { |
| 599 | 601 | "id": "org.tabertron.vsembed", |
| 600 | "version": "0.17.1", | |
| 602 | "version": "0.18.0", | |
| 601 | 603 | "name": "Tabertron VsEmbed", |
| 602 | 604 | "description": "Self-hosted HLS extractor with TMDB browse + search, played via vidcore", |
| 603 | 605 | "types": ["movie", "series"], |
| … | … | @@ -1661,6 +1663,269 @@ def _media_duration(path: str) -> Optional[float]: |
| 1661 | 1663 | return None |
| 1662 | 1664 | |
| 1663 | 1665 | |
| 1666 | # A remux that stops producing bytes is dead, and the wall clock is the worst | |
| 1667 | # possible way to find out. 2026-09-03, No Other Choice (tmdb 639988): ffmpeg | |
| 1668 | # wrote 2.85 GB in ~9 minutes (~5.3 MB/s), then produced nothing for 36 more | |
| 1669 | # until MP4_REMUX_TIMEOUT killed it -- reported as "timed out after 2700 | |
| 1670 | # seconds", which reads as too-slow and was not. Five minutes without a byte, | |
| 1671 | # against a copy that runs at megabytes per second, is unambiguous. | |
| 1672 | MP4_STALL_TIMEOUT = 5 * 60 | |
| 1673 | # How often the watchdog looks. Cheap (one stat), and the resolution only needs | |
| 1674 | # to be fine relative to the stall window. | |
| 1675 | MP4_STALL_POLL = 2.0 | |
| 1676 | ||
| 1677 | ||
| 1678 | class RemuxStalled(RuntimeError): | |
| 1679 | """The output stopped growing. Distinct from RemuxTimedOut on purpose: one | |
| 1680 | says the source went quiet, the other says the copy is genuinely too long, | |
| 1681 | and they want different responses.""" | |
| 1682 | ||
| 1683 | ||
| 1684 | class RemuxTimedOut(RuntimeError): | |
| 1685 | """The copy exceeded its wall-clock budget while still making progress.""" | |
| 1686 | ||
| 1687 | ||
| 1688 | def _read_back(f) -> str: | |
| 1689 | """Everything written to an open temp file, from the start. Never raises: | |
| 1690 | this only ever supplies a diagnostic, and a diagnostic that can fail the | |
| 1691 | operation it describes is worse than no diagnostic.""" | |
| 1692 | try: | |
| 1693 | f.seek(0) | |
| 1694 | return f.read() | |
| 1695 | except Exception: | |
| 1696 | return "" | |
| 1697 | ||
| 1698 | ||
| 1699 | class _RemuxResult: | |
| 1700 | """What subprocess.run would have returned, minus the parts nothing reads. | |
| 1701 | ||
| 1702 | Named rather than a tuple because the caller checks `.returncode` and | |
| 1703 | `.stderr`, and those two being positional is how they get swapped. | |
| 1704 | """ | |
| 1705 | ||
| 1706 | __slots__ = ("returncode", "stderr") | |
| 1707 | ||
| 1708 | def __init__(self, returncode: int, stderr: str): | |
| 1709 | self.returncode = returncode | |
| 1710 | self.stderr = stderr | |
| 1711 | ||
| 1712 | ||
| 1713 | def _run_remux(cmd: list[str], out_path: str, stall_timeout: float, | |
| 1714 | hard_timeout: float): | |
| 1715 | """Run `cmd`, killing it if `out_path` stops growing for `stall_timeout`. | |
| 1716 | ||
| 1717 | The watchdog is OUTSIDE ffmpeg deliberately. ffmpeg's own -rw_timeout and | |
| 1718 | -reconnect flags only cover the stalls ffmpeg itself notices; this covers | |
| 1719 | every mode, including the one where it reconnects happily and still writes | |
| 1720 | nothing. | |
| 1721 | ||
| 1722 | Progress is measured on the output file rather than on ffmpeg's stderr, | |
| 1723 | because stderr is silent at -v error precisely when things are going well. | |
| 1724 | A missing file counts as no progress -- ffmpeg may not have created it yet, | |
| 1725 | and "not created for five minutes" is also a stall. | |
| 1726 | """ | |
| 1727 | # stderr goes to a FILE, not a pipe. A pipe deadlocks: ffmpeg fills the 64K | |
| 1728 | # buffer, blocks writing, therefore stops writing the OUTPUT too -- and the | |
| 1729 | # watchdog below reads that as a stall and kills a process that was only | |
| 1730 | # waiting for us to read. The failing case is exactly the one whose stderr | |
| 1731 | # is worth having. | |
| 1732 | def _size() -> int: | |
| 1733 | try: | |
| 1734 | return os.path.getsize(out_path) | |
| 1735 | except OSError: | |
| 1736 | return 0 | |
| 1737 | ||
| 1738 | with tempfile.TemporaryFile(mode="w+", encoding="utf-8", | |
| 1739 | errors="replace") as errf: | |
| 1740 | proc = subprocess.Popen(cmd, stdout=subprocess.DEVNULL, stderr=errf) | |
| 1741 | started = time.time() | |
| 1742 | last_size = -1 | |
| 1743 | last_growth = started | |
| 1744 | try: | |
| 1745 | while True: | |
| 1746 | try: | |
| 1747 | proc.wait(timeout=MP4_STALL_POLL) | |
| 1748 | break | |
| 1749 | except subprocess.TimeoutExpired: | |
| 1750 | pass | |
| 1751 | now = time.time() | |
| 1752 | size = _size() | |
| 1753 | if size > last_size: | |
| 1754 | last_size, last_growth = size, now | |
| 1755 | if now - last_growth > stall_timeout: | |
| 1756 | raise RemuxStalled( | |
| 1757 | f"no output for {now - last_growth:.0f}s " | |
| 1758 | f"after {last_size} bytes in {now - started:.0f}s") | |
| 1759 | if now - started > hard_timeout: | |
| 1760 | raise RemuxTimedOut( | |
| 1761 | f"still running after {now - started:.0f}s " | |
| 1762 | f"({last_size} bytes)") | |
| 1763 | # Read stderr back even on the way out: a stall or a timeout is exactly | |
| 1764 | # when ffmpeg's last words are worth having, and the caller only ever | |
| 1765 | # sees the exception message. | |
| 1766 | except BaseException as exc: | |
| 1767 | # Kill rather than terminate: ffmpeg reading a stalled socket does | |
| 1768 | # not always act on SIGTERM, and leaving it alive holds the .part | |
| 1769 | # open and the semaphore slot with it. | |
| 1770 | proc.kill() | |
| 1771 | proc.wait() | |
| 1772 | tail = _read_back(errf)[-300:] | |
| 1773 | if tail and isinstance(exc, (RemuxStalled, RemuxTimedOut)): | |
| 1774 | raise type(exc)(f"{exc} — ffmpeg said: {tail}") from None | |
| 1775 | raise | |
| 1776 | return _RemuxResult(proc.returncode, _read_back(errf)) | |
| 1777 | ||
| 1778 | ||
| 1779 | # ---------------------------------------------------------------- subtitles | |
| 1780 | # | |
| 1781 | # The HLS stream carries none. Measured 2026-09-04 on No Other Choice (tmdb | |
| 1782 | # 639988): the master lists four muxed A/V variants and no #EXT-X-MEDIA at all, | |
| 1783 | # so `-c copy` is not dropping anything -- there is nothing there. | |
| 1784 | # | |
| 1785 | # The vidcore PLAYER gets them from a separate endpoint, found in the 2026-07-21 | |
| 1786 | # deobfuscation of vidcore's bundle and re-verified today: `/wyzie` is a proxy | |
| 1787 | # in front of Wyzie, returning `[{display, language, url, encoding}]`. It needs | |
| 1788 | # no browser and, unlike sub.wyzie.ru direct (now `401 API key required`), no | |
| 1789 | # key of ours. | |
| 1790 | # | |
| 1791 | # All 32 tracks for that movie fetch in 2.9s / 3.6 MB with 8 workers, every one | |
| 1792 | # plain SRT, so there is no case for choosing a language here. Everything is | |
| 1793 | # taken and the player picks. | |
| 1794 | WYZIE_URL = "https://vidcore.net/wyzie" | |
| 1795 | SUBTITLE_TIMEOUT = 25 | |
| 1796 | # Measured: 32 tracks in 2.9s at 8, slowest single request 0.95s. Higher buys | |
| 1797 | # nothing against a set this size and is rude to a host doing us a favour. | |
| 1798 | SUBTITLE_WORKERS = 8 | |
| 1799 | ||
| 1800 | # wyzie speaks OpenSubtitles' codes, and several are not ISO at all. Verified on | |
| 1801 | # 639988 against the endpoint's own `display` field. Anything not listed is | |
| 1802 | # passed through: an unrecognised code still names a distinct file, and dropping | |
| 1803 | # a subtitle because we did not recognise its label is the worse failure. | |
| 1804 | WYZIE_LANG_MAP = { | |
| 1805 | "pb": "pt-BR", # Portuguese (BR) | |
| 1806 | "ea": "es-419", # Spanish (LA) | |
| 1807 | "zt": "zh-Hant", # Chinese (traditional) | |
| 1808 | "zc": "yue", # Chinese (Cantonese) | |
| 1809 | "ze": "zh", # Chinese bilingual | |
| 1810 | } | |
| 1811 | ||
| 1812 | ||
| 1813 | def _http_json(url: str, timeout: int = REQUEST_TIMEOUT): | |
| 1814 | """GET and parse JSON. Raises on anything but a 2xx with a JSON body. | |
| 1815 | ||
| 1816 | Deliberately NOT _http_get: that injects a random X-Forwarded-For, which is | |
| 1817 | for evading a source's rate limit and has no business on an endpoint that is | |
| 1818 | answering us honestly. | |
| 1819 | """ | |
| 1820 | r = requests.get(url, timeout=timeout, | |
| 1821 | headers={"User-Agent": UA, "Referer": "https://vidcore.net/"}) | |
| 1822 | r.raise_for_status() | |
| 1823 | return r.json() | |
| 1824 | ||
| 1825 | ||
| 1826 | def _fetch_subtitle(url: str, timeout: int = SUBTITLE_TIMEOUT) -> bytes: | |
| 1827 | """One subtitle file's bytes. Raises on anything but a 2xx.""" | |
| 1828 | r = requests.get(url, timeout=timeout, | |
| 1829 | headers={"User-Agent": UA, "Referer": "https://vidcore.net/"}) | |
| 1830 | r.raise_for_status() | |
| 1831 | return r.content | |
| 1832 | ||
| 1833 | ||
| 1834 | def _wyzie_tracks(tmdb_id: str, season: int = 0, episode: int = 0) -> list[dict]: | |
| 1835 | """Available subtitle tracks, or [] -- never an exception. | |
| 1836 | ||
| 1837 | A title Wyzie lacks answers 404 with a zero-length body (verified on X-Men | |
| 1838 | '97 S2E8, which genuinely has none while S1E1 has 34). That is an ordinary | |
| 1839 | outcome, not a failure, and an ingest must not fail over it. The same goes | |
| 1840 | for the endpoint being down: a copy without subtitles is worth having. | |
| 1841 | """ | |
| 1842 | url = f"{WYZIE_URL}?id={tmdb_id}" | |
| 1843 | if season and episode: | |
| 1844 | url += f"&season={season}&episode={episode}" | |
| 1845 | try: | |
| 1846 | data = _http_json(url) | |
| 1847 | except Exception as exc: | |
| 1848 | log.info("subtitles: none for %s (%s)", tmdb_id, exc) | |
| 1849 | return [] | |
| 1850 | if not isinstance(data, list): | |
| 1851 | log.warning("subtitles: unexpected answer shape for %s", tmdb_id) | |
| 1852 | return [] | |
| 1853 | tracks = [t for t in data if isinstance(t, dict) and t.get("url")] | |
| 1854 | log.info("subtitles: %d track(s) for %s", len(tracks), tmdb_id) | |
| 1855 | return tracks | |
| 1856 | ||
| 1857 | ||
| 1858 | def _sidecar_paths(dest_mp4: str, tracks: list[dict]) -> list[tuple[str, dict]]: | |
| 1859 | """(path, track) for each track, with names that cannot collide or escape. | |
| 1860 | ||
| 1861 | The language code arrives inside a remote JSON body and ends up in a | |
| 1862 | filename, so it goes through _safe_component like everything else that | |
| 1863 | composes a library path. | |
| 1864 | """ | |
| 1865 | stem = os.path.splitext(dest_mp4)[0] | |
| 1866 | seen: dict[str, int] = {} | |
| 1867 | out = [] | |
| 1868 | for track in tracks: | |
| 1869 | code = WYZIE_LANG_MAP.get(track.get("language"), track.get("language")) | |
| 1870 | code = _safe_component(code or "und") | |
| 1871 | n = seen[code] = seen.get(code, 0) + 1 | |
| 1872 | # The first of a language gets the bare suffix, which is the one both | |
| 1873 | # Jellyfin and Kodi read. Later ones are still offered, just ranked | |
| 1874 | # below it, rather than silently overwriting it. | |
| 1875 | suffix = code if n == 1 else f"{code}.{n}" | |
| 1876 | out.append((f"{stem}.{suffix}.srt", track)) | |
| 1877 | return out | |
| 1878 | ||
| 1879 | ||
| 1880 | def _write_sidecars(dest_mp4: str, tracks: list[dict]) -> list[str]: | |
| 1881 | """Download `tracks` next to `dest_mp4`. Returns the paths that exist after. | |
| 1882 | ||
| 1883 | Sidecars rather than a muxed track because Jellyfin indexes the directory, | |
| 1884 | and because a subtitle added later must not mean re-copying 3 GB of video. | |
| 1885 | ||
| 1886 | Nothing here may fail the ingest: subtitles are an addition to a copy that | |
| 1887 | is already complete and published. One track failing costs that track. | |
| 1888 | """ | |
| 1889 | if not tracks: | |
| 1890 | return [] | |
| 1891 | planned = _sidecar_paths(dest_mp4, tracks) | |
| 1892 | todo = [(p, t) for p, t in planned | |
| 1893 | if not (os.path.exists(p) and os.path.getsize(p) > 0)] | |
| 1894 | done = [p for p, _ in planned if p not in {q for q, _ in todo}] | |
| 1895 | ||
| 1896 | def _one(item): | |
| 1897 | path, track = item | |
| 1898 | try: | |
| 1899 | body = _fetch_subtitle(track["url"]) | |
| 1900 | except Exception as exc: | |
| 1901 | log.info("subtitles: %s failed (%s)", os.path.basename(path), exc) | |
| 1902 | return None | |
| 1903 | if not body: | |
| 1904 | # An empty file is worse than none: the player offers a track that | |
| 1905 | # shows nothing, and the "already on disk" check would keep it. | |
| 1906 | log.info("subtitles: %s came back empty", os.path.basename(path)) | |
| 1907 | return None | |
| 1908 | tmp = _part_path(path) | |
| 1909 | try: | |
| 1910 | with open(tmp, "wb") as f: | |
| 1911 | f.write(body) | |
| 1912 | os.replace(tmp, path) | |
| 1913 | except Exception as exc: | |
| 1914 | log.warning("subtitles: could not write %s (%s)", | |
| 1915 | os.path.basename(path), exc) | |
| 1916 | return None | |
| 1917 | return path | |
| 1918 | ||
| 1919 | if todo: | |
| 1920 | with ThreadPoolExecutor(max_workers=SUBTITLE_WORKERS) as pool: | |
| 1921 | done += [p for p in pool.map(_one, todo) if p] | |
| 1922 | if done: | |
| 1923 | log.info("subtitles: %d file(s) beside %s", | |
| 1924 | len(done), os.path.basename(dest_mp4)) | |
| 1925 | return sorted(done) | |
| 1926 | ||
| 1927 | ||
| 1928 | ||
| 1664 | 1929 | def _remux_to_mp4(variant_url: str, out_path: str) -> None: |
| 1665 | 1930 | """Stream-copy an HLS variant into a progressive MP4. |
| 1666 | 1931 | |
| … | … | @@ -1677,15 +1942,27 @@ def _remux_to_mp4(variant_url: str, out_path: str) -> None: |
| 1677 | 1942 | # Asked BEFORE the remux: these playlists are signed and expire in ~4h, so |
| 1678 | 1943 | # after a 45-minute copy the URL may no longer answer. |
| 1679 | 1944 | expected = _playlist_duration(variant_url) |
| 1680 | proc = subprocess.run( | |
| 1945 | proc = _run_remux( | |
| 1681 | 1946 | ["ffmpeg", "-v", "error", "-y", |
| 1682 | 1947 | "-headers", "Referer: https://vidcore.net/\r\n", |
| 1948 | # An I/O operation that goes quiet errors instead of blocking, and the | |
| 1949 | # reconnect flags then retry it. Without these a CDN that stops | |
| 1950 | # answering mid-file is indistinguishable from one that is merely slow, | |
| 1951 | # and ffmpeg waits on it for ever (2026-09-03: 36 minutes). | |
| 1952 | # -seg_max_retry defaults to 0 on the hls demuxer, i.e. one bad segment | |
| 1953 | # is fatal, which is the wrong default for a 45-minute copy. | |
| 1954 | "-rw_timeout", "30000000", | |
| 1955 | "-reconnect", "1", | |
| 1956 | "-reconnect_streamed", "1", | |
| 1957 | "-reconnect_on_network_error", "1", | |
| 1958 | "-reconnect_delay_max", "30", | |
| 1959 | "-seg_max_retry", "3", | |
| 1683 | 1960 | # -f mp4 is REQUIRED: ffmpeg picks the muxer from the file extension |
| 1684 | 1961 | # and the output is written to `<name>.part`, which it does not |
| 1685 | 1962 | # recognise ("Error initializing the muxer ...: Invalid argument"). |
| 1686 | 1963 | "-i", variant_url, "-c", "copy", "-movflags", "+faststart", |
| 1687 | 1964 | "-f", "mp4", tmp], |
| 1688 | capture_output=True, text=True, timeout=MP4_REMUX_TIMEOUT, | |
| 1965 | tmp, stall_timeout=MP4_STALL_TIMEOUT, hard_timeout=MP4_REMUX_TIMEOUT, | |
| 1689 | 1966 | ) |
| 1690 | 1967 | if proc.returncode != 0 or not os.path.exists(tmp) or os.path.getsize(tmp) == 0: |
| 1691 | 1968 | raise RuntimeError(f"ffmpeg exit {proc.returncode}: {(proc.stderr or '')[-300:]}") |
| … | … | @@ -1713,6 +1990,21 @@ def _remux_to_mp4(variant_url: str, out_path: str) -> None: |
| 1713 | 1990 | os.replace(tmp, out_path) |
| 1714 | 1991 | |
| 1715 | 1992 | |
| 1993 | def _fetch_subtitles_for(dest: str, meta: dict, season: int, episode: int) -> None: | |
| 1994 | """Put whatever subtitles exist beside a published copy. Never raises. | |
| 1995 | ||
| 1996 | Called AFTER the copy is complete and published, so there is nothing a | |
| 1997 | subtitle failure could usefully abort -- and an ingest that reported failure | |
| 1998 | because a subtitle host was down would send the whole 3 GB round again. | |
| 1999 | """ | |
| 2000 | try: | |
| 2001 | tracks = _wyzie_tracks(meta["tmdb_id"], season=season, episode=episode) | |
| 2002 | _write_sidecars(dest, tracks) | |
| 2003 | except Exception as exc: | |
| 2004 | log.warning("subtitles: skipped for %s (%s)", | |
| 2005 | os.path.basename(dest), exc) | |
| 2006 | ||
| 2007 | ||
| 1716 | 2008 | def _run_ingest(kind: str, ident: str, season: int, episode: int, |
| 1717 | 2009 | height: int, force: bool = False) -> str: |
| 1718 | 2010 | """Fetch one title into the library. Synchronous; returns the final state. |
| … | … | @@ -1751,6 +2043,11 @@ def _run_ingest(kind: str, ident: str, season: int, episode: int, |
| 1751 | 2043 | log.info("ingest: forced re-fetch of %s", os.path.basename(dest)) |
| 1752 | 2044 | elif _library_has(kind, ident, season, episode): |
| 1753 | 2045 | log.info("ingest: library already holds %s", os.path.basename(dest)) |
| 2046 | # Still ask for subtitles. This early return exists so an existing copy | |
| 2047 | # costs no browser, and one HTTP call does not change that -- while | |
| 2048 | # skipping it would mean the library only ever gains subtitles for | |
| 2049 | # things fetched after today. | |
| 2050 | _fetch_subtitles_for(dest, meta, season, episode) | |
| 1754 | 2051 | return "done" |
| 1755 | 2052 | url = _resolve_vidcore(kind, ident, season, episode) |
| 1756 | 2053 | if not url: |
| … | … | @@ -1765,6 +2062,7 @@ def _run_ingest(kind: str, ident: str, season: int, episode: int, |
| 1765 | 2062 | return f"failed: {exc}" |
| 1766 | 2063 | log.info("ingest done: %s in %.0fs (%.2f GB)", key, |
| 1767 | 2064 | time.time() - started, os.path.getsize(dest) / 1e9) |
| 2065 | _fetch_subtitles_for(dest, meta, season, episode) | |
| 1768 | 2066 | return "done" |
| 1769 | 2067 | |
| 1770 | 2068 |
| … | … | @@ -291,15 +291,20 @@ def fake_ffmpeg(monkeypatch): |
| 291 | 291 | returncode = 0 |
| 292 | 292 | stderr = "" |
| 293 | 293 | |
| 294 | def fake_run(argv, **kw): | |
| 294 | # The seam is _run_remux, NOT subprocess.run. A stalled copy is now ended by | |
| 295 | # watching the output file grow, which needs Popen and a poll loop -- so a | |
| 296 | # fixture that patches subprocess.run stops standing in for anything and | |
| 297 | # every test using it silently shells out to real ffmpeg against | |
| 298 | # `http://cdn/x.m3u8`, where the reconnect flags make it retry for minutes. | |
| 299 | def fake_run(argv, out_path, stall_timeout=None, hard_timeout=None): | |
| 295 | 300 | Fake.runs.append(argv) |
| 296 | 301 | Result.returncode = 1 if Fake.fail else 0 |
| 297 | 302 | if not Fake.fail: |
| 298 | with open(argv[-1], "wb") as f: | |
| 303 | with open(out_path, "wb") as f: | |
| 299 | 304 | f.write(b"video" * 100) |
| 300 | 305 | return Result |
| 301 | 306 | |
| 302 | monkeypatch.setattr(addon.subprocess, "run", fake_run) | |
| 307 | monkeypatch.setattr(addon, "_run_remux", fake_run) | |
| 303 | 308 | monkeypatch.setattr(addon, "_playlist_duration", |
| 304 | 309 | lambda url: Fake.playlist_seconds) |
| 305 | 310 | monkeypatch.setattr(addon, "_media_duration", |
| … | … | @@ -0,0 +1,138 @@ |
| 1 | """A remux that stops producing bytes is killed promptly, not at the wall clock. | |
| 2 | ||
| 3 | 2026-09-03, No Other Choice (tmdb 639988): ffmpeg started at 21:07:50 and wrote | |
| 4 | 2,848,981,040 bytes by 21:16 -- then produced NOTHING for 36 minutes, until | |
| 5 | `subprocess.run(timeout=MP4_REMUX_TIMEOUT)` killed it at 21:52:51. The job | |
| 6 | reported "timed out after 2700 seconds", which reads as "too slow" and is wrong: | |
| 7 | the copy ran at ~5.3 MB/s while it ran at all. It stalled. | |
| 8 | ||
| 9 | The cost is not the wasted 36 minutes. It is that the wall-clock timeout is the | |
| 10 | ONLY thing that ever ends a stall, so the failure always arrives 45 minutes | |
| 11 | late, and `.part` -- the evidence and the resume point -- looks the same either | |
| 12 | way. Watching the output grow distinguishes them at the only place the | |
| 13 | difference shows. | |
| 14 | ||
| 15 | ffmpeg's own -rw_timeout/-reconnect flags are set too, but they cannot be tested | |
| 16 | here without a CDN that stalls on demand, and they only cover the stalls ffmpeg | |
| 17 | itself notices. This watchdog is outside ffmpeg and covers all of them. | |
| 18 | ||
| 19 | Run: .venv/bin/python3 -m pytest test_remux_stall.py -q | |
| 20 | """ | |
| 21 | ||
| 22 | import os | |
| 23 | import time | |
| 24 | ||
| 25 | import pytest | |
| 26 | ||
| 27 | import addon | |
| 28 | ||
| 29 | ||
| 30 | @pytest.fixture | |
| 31 | def out_file(tmp_path): | |
| 32 | return str(tmp_path / "out.part") | |
| 33 | ||
| 34 | ||
| 35 | def test_a_command_that_never_writes_is_killed_at_the_stall_deadline(out_file): | |
| 36 | started = time.time() | |
| 37 | with pytest.raises(addon.RemuxStalled): | |
| 38 | addon._run_remux(["sh", "-c", "sleep 30"], out_file, | |
| 39 | stall_timeout=2, hard_timeout=30) | |
| 40 | elapsed = time.time() - started | |
| 41 | # The point of the whole change: it ends at the STALL deadline, nowhere near | |
| 42 | # the hard one. Generous upper bound so a loaded machine cannot flake it. | |
| 43 | assert elapsed < 10, f"took {elapsed:.1f}s — did not end at the stall deadline" | |
| 44 | ||
| 45 | ||
| 46 | def test_a_command_that_keeps_writing_is_left_alone(out_file): | |
| 47 | # Appends a byte every 0.2s for ~3s, i.e. across several stall windows. | |
| 48 | cmd = ["sh", "-c", f"for i in $(seq 15); do printf x >> {out_file}; sleep 0.2; done"] | |
| 49 | proc = addon._run_remux(cmd, out_file, stall_timeout=1, hard_timeout=30) | |
| 50 | assert proc.returncode == 0 | |
| 51 | assert os.path.getsize(out_file) == 15 | |
| 52 | ||
| 53 | ||
| 54 | def test_the_hard_timeout_still_applies_to_a_command_that_writes_for_ever(out_file): | |
| 55 | cmd = ["sh", "-c", f"while :; do printf x >> {out_file}; sleep 0.1; done"] | |
| 56 | started = time.time() | |
| 57 | with pytest.raises(addon.RemuxTimedOut): | |
| 58 | addon._run_remux(cmd, out_file, stall_timeout=30, hard_timeout=2) | |
| 59 | assert time.time() - started < 10 | |
| 60 | ||
| 61 | ||
| 62 | def test_a_stall_is_measured_from_the_last_byte_not_from_the_start(out_file): | |
| 63 | # Writes for ~1s, then goes quiet. The stall deadline must be counted from | |
| 64 | # the last write -- otherwise a copy that starts slowly is killed for it. | |
| 65 | cmd = ["sh", "-c", f"for i in $(seq 5); do printf x >> {out_file}; sleep 0.2; done; sleep 30"] | |
| 66 | started = time.time() | |
| 67 | with pytest.raises(addon.RemuxStalled): | |
| 68 | addon._run_remux(cmd, out_file, stall_timeout=2, hard_timeout=30) | |
| 69 | elapsed = time.time() - started | |
| 70 | assert elapsed > 2, f"ended in {elapsed:.1f}s — did not wait out the writing phase" | |
| 71 | assert elapsed < 12, f"took {elapsed:.1f}s — did not end at the stall deadline" | |
| 72 | ||
| 73 | ||
| 74 | def test_a_missing_output_file_counts_as_no_progress(out_file): | |
| 75 | # ffmpeg has not created the .part yet. That is still no progress, and must | |
| 76 | # not raise its own error on os.path.getsize. | |
| 77 | assert not os.path.exists(out_file) | |
| 78 | with pytest.raises(addon.RemuxStalled): | |
| 79 | addon._run_remux(["sh", "-c", "sleep 30"], out_file, | |
| 80 | stall_timeout=2, hard_timeout=30) | |
| 81 | ||
| 82 | ||
| 83 | def test_the_reconnect_flags_are_actually_passed_to_ffmpeg(monkeypatch, tmp_path): | |
| 84 | """The flags are the half of the fix this file cannot exercise directly, so | |
| 85 | at minimum assert they reach the command line.""" | |
| 86 | seen = {} | |
| 87 | ||
| 88 | def fake_run(cmd, out_path, stall_timeout, hard_timeout): | |
| 89 | seen["cmd"] = cmd | |
| 90 | with open(out_path, "wb") as f: | |
| 91 | f.write(b"x") | |
| 92 | ||
| 93 | class P: | |
| 94 | returncode = 0 | |
| 95 | stderr = "" | |
| 96 | return P() | |
| 97 | ||
| 98 | monkeypatch.setattr(addon, "_run_remux", fake_run) | |
| 99 | monkeypatch.setattr(addon, "_playlist_duration", lambda url: None) | |
| 100 | dest = str(tmp_path / "movie.mp4") | |
| 101 | addon._remux_to_mp4("http://example.invalid/v.m3u8", dest) | |
| 102 | cmd = seen["cmd"] | |
| 103 | assert "-rw_timeout" in cmd | |
| 104 | assert "-reconnect" in cmd | |
| 105 | assert "-reconnect_streamed" in cmd | |
| 106 | assert "-reconnect_on_network_error" in cmd | |
| 107 | # The hls demuxer defaults seg_max_retry to 0 -- one bad segment is fatal. | |
| 108 | assert "-seg_max_retry" in cmd | |
| 109 | ||
| 110 | ||
| 111 | def test_stderr_is_captured_on_success(out_file): | |
| 112 | cmd = ["sh", "-c", f"printf x >> {out_file}; echo 'a warning' >&2"] | |
| 113 | proc = addon._run_remux(cmd, out_file, stall_timeout=10, hard_timeout=30) | |
| 114 | assert proc.returncode == 0 | |
| 115 | assert "a warning" in proc.stderr | |
| 116 | ||
| 117 | ||
| 118 | def test_a_chatty_stderr_does_not_look_like_a_stall(out_file): | |
| 119 | """The reason stderr is a temp file and not a pipe. | |
| 120 | ||
| 121 | A pipe holds 64K. Past that ffmpeg blocks on the write, therefore stops | |
| 122 | writing the OUTPUT, and the watchdog sees a stall -- killing a process that | |
| 123 | was only waiting for us to read, in exactly the failing case whose stderr is | |
| 124 | worth having. 400K is comfortably past any pipe buffer. | |
| 125 | """ | |
| 126 | cmd = ["sh", "-c", | |
| 127 | f"yes 'ffmpeg noise on stderr, at length' | head -c 400000 >&2; " | |
| 128 | f"printf x >> {out_file}"] | |
| 129 | proc = addon._run_remux(cmd, out_file, stall_timeout=5, hard_timeout=30) | |
| 130 | assert proc.returncode == 0 | |
| 131 | assert len(proc.stderr) > 100000, f"only captured {len(proc.stderr)} bytes" | |
| 132 | ||
| 133 | ||
| 134 | def test_ffmpeg_s_last_words_survive_a_stall(out_file): | |
| 135 | cmd = ["sh", "-c", "echo 'Server returned 403 Forbidden' >&2; sleep 30"] | |
| 136 | with pytest.raises(addon.RemuxStalled) as e: | |
| 137 | addon._run_remux(cmd, out_file, stall_timeout=2, hard_timeout=30) | |
| 138 | assert "403 Forbidden" in str(e.value) |
| … | … | @@ -0,0 +1,266 @@ |
| 1 | """Subtitles are fetched alongside a library copy, as sidecar files. | |
| 2 | ||
| 3 | The HLS stream carries NONE. Measured 2026-09-04 on No Other Choice (tmdb | |
| 4 | 639988): the master lists four muxed A/V variants and no #EXT-X-MEDIA at all, so | |
| 5 | there is nothing for `-c copy` to have dropped. The vidcore PLAYER gets them | |
| 6 | from a separate endpoint -- `vidcore.net/wyzie?id=<tmdb>[&season&episode]`, | |
| 7 | found in the 2026-07-21 deobfuscation -- which returns | |
| 8 | `[{display, language, url, encoding}]` and needs no key and no browser. All 32 | |
| 9 | tracks for that movie fetch in 2.9s / 3.6 MB with 8 workers, every one plain | |
| 10 | SRT, so there is no case for being selective. | |
| 11 | ||
| 12 | They are written as sidecars next to the mp4 because that is what Jellyfin | |
| 13 | indexes. Kodi cannot discover a sidecar over HTTP -- local copies are served | |
| 14 | through `/mp4/...` with send_file -- so the Kodi side fetches its own; this half | |
| 15 | is what makes the LIBRARY carry subtitles. | |
| 16 | ||
| 17 | A title with no subtitles is not an error. vidcore answers 404 with a zero-length | |
| 18 | body for those (verified: X-Men '97 S2E8), and an ingest must not fail over it. | |
| 19 | ||
| 20 | Run: .venv/bin/python3 -m pytest test_subtitles.py -q | |
| 21 | """ | |
| 22 | ||
| 23 | import json | |
| 24 | import os | |
| 25 | ||
| 26 | import pytest | |
| 27 | ||
| 28 | import addon | |
| 29 | ||
| 30 | ||
| 31 | @pytest.fixture | |
| 32 | def mp4(tmp_path): | |
| 33 | p = tmp_path / "No Other Choice (2025).mp4" | |
| 34 | p.write_bytes(b"not really an mp4") | |
| 35 | return str(p) | |
| 36 | ||
| 37 | ||
| 38 | def _tracks(*pairs): | |
| 39 | return [{"display": d, "language": l, "url": f"http://x.invalid/{l}", | |
| 40 | "encoding": "UTF-8"} for l, d in pairs] | |
| 41 | ||
| 42 | ||
| 43 | @pytest.fixture | |
| 44 | def fetched(monkeypatch): | |
| 45 | """Record what was asked for; serve deterministic bytes.""" | |
| 46 | calls = [] | |
| 47 | ||
| 48 | def fake_get(url, timeout=None): | |
| 49 | calls.append(url) | |
| 50 | return b"1\n00:00:01,000 --> 00:00:02,000\n" + url.encode() + b"\n" | |
| 51 | ||
| 52 | monkeypatch.setattr(addon, "_fetch_subtitle", fake_get) | |
| 53 | return calls | |
| 54 | ||
| 55 | ||
| 56 | # ---------------------------------------------------------------- the endpoint | |
| 57 | ||
| 58 | def test_a_movie_is_asked_for_by_tmdb_id_alone(monkeypatch): | |
| 59 | seen = {} | |
| 60 | monkeypatch.setattr(addon, "_http_json", | |
| 61 | lambda url, **k: seen.setdefault("url", url) and []) | |
| 62 | addon._wyzie_tracks("639988") | |
| 63 | assert "id=639988" in seen["url"] | |
| 64 | assert "season" not in seen["url"] | |
| 65 | assert "episode" not in seen["url"] | |
| 66 | ||
| 67 | ||
| 68 | def test_an_episode_carries_season_and_episode(monkeypatch): | |
| 69 | seen = {} | |
| 70 | monkeypatch.setattr(addon, "_http_json", | |
| 71 | lambda url, **k: seen.setdefault("url", url) and []) | |
| 72 | addon._wyzie_tracks("138502", season=1, episode=1) | |
| 73 | assert "id=138502" in seen["url"] | |
| 74 | assert "season=1" in seen["url"] | |
| 75 | assert "episode=1" in seen["url"] | |
| 76 | ||
| 77 | ||
| 78 | def test_no_subtitles_is_an_empty_list_not_an_error(monkeypatch): | |
| 79 | """vidcore answers 404 with an empty body for a title Wyzie lacks.""" | |
| 80 | def boom(url, **k): | |
| 81 | raise RuntimeError("HTTP 404") | |
| 82 | monkeypatch.setattr(addon, "_http_json", boom) | |
| 83 | assert addon._wyzie_tracks("138502", season=2, episode=8) == [] | |
| 84 | ||
| 85 | ||
| 86 | def test_a_malformed_answer_is_an_empty_list(monkeypatch): | |
| 87 | monkeypatch.setattr(addon, "_http_json", lambda url, **k: {"not": "a list"}) | |
| 88 | assert addon._wyzie_tracks("639988") == [] | |
| 89 | ||
| 90 | ||
| 91 | def test_entries_without_a_url_are_dropped(monkeypatch): | |
| 92 | monkeypatch.setattr(addon, "_http_json", lambda url, **k: [ | |
| 93 | {"display": "English", "language": "en", "url": "http://x/en"}, | |
| 94 | {"display": "Broken", "language": "xx"}, | |
| 95 | ]) | |
| 96 | got = addon._wyzie_tracks("639988") | |
| 97 | assert [t["language"] for t in got] == ["en"] | |
| 98 | ||
| 99 | ||
| 100 | # ------------------------------------------------------------------- the files | |
| 101 | ||
| 102 | def test_sidecars_land_next_to_the_mp4_with_the_same_stem(mp4, fetched): | |
| 103 | paths = addon._write_sidecars(mp4, _tracks(("en", "English"), ("fr", "French"))) | |
| 104 | assert len(paths) == 2 | |
| 105 | for p in paths: | |
| 106 | assert os.path.dirname(p) == os.path.dirname(mp4) | |
| 107 | assert os.path.basename(p).startswith("No Other Choice (2025).") | |
| 108 | assert p.endswith(".srt") | |
| 109 | assert os.path.getsize(p) > 0 | |
| 110 | ||
| 111 | ||
| 112 | def test_the_language_code_is_the_suffix_jellyfin_reads(mp4, fetched): | |
| 113 | paths = addon._write_sidecars(mp4, _tracks(("en", "English"))) | |
| 114 | assert os.path.basename(paths[0]) == "No Other Choice (2025).en.srt" | |
| 115 | ||
| 116 | ||
| 117 | def test_opensubtitles_only_codes_are_mapped_to_real_ones(mp4, fetched): | |
| 118 | """wyzie speaks OpenSubtitles' codes, several of which are not ISO at all. | |
| 119 | Verified on 639988: pb='Portuguese (BR)', ea='Spanish (LA)', | |
| 120 | zt='Chinese (traditional)', zc='Chinese (Cantonese)', ze='Chinese bilingual'.""" | |
| 121 | paths = addon._write_sidecars(mp4, _tracks( | |
| 122 | ("pb", "Portuguese (BR)"), ("ea", "Spanish (LA)"), | |
| 123 | ("zt", "Chinese (traditional)"), ("zc", "Chinese (Cantonese)"))) | |
| 124 | names = sorted(os.path.basename(p) for p in paths) | |
| 125 | assert names == sorted([ | |
| 126 | "No Other Choice (2025).pt-BR.srt", | |
| 127 | "No Other Choice (2025).es-419.srt", | |
| 128 | "No Other Choice (2025).zh-Hant.srt", | |
| 129 | "No Other Choice (2025).yue.srt", | |
| 130 | ]) | |
| 131 | ||
| 132 | ||
| 133 | def test_two_tracks_in_one_language_do_not_overwrite_each_other(mp4, fetched): | |
| 134 | paths = addon._write_sidecars(mp4, _tracks(("en", "English"), ("en", "English"))) | |
| 135 | assert len(set(paths)) == 2, "the second track overwrote the first" | |
| 136 | assert all(os.path.exists(p) for p in paths) | |
| 137 | ||
| 138 | ||
| 139 | def test_an_unknown_code_is_passed_through_rather_than_dropped(mp4, fetched): | |
| 140 | paths = addon._write_sidecars(mp4, _tracks(("qq", "Klingon"))) | |
| 141 | assert os.path.basename(paths[0]) == "No Other Choice (2025).qq.srt" | |
| 142 | ||
| 143 | ||
| 144 | def test_a_code_that_would_escape_the_directory_is_refused(mp4, fetched): | |
| 145 | """The code reaches a filename straight from a remote JSON body.""" | |
| 146 | paths = addon._write_sidecars(mp4, _tracks(("../../etc/passwd", "Evil"))) | |
| 147 | for p in paths: | |
| 148 | assert os.path.dirname(os.path.realpath(p)) == \ | |
| 149 | os.path.realpath(os.path.dirname(mp4)) | |
| 150 | ||
| 151 | ||
| 152 | def test_one_failed_download_does_not_lose_the_others(mp4, monkeypatch): | |
| 153 | def flaky(url, timeout=None): | |
| 154 | if url.endswith("/fr"): | |
| 155 | raise RuntimeError("connection reset") | |
| 156 | return b"1\n00:00:01,000 --> 00:00:02,000\nhi\n" | |
| 157 | monkeypatch.setattr(addon, "_fetch_subtitle", flaky) | |
| 158 | paths = addon._write_sidecars(mp4, _tracks( | |
| 159 | ("en", "English"), ("fr", "French"), ("de", "German"))) | |
| 160 | assert sorted(os.path.basename(p) for p in paths) == [ | |
| 161 | "No Other Choice (2025).de.srt", "No Other Choice (2025).en.srt"] | |
| 162 | ||
| 163 | ||
| 164 | def test_an_empty_body_is_not_written(mp4, monkeypatch): | |
| 165 | monkeypatch.setattr(addon, "_fetch_subtitle", lambda url, timeout=None: b"") | |
| 166 | assert addon._write_sidecars(mp4, _tracks(("en", "English"))) == [] | |
| 167 | assert not [f for f in os.listdir(os.path.dirname(mp4)) if f.endswith(".srt")] | |
| 168 | ||
| 169 | ||
| 170 | def test_no_tracks_writes_nothing_and_does_not_raise(mp4, fetched): | |
| 171 | assert addon._write_sidecars(mp4, []) == [] | |
| 172 | ||
| 173 | ||
| 174 | def test_a_sidecar_already_on_disk_is_not_refetched(mp4, fetched): | |
| 175 | addon._write_sidecars(mp4, _tracks(("en", "English"))) | |
| 176 | assert len(fetched) == 1 | |
| 177 | addon._write_sidecars(mp4, _tracks(("en", "English"))) | |
| 178 | assert len(fetched) == 1, "refetched a subtitle already on disk" | |
| 179 | ||
| 180 | ||
| 181 | # ------------------------------------------------------- wired into the ingest | |
| 182 | ||
| 183 | @pytest.fixture | |
| 184 | def ingest(monkeypatch, tmp_path): | |
| 185 | """A _run_ingest with everything outside the subtitle question stubbed.""" | |
| 186 | state = {"tracks": [{"display": "English", "language": "en", | |
| 187 | "url": "http://x.invalid/en"}], | |
| 188 | "sidecars": [], "remuxed": False, "have": False} | |
| 189 | ||
| 190 | monkeypatch.setattr(addon, "ingest_source", lambda: object()) | |
| 191 | monkeypatch.setattr(addon, "_enough_space", lambda: True) | |
| 192 | monkeypatch.setattr(addon, "_library_meta", lambda *a, **k: { | |
| 193 | "title": "No Other Choice", "year": "2025", "tmdb_id": "639988", | |
| 194 | "episode_title": ""}) | |
| 195 | monkeypatch.setattr(addon, "LIBRARY_ROOT", str(tmp_path)) | |
| 196 | monkeypatch.setattr(addon, "_library_has", lambda *a, **k: state["have"]) | |
| 197 | monkeypatch.setattr(addon, "_resolve_vidcore", | |
| 198 | lambda *a, **k: "http://x.invalid/master.m3u8") | |
| 199 | monkeypatch.setattr(addon, "_cap_variant", lambda url, h: url) | |
| 200 | ||
| 201 | def fake_remux(variant, dest): | |
| 202 | state["remuxed"] = True | |
| 203 | os.makedirs(os.path.dirname(dest), exist_ok=True) | |
| 204 | with open(dest, "wb") as f: | |
| 205 | f.write(b"video") | |
| 206 | ||
| 207 | monkeypatch.setattr(addon, "_remux_to_mp4", fake_remux) | |
| 208 | monkeypatch.setattr(addon, "_wyzie_tracks", | |
| 209 | lambda *a, **k: state["tracks"]) | |
| 210 | ||
| 211 | def fake_sidecars(dest, tracks): | |
| 212 | state["sidecars"].append((dest, list(tracks))) | |
| 213 | return [dest + ".en.srt"] if tracks else [] | |
| 214 | ||
| 215 | monkeypatch.setattr(addon, "_write_sidecars", fake_sidecars) | |
| 216 | return state | |
| 217 | ||
| 218 | ||
| 219 | def test_a_finished_copy_gets_its_subtitles(ingest): | |
| 220 | assert addon._run_ingest("movie", "tmdb:639988", 0, 0, 1080) == "done" | |
| 221 | assert len(ingest["sidecars"]) == 1 | |
| 222 | dest, tracks = ingest["sidecars"][0] | |
| 223 | assert dest.endswith("No Other Choice (2025).mp4") | |
| 224 | assert [t["language"] for t in tracks] == ["en"] | |
| 225 | ||
| 226 | ||
| 227 | def test_a_copy_already_in_the_library_is_backfilled(ingest): | |
| 228 | """The early return exists so an existing copy costs no browser. It should | |
| 229 | still gain subtitles -- that is the whole library, not just new fetches.""" | |
| 230 | ingest["have"] = True | |
| 231 | assert addon._run_ingest("movie", "tmdb:639988", 0, 0, 1080) == "done" | |
| 232 | assert not ingest["remuxed"], "resolved and remuxed a copy we already hold" | |
| 233 | assert len(ingest["sidecars"]) == 1 | |
| 234 | ||
| 235 | ||
| 236 | def test_a_title_with_no_subtitles_still_succeeds(ingest): | |
| 237 | ingest["tracks"] = [] | |
| 238 | assert addon._run_ingest("movie", "tmdb:639988", 0, 0, 1080) == "done" | |
| 239 | ||
| 240 | ||
| 241 | def test_a_failed_subtitle_fetch_never_fails_the_ingest(ingest, monkeypatch): | |
| 242 | """The copy is complete and published before this runs. Nothing about a | |
| 243 | subtitle may take that away.""" | |
| 244 | def boom(*a, **k): | |
| 245 | raise RuntimeError("wyzie is down") | |
| 246 | monkeypatch.setattr(addon, "_wyzie_tracks", boom) | |
| 247 | assert addon._run_ingest("movie", "tmdb:639988", 0, 0, 1080) == "done" | |
| 248 | ||
| 249 | ||
| 250 | def test_a_failed_sidecar_write_never_fails_the_ingest(ingest, monkeypatch): | |
| 251 | def boom(*a, **k): | |
| 252 | raise OSError("read-only filesystem") | |
| 253 | monkeypatch.setattr(addon, "_write_sidecars", boom) | |
| 254 | assert addon._run_ingest("movie", "tmdb:639988", 0, 0, 1080) == "done" | |
| 255 | ||
| 256 | ||
| 257 | def test_an_episode_passes_its_season_and_episode_through(ingest, monkeypatch): | |
| 258 | seen = {} | |
| 259 | monkeypatch.setattr(addon, "_wyzie_tracks", | |
| 260 | lambda tmdb, season=0, episode=0: seen.update( | |
| 261 | tmdb=tmdb, season=season, episode=episode) or []) | |
| 262 | monkeypatch.setattr(addon, "_library_meta", lambda *a, **k: { | |
| 263 | "title": "X-Men 97", "year": "2024", "tmdb_id": "138502", | |
| 264 | "episode_title": "Pilot"}) | |
| 265 | addon._run_ingest("series", "tmdb:138502", 1, 1, 1080) | |
| 266 | assert seen == {"tmdb": "138502", "season": 1, "episode": 1} |