| … | … | @@ -4,6 +4,7 @@ with a vsembed.ru/CloudStream Pro fallback (currently broken — Cloudflare Turn |
| 4 | 4 | The fmovies path runs node + Playwright via subprocess to handle the JS-driven extraction. |
| 5 | 5 | """ |
| 6 | 6 | |
| 7 | import bisect |
| 7 | 8 | import calendar |
| 8 | 9 | import contextlib |
| 9 | 10 | import html |
| … | … | @@ -3200,6 +3201,9 @@ def _subs_by_id(kind: str, ident: str) -> dict: |
| 3200 | 3201 | |
| 3201 | 3202 | @app.route("/subs/<kind>/<path:ident>.json") |
| 3202 | 3203 | def subs_list(kind: str, ident: str): |
| 3204 | live = _live_request(kind, ident) |
| 3205 | if live: |
| 3206 | return jsonify(_live_answer(kind, ident, *live)) |
| 3203 | 3207 | out = [] |
| 3204 | 3208 | for sid, p in sorted(_subs_by_id(kind, ident).items()): |
| 3205 | 3209 | out.append({"id": sid, "lang": re.sub(r"\.\d+$", "", sid), |
| … | … | @@ -3211,13 +3215,375 @@ def subs_list(kind: str, ident: str): |
| 3211 | 3215 | |
| 3212 | 3216 | @app.route("/subs/<kind>/<path:ident>/<sid>.srt") |
| 3213 | 3217 | def subs_file(kind: str, ident: str, sid: str): |
| 3214 | | # Looked up among the copy's own sidecars, never joined onto a path. |
| 3215 | | path = _subs_by_id(kind, ident).get(sid) |
| 3218 | # Looked up among the copy's own sidecars (or the live sync's own list), |
| 3219 | # never joined onto a path. |
| 3220 | live = _live_request(kind, ident) |
| 3221 | path = (_live_file(kind, ident, sid, *live) if live |
| 3222 | else _subs_by_id(kind, ident).get(sid)) |
| 3216 | 3223 | if not path: |
| 3217 | 3224 | abort(404) |
| 3218 | 3225 | return send_file(path, mimetype="application/x-subrip") |
| 3219 | 3226 | |
| 3220 | 3227 | |
| 3228 | # ---------------------------- live streams: subtitles synced to the stream --- |
| 3229 | # A stream played from vidcore or HdHub has no copy to measure against, and |
| 3230 | # taking all its audio costs what the download does (Hope: 767 s). So six |
| 3231 | # 150 s audio windows of the stream being played are fetched (~150 MB, ~1 min) |
| 3232 | # and window_align places each English track against them -- see |
| 3233 | # test_window_align.py for why that, and not ffsubsync's own search. Only a |
| 3234 | # track the windows could place is offered; the player keeps what it had for |
| 3235 | # the rest. Results live in LIVE_SUBS_DIR, outside the library: a cache, not |
| 3236 | # library state. |
| 3237 | LIVE_SUBS_DIR = os.path.join(os.path.dirname(os.path.abspath(__file__)), "live-subs") |
| 3238 | LIVE_SOURCES = ("vidcore", "hdhub", "mp4") |
| 3239 | LIVE_WINDOW_FRACTIONS = (0.1, 0.25, 0.4, 0.55, 0.7, 0.85) |
| 3240 | # 6 x 150 s, not 3 x 300 s: on Hope 3 x 300 left one window that matched (a |
| 3241 | # quiet stretch, then music the VAD called 97% speech) and no answer; six |
| 3242 | # shorter windows gave three that agreed. |
| 3243 | LIVE_WINDOW_SECONDS = 150 |
| 3244 | LIVE_MAX_TRACKS = 3 |
| 3245 | LIVE_READY_TTL = 3 * 86400 |
| 3246 | LIVE_NONE_RETRY = 30 * 60 # "no answer" may be a 503 that clears |
| 3247 | _live_jobs: dict[str, str] = {} |
| 3248 | _live_guard = threading.Lock() |
| 3249 | # One at a time: each is ~150 MB of segments and six VAD passes on stortron. |
| 3250 | _live_slot = threading.Semaphore(1) |
| 3251 | |
| 3252 | |
| 3253 | def _live_request(kind: str, ident: str) -> Optional[tuple]: |
| 3254 | """(src, height) when this asks about a stream with no copy behind it, |
| 3255 | else None. A held copy is what plays, whatever `src` says: /vidcore/ and |
| 3256 | /hdhub/ redirect to it.""" |
| 3257 | src = request.args.get("src") |
| 3258 | if src is None: |
| 3259 | return None |
| 3260 | if src not in LIVE_SOURCES: |
| 3261 | abort(404) |
| 3262 | target = _subs_target(kind, ident) |
| 3263 | if not target: |
| 3264 | abort(404) |
| 3265 | if _playable_path(*target): |
| 3266 | return None |
| 3267 | try: |
| 3268 | height = int(request.args.get("h") or VIDCORE_MAX_HEIGHT) |
| 3269 | except ValueError: |
| 3270 | abort(404) |
| 3271 | if height not in VIDCORE_HEIGHTS: |
| 3272 | height = VIDCORE_MAX_HEIGHT |
| 3273 | return src, height |
| 3274 | |
| 3275 | |
| 3276 | def _live_key(kind: str, ident: str, src: str, height: int) -> str: |
| 3277 | return f"{kind}:{ident}:{src}:{height}" |
| 3278 | |
| 3279 | |
| 3280 | def _live_dir(key: str) -> str: |
| 3281 | return os.path.join(LIVE_SUBS_DIR, re.sub(r"[^\w.-]", "_", key)) |
| 3282 | |
| 3283 | |
| 3284 | def _live_state(key: str) -> Optional[dict]: |
| 3285 | """The finished result for `key`, or None (none yet, or out of date).""" |
| 3286 | try: |
| 3287 | with open(os.path.join(_live_dir(key), "state.json")) as f: |
| 3288 | state = json.load(f) |
| 3289 | except Exception: |
| 3290 | return None |
| 3291 | age = time.time() - float(state.get("ts") or 0) |
| 3292 | ttl = LIVE_READY_TTL if state.get("status") == "ready" else LIVE_NONE_RETRY |
| 3293 | return state if age < ttl else None |
| 3294 | |
| 3295 | |
| 3296 | def _start_live_sync(key: str, target) -> None: |
| 3297 | threading.Thread(target=target, name=f"livesubs-{key}", daemon=True).start() |
| 3298 | |
| 3299 | |
| 3300 | def _live_answer(kind: str, ident: str, src: str, height: int) -> dict: |
| 3301 | key = _live_key(kind, ident, src, height) |
| 3302 | state = _live_state(key) |
| 3303 | if state is None: |
| 3304 | with _live_guard: |
| 3305 | start = key not in _live_jobs |
| 3306 | if start: |
| 3307 | _live_jobs[key] = "running" |
| 3308 | # Started outside the guard: the job takes it again when it finishes. |
| 3309 | if start: |
| 3310 | _start_live_sync(key, lambda: _run_live_sync(key, kind, ident, |
| 3311 | src, height)) |
| 3312 | state = _live_state(key) |
| 3313 | if state is None: |
| 3314 | return {"status": "pending", "subtitles": []} |
| 3315 | return {"status": state["status"], "subtitles": [ |
| 3316 | dict(t, url=(f"http://{request.host}/subs/{kind}/{ident}/{t['id']}.srt" |
| 3317 | f"?src={src}&h={height}")) |
| 3318 | for t in state.get("tracks") or []]} |
| 3319 | |
| 3320 | |
| 3321 | def _live_file(kind: str, ident: str, sid: str, src: str, |
| 3322 | height: int) -> Optional[str]: |
| 3323 | key = _live_key(kind, ident, src, height) |
| 3324 | state = _live_state(key) or {} |
| 3325 | if sid not in {t["id"] for t in state.get("tracks") or []}: |
| 3326 | return None |
| 3327 | path = os.path.join(_live_dir(key), sid + ".srt") |
| 3328 | return path if os.path.exists(path) else None |
| 3329 | |
| 3330 | |
| 3331 | def _run_live_sync(key: str, kind: str, ident: str, src: str, height: int) -> None: |
| 3332 | """The job: never raises, always leaves a state behind.""" |
| 3333 | state = {"status": "none", "tracks": []} |
| 3334 | try: |
| 3335 | _, base, season, episode = _subs_target(kind, ident) |
| 3336 | with _live_slot: |
| 3337 | state = _live_sync(kind, base, season, episode, src, height, |
| 3338 | _live_dir(key)) |
| 3339 | except Exception as exc: |
| 3340 | log.warning("livesubs: %s failed (%s)", key, exc) |
| 3341 | finally: |
| 3342 | state["ts"] = time.time() |
| 3343 | try: |
| 3344 | os.makedirs(_live_dir(key), exist_ok=True) |
| 3345 | tmp = os.path.join(_live_dir(key), "state.json.part") |
| 3346 | with open(tmp, "w") as f: |
| 3347 | json.dump(state, f) |
| 3348 | os.replace(tmp, os.path.join(_live_dir(key), "state.json")) |
| 3349 | except Exception as exc: |
| 3350 | log.warning("livesubs: %s state not saved (%s)", key, exc) |
| 3351 | with _live_guard: |
| 3352 | _live_jobs.pop(key, None) |
| 3353 | log.info("livesubs: %s -> %s (%s)", key, state["status"], |
| 3354 | ", ".join(f"{t['id']} {t['status']} {t['offset']:+.2f}s" |
| 3355 | for t in state.get("tracks") or []) or "nothing placed") |
| 3356 | |
| 3357 | |
| 3358 | def _live_sync(kind: str, ident: str, season: int, episode: int, src: str, |
| 3359 | height: int, out_dir: str) -> dict: |
| 3360 | import window_align as wa # numpy: only this path needs it |
| 3361 | source = _live_source(kind, ident, season, episode, src, height) |
| 3362 | if not source: |
| 3363 | return {"status": "none", "tracks": []} |
| 3364 | tracks = _live_tracks(kind, ident, season, episode) |
| 3365 | if not tracks: |
| 3366 | return {"status": "none", "tracks": []} |
| 3367 | with tempfile.TemporaryDirectory(prefix="livesubs-") as work: |
| 3368 | windows = _live_windows(source, work) |
| 3369 | os.makedirs(out_dir, exist_ok=True) |
| 3370 | placed = [] |
| 3371 | for sid, text in tracks: |
| 3372 | m = wa.align(wa.srt_cues(text), windows) |
| 3373 | if not m: |
| 3374 | log.info("livesubs: %s %s -- the windows do not agree", ident, sid) |
| 3375 | continue |
| 3376 | entry = {"id": sid, "lang": re.sub(r"\.\d+$", "", sid), |
| 3377 | "offset": m["offset"], "scale": m["scale"]} |
| 3378 | if _in_tolerance(m): |
| 3379 | entry["status"], body = "in_sync", text |
| 3380 | else: |
| 3381 | body = wa.apply(text, m["scale"], m["offset"]) |
| 3382 | if not _in_tolerance(wa.align(wa.srt_cues(body), windows)): |
| 3383 | log.info("livesubs: %s %s -- correction does not hold", ident, sid) |
| 3384 | continue |
| 3385 | entry["status"] = "corrected" |
| 3386 | part = os.path.join(out_dir, sid + ".srt.part") |
| 3387 | with open(part, "w", encoding="utf-8") as f: |
| 3388 | f.write(body) |
| 3389 | os.replace(part, os.path.join(out_dir, sid + ".srt")) |
| 3390 | placed.append(entry) |
| 3391 | return {"status": "ready" if placed else "none", "tracks": placed} |
| 3392 | |
| 3393 | |
| 3394 | def _live_tracks(kind: str, ident: str, season: int, episode: int) -> list: |
| 3395 | """[(id, text)] English tracks: wyzie's, else OpenSubtitles'.""" |
| 3396 | meta = _library_meta(kind, ident, season, episode) |
| 3397 | if not meta: |
| 3398 | return [] |
| 3399 | |
| 3400 | def fetch(tracks): |
| 3401 | out = [] |
| 3402 | for t in tracks: |
| 3403 | if t.get("language") != "en" or not t.get("url"): |
| 3404 | continue |
| 3405 | try: |
| 3406 | body = _fetch_subtitle(t["url"]) |
| 3407 | except Exception as exc: |
| 3408 | log.info("livesubs: track failed (%s)", exc) |
| 3409 | continue |
| 3410 | if body: |
| 3411 | out.append(body.decode("utf-8", errors="replace")) |
| 3412 | if len(out) >= LIVE_MAX_TRACKS: |
| 3413 | break |
| 3414 | return out |
| 3415 | |
| 3416 | texts = fetch(_wyzie_tracks(meta["tmdb_id"], season=season, episode=episode)) |
| 3417 | if not texts: |
| 3418 | try: |
| 3419 | texts = fetch(_opensubs_for(kind, meta["tmdb_id"], season, episode)) |
| 3420 | except Exception as exc: |
| 3421 | log.info("livesubs: opensubs skipped (%s)", exc) |
| 3422 | return [("en" if n == 0 else f"en.{n + 1}", t) for n, t in enumerate(texts)] |
| 3423 | |
| 3424 | |
| 3425 | def _live_source(kind: str, ident: str, season: int, episode: int, src: str, |
| 3426 | height: int) -> Optional[dict]: |
| 3427 | """What the player is actually playing, for its audio. /mp4/ without a |
| 3428 | copy serves HdHub first, then vidcore -- the same order _serve_mp4 uses.""" |
| 3429 | if src == "mp4": |
| 3430 | src = ("hdhub" if _hdhub_enabled() |
| 3431 | and _hdhub_pick(kind, ident, season, episode, height) else "vidcore") |
| 3432 | if src == "vidcore": |
| 3433 | master = _resolve_vidcore(kind, ident, season, episode) |
| 3434 | if not master: |
| 3435 | return None |
| 3436 | # Every variant carries the same audio on the same clock; the |
| 3437 | # smallest is the cheapest to sample (Hope 480p: 144 MB for 6 x 150 s). |
| 3438 | return {"kind": "hls", "url": _cap_variant(master, min(VIDCORE_HEIGHTS)), |
| 3439 | "headers": {"User-Agent": UA, "Referer": "https://vidcore.net/"}} |
| 3440 | picked = _hdhub_pick(kind, ident, season, episode, height) |
| 3441 | if not picked: |
| 3442 | return None |
| 3443 | f, probe = picked |
| 3444 | audio = _probe_audio(probe) |
| 3445 | english = [a for a in audio if _audio_lang(a) in ("eng", "en")] or audio |
| 3446 | try: |
| 3447 | duration = float(((probe or {}).get("format") or {}).get("duration")) |
| 3448 | except (TypeError, ValueError): |
| 3449 | return None |
| 3450 | return {"kind": "file", "url": f.url, "headers": {}, "duration": duration, |
| 3451 | "audio": english[0]["index"] if english else None} |
| 3452 | |
| 3453 | |
| 3454 | def _hls_segments(body: str, base: str) -> tuple: |
| 3455 | """(init url or None, [(start seconds, url)]) from a media playlist.""" |
| 3456 | init, segs, t, dur = None, [], 0.0, None |
| 3457 | for line in body.splitlines(): |
| 3458 | line = line.strip() |
| 3459 | if line.startswith("#EXT-X-MAP:"): |
| 3460 | m = re.search(r'URI="([^"]+)"', line) |
| 3461 | init = urljoin(base, m.group(1)) if m else None |
| 3462 | elif line.startswith("#EXTINF:"): |
| 3463 | try: |
| 3464 | dur = float(line[8:].split(",")[0]) |
| 3465 | except ValueError: |
| 3466 | dur = None |
| 3467 | elif line and not line.startswith("#") and dur is not None: |
| 3468 | segs.append((round(t, 3), urljoin(base, line))) |
| 3469 | t += dur |
| 3470 | dur = None |
| 3471 | return init, segs |
| 3472 | |
| 3473 | |
| 3474 | def _window_plan(segs: list, fractions=LIVE_WINDOW_FRACTIONS, |
| 3475 | seconds: float = LIVE_WINDOW_SECONDS) -> list: |
| 3476 | """[(first, end)] segment index ranges covering each window.""" |
| 3477 | if len(segs) < 2: |
| 3478 | return [] |
| 3479 | starts = [s for s, _ in segs] |
| 3480 | total = starts[-1] + (starts[-1] - starts[-2]) |
| 3481 | plan = [] |
| 3482 | for f in fractions: |
| 3483 | i0 = bisect.bisect_left(starts, total * f) |
| 3484 | i1 = bisect.bisect_left(starts, total * f + seconds) |
| 3485 | if i0 < i1: |
| 3486 | plan.append((i0, i1)) |
| 3487 | return plan |
| 3488 | |
| 3489 | |
| 3490 | def _probe_start(path: str, stream: Optional[str] = None) -> float: |
| 3491 | args = ["ffprobe", "-v", "error"] |
| 3492 | args += (["-select_streams", stream, "-show_entries", "stream=start_time"] |
| 3493 | if stream else ["-show_entries", "format=start_time"]) |
| 3494 | out = subprocess.run(args + ["-of", "csv=p=0", path], capture_output=True, |
| 3495 | text=True, timeout=60).stdout |
| 3496 | return float(out.strip().splitlines()[0].strip(",")) |
| 3497 | |
| 3498 | |
| 3499 | def _hls_window_wavs(source: dict, work: str) -> list: |
| 3500 | """[(position, wav)] for an HLS variant. ffmpeg cannot seek vidcore's |
| 3501 | muxed fMP4 (it read for 2.5 min and wrote nothing), so each window is the |
| 3502 | init segment plus the segments covering it, concatenated: a valid fMP4 |
| 3503 | whose own timestamps place it. Position = its audio start minus the |
| 3504 | stream's start, i.e. the player's clock.""" |
| 3505 | headers = source["headers"] |
| 3506 | r = requests.get(source["url"], headers=headers, timeout=REQUEST_TIMEOUT) |
| 3507 | r.raise_for_status() |
| 3508 | init, segs = _hls_segments(r.text, source["url"]) |
| 3509 | init_bytes = b"" |
| 3510 | if init: |
| 3511 | init_bytes = requests.get(init, headers=headers, timeout=REQUEST_TIMEOUT).content |
| 3512 | |
| 3513 | def get(url): |
| 3514 | resp = requests.get(url, headers=headers, timeout=30) |
| 3515 | resp.raise_for_status() |
| 3516 | return resp.content |
| 3517 | |
| 3518 | def assemble(name, urls): |
| 3519 | path = os.path.join(work, name) |
| 3520 | with ThreadPoolExecutor(max_workers=4) as pool: |
| 3521 | parts = list(pool.map(get, urls)) |
| 3522 | with open(path, "wb") as f: |
| 3523 | f.write(init_bytes) |
| 3524 | for p in parts: |
| 3525 | f.write(p) |
| 3526 | return path |
| 3527 | |
| 3528 | out = [] |
| 3529 | if len(segs) < 2: |
| 3530 | return out |
| 3531 | base = _probe_start(assemble("first.mp4", [segs[0][1]])) |
| 3532 | for n, (i0, i1) in enumerate(_window_plan(segs)): |
| 3533 | try: |
| 3534 | path = assemble(f"win{n}.mp4", [u for _, u in segs[i0:i1]]) |
| 3535 | pos = _probe_start(path, "a:0") - base |
| 3536 | wav = os.path.join(work, f"w{n}.wav") |
| 3537 | subprocess.run(["ffmpeg", "-v", "error", "-y", "-i", path, "-vn", |
| 3538 | "-ac", "1", "-ar", "16000", "-af", "aresample=async=1", |
| 3539 | wav], check=True, capture_output=True, timeout=300) |
| 3540 | out.append((pos, wav)) |
| 3541 | except Exception as exc: |
| 3542 | log.info("livesubs: window %d skipped (%s)", n, exc) |
| 3543 | return out |
| 3544 | |
| 3545 | |
| 3546 | def _file_window_wavs(source: dict, work: str) -> list: |
| 3547 | """[(position, wav)] for a progressive file: ranged seeks, which a file |
| 3548 | (unlike vidcore's HLS) allows. The English audio stream, whatever the |
| 3549 | player has selected: timing is the video's either way.""" |
| 3550 | out = [] |
| 3551 | for n, f in enumerate(LIVE_WINDOW_FRACTIONS): |
| 3552 | pos = int(source["duration"] * f) |
| 3553 | wav = os.path.join(work, f"w{n}.wav") |
| 3554 | cmd = ["ffmpeg", "-v", "error", "-y", "-rw_timeout", "30000000"] |
| 3555 | if source.get("headers"): |
| 3556 | cmd += ["-headers", "".join(f"{k}: {v}\r\n" |
| 3557 | for k, v in source["headers"].items())] |
| 3558 | cmd += ["-ss", str(pos), "-t", str(LIVE_WINDOW_SECONDS), "-i", source["url"]] |
| 3559 | cmd += (["-map", f"0:{source['audio']}"] if source.get("audio") is not None |
| 3560 | else ["-vn"]) |
| 3561 | cmd += ["-ac", "1", "-ar", "16000", |
| 3562 | "-af", "aresample=async=1:first_pts=0", wav] |
| 3563 | try: |
| 3564 | subprocess.run(cmd, check=True, capture_output=True, timeout=300) |
| 3565 | out.append((float(pos), wav)) |
| 3566 | except Exception as exc: |
| 3567 | log.info("livesubs: window %d skipped (%s)", n, exc) |
| 3568 | return out |
| 3569 | |
| 3570 | |
| 3571 | def _live_windows(source: dict, work: str) -> list: |
| 3572 | """[(position, speech array)] -- VAD by ffsubsync, one window at a time.""" |
| 3573 | import numpy as np |
| 3574 | wavs = (_hls_window_wavs if source["kind"] == "hls" else _file_window_wavs)( |
| 3575 | source, work) |
| 3576 | out = [] |
| 3577 | for pos, wav in wavs: |
| 3578 | try: |
| 3579 | subprocess.run([SUBSYNC_BIN, wav, "--serialize-speech"], cwd=work, |
| 3580 | capture_output=True, timeout=SUBSYNC_TIMEOUT) |
| 3581 | out.append((pos, np.load(os.path.splitext(wav)[0] + ".npz")["speech"])) |
| 3582 | except Exception as exc: |
| 3583 | log.info("livesubs: no speech from %s (%s)", os.path.basename(wav), exc) |
| 3584 | return out |
| 3585 | |
| 3586 | |
| 3221 | 3587 | # ------------------------------ HdHub (direct files) ------------------------ |
| 3222 | 3588 | # The second live source, and the first that hands over FILES rather than a |
| 3223 | 3589 | # player. Found 2026-09-28 in cinewave's bundle (watch.cinewave.qzz.io), where it |