| … | … | @@ -10,6 +10,7 @@ import logging |
| 10 | 10 | import os |
| 11 | 11 | import random |
| 12 | 12 | import re |
| 13 | import shutil |
| 13 | 14 | import subprocess |
| 14 | 15 | import threading |
| 15 | 16 | import time |
| … | … | @@ -596,7 +597,7 @@ TV_GENRES = [ |
| 596 | 597 | |
| 597 | 598 | MANIFEST = { |
| 598 | 599 | "id": "org.tabertron.vsembed", |
| 599 | | "version": "0.16.1", |
| 600 | "version": "0.17.1", |
| 600 | 601 | "name": "Tabertron VsEmbed", |
| 601 | 602 | "description": "Self-hosted HLS extractor with TMDB browse + search, played via vidcore", |
| 602 | 603 | "types": ["movie", "series"], |
| … | … | @@ -1208,14 +1209,30 @@ INGEST_QUEUE_FILE = os.path.join(os.path.dirname(os.path.abspath(__file__)), |
| 1208 | 1209 | _queue_guard = threading.Lock() |
| 1209 | 1210 | |
| 1210 | 1211 | |
| 1212 | def _read_job_queue() -> dict: |
| 1213 | """The queue as stored. RAISES if the file exists but cannot be read. |
| 1214 | |
| 1215 | The raising version exists so a writer can tell "there is no queue yet" from |
| 1216 | "there is a queue and I could not read it". Those must not be treated alike: |
| 1217 | the first is an empty dict, the second is every job on record. |
| 1218 | """ |
| 1219 | try: |
| 1220 | with open(INGEST_QUEUE_FILE) as f: |
| 1221 | data = json.load(f) |
| 1222 | except FileNotFoundError: |
| 1223 | return {} |
| 1224 | if not isinstance(data, dict): |
| 1225 | raise ValueError("ingest queue is not a JSON object") |
| 1226 | return data |
| 1227 | |
| 1228 | |
| 1211 | 1229 | def _load_job_queue() -> dict: |
| 1212 | 1230 | """Persisted job states. Never raises — this is a convenience, and a bad |
| 1213 | 1231 | file must not be a reason to refuse to boot.""" |
| 1214 | 1232 | try: |
| 1215 | | with open(INGEST_QUEUE_FILE) as f: |
| 1216 | | data = json.load(f) |
| 1217 | | return data if isinstance(data, dict) else {} |
| 1218 | | except Exception: |
| 1233 | return _read_job_queue() |
| 1234 | except Exception as exc: |
| 1235 | log.warning("ingest queue unreadable (%s) — reading as empty", exc) |
| 1219 | 1236 | return {} |
| 1220 | 1237 | |
| 1221 | 1238 | |
| … | … | @@ -1227,7 +1244,12 @@ def _set_job_state(key: str, state: str, args: Optional[tuple] = None) -> None: |
| 1227 | 1244 | _mp4_jobs[key] = state |
| 1228 | 1245 | try: |
| 1229 | 1246 | with _queue_guard: |
| 1230 | | queue = _load_job_queue() |
| 1247 | # The RAISING read, deliberately. _load_job_queue answers {} for a |
| 1248 | # file it cannot parse, which is right for a reader and catastrophic |
| 1249 | # for a writer: it would rebuild the queue from nothing and persist |
| 1250 | # that, so one unreadable read destroys every job on record. Letting |
| 1251 | # it raise here costs a single unrecorded state change instead. |
| 1252 | queue = _read_job_queue() |
| 1231 | 1253 | entry = queue.get(key) or {} |
| 1232 | 1254 | entry["state"] = state |
| 1233 | 1255 | if args is not None: |
| … | … | @@ -1711,6 +1733,10 @@ def _run_ingest(kind: str, ident: str, season: int, episode: int, |
| 1711 | 1733 | # speculative indirection is how the special-casing this replaces began. |
| 1712 | 1734 | if not ingest_source(): |
| 1713 | 1735 | return "failed: no ingestable source" |
| 1736 | # Checked before the resolve, so a full volume costs no browser at all. |
| 1737 | if not _enough_space(): |
| 1738 | log.warning("ingest: declining %s — under the free-space floor", key) |
| 1739 | return "failed: below the free-space floor" |
| 1714 | 1740 | meta = _library_meta(kind, ident, season, episode) |
| 1715 | 1741 | if not meta: |
| 1716 | 1742 | return f"failed: cannot identify {ident}" |
| … | … | @@ -2482,6 +2508,120 @@ def meta(ctype: str, meta_id: str): |
| 2482 | 2508 | return jsonify({"meta": meta_obj}) |
| 2483 | 2509 | |
| 2484 | 2510 | |
| 2511 | # -------------------------------- retention --------------------------------- |
| 2512 | # The library is NOT a cache. test_library_ingest says so in its first line, and |
| 2513 | # the volume agrees: 745 GB of it, the overwhelming majority never fetched by |
| 2514 | # this addon at all. So retention has no authority over Shows/ and Movies/. It |
| 2515 | # may reclaim only what it can PROVE is derived, and it may never delete an |
| 2516 | # episode to make room for another one. |
| 2517 | # |
| 2518 | # What protects the disk is therefore refusal, not deletion: below the floor an |
| 2519 | # ingest declines to start. That is the honest order of preference -- a copy we |
| 2520 | # have is worth more than a copy we were about to make, so the new one yields. |
| 2521 | # It also fails visibly, as a row on the audit page, instead of quietly eating |
| 2522 | # the last of the volume. |
| 2523 | # |
| 2524 | # The floor is deliberately generous. At roughly 1.3 GB an episode it leaves |
| 2525 | # about twenty episodes of headroom, which is enough to notice and act before |
| 2526 | # anything is actually at risk. |
| 2527 | LIBRARY_MIN_FREE_BYTES = 25 * 1024 ** 3 |
| 2528 | |
| 2529 | |
| 2530 | def _disk() -> Optional[tuple[int, int, int]]: |
| 2531 | """(total, used, free) for the library volume, or None if unreadable.""" |
| 2532 | try: |
| 2533 | usage = shutil.disk_usage(LIBRARY_ROOT) |
| 2534 | return usage.total, usage.used, usage.free |
| 2535 | except OSError as exc: |
| 2536 | log.warning("retention: cannot stat %s (%s)", LIBRARY_ROOT, exc) |
| 2537 | return None |
| 2538 | |
| 2539 | |
| 2540 | def _enough_space() -> bool: |
| 2541 | """False when an ingest should decline to start. |
| 2542 | |
| 2543 | Unreadable disk means True: this is a brake, and a brake that cannot read |
| 2544 | the road must not stop the car. The same rule the completeness gate follows |
| 2545 | for a missing runtime. |
| 2546 | """ |
| 2547 | disk = _disk() |
| 2548 | if disk is None: |
| 2549 | return True |
| 2550 | return disk[2] >= LIBRARY_MIN_FREE_BYTES |
| 2551 | |
| 2552 | |
| 2553 | def _reclaimable() -> list[dict]: |
| 2554 | """Artifacts that are PROVABLY derived, and therefore safe to remove. |
| 2555 | |
| 2556 | Driven from the ingest queue rather than by walking the volume. That keeps |
| 2557 | it bounded -- twenty stats instead of a crawl over 745 GB -- and it keeps |
| 2558 | the scope honest: these are the copies this addon made, and the only ones it |
| 2559 | can reason about. A file it cannot tie back to a job it ran is left alone, |
| 2560 | which is why the single legacy copy in mp4-cache survives: X-Men '97 S02E07 |
| 2561 | was never ingested to the library, so it is the ONLY copy of that episode |
| 2562 | and nothing here is entitled to touch it. |
| 2563 | """ |
| 2564 | out: list[dict] = [] |
| 2565 | now = time.time() |
| 2566 | for key, entry in _load_job_queue().items(): |
| 2567 | args = entry.get("args") or [] |
| 2568 | if len(args) < 5: |
| 2569 | continue |
| 2570 | kind, ident, season, episode, height = args[:5] |
| 2571 | season, episode, height = int(season), int(episode), int(height) |
| 2572 | meta = _library_meta(kind, ident, season, episode) |
| 2573 | if not meta: |
| 2574 | continue |
| 2575 | dest = _library_path(kind, meta["title"], meta["year"], meta["tmdb_id"], |
| 2576 | season, episode, meta["episode_title"]) |
| 2577 | part = _part_path(dest) |
| 2578 | try: |
| 2579 | st = os.stat(part) |
| 2580 | # Only once it cannot still be in flight. A remux may legitimately |
| 2581 | # run for MP4_REMUX_TIMEOUT, and the running job would rather find |
| 2582 | # its own file than a surprise. |
| 2583 | if (_mp4_jobs.get(key) != "running" |
| 2584 | and now - st.st_mtime > MP4_REMUX_TIMEOUT): |
| 2585 | out.append({"path": part, "bytes": st.st_size, |
| 2586 | "reason": "abandoned partial remux"}) |
| 2587 | except OSError: |
| 2588 | pass |
| 2589 | # A legacy mp4-cache copy is redundant ONLY once the library holds the |
| 2590 | # same episode and that copy passes the completeness gate. Both halves |
| 2591 | # matter: without the second, a truncated library copy would be grounds |
| 2592 | # for deleting the good legacy one. |
| 2593 | legacy = _mp4_path(kind, ident, season, episode, height) |
| 2594 | if legacy != dest and os.path.exists(legacy) and _library_has( |
| 2595 | kind, ident, season, episode): |
| 2596 | try: |
| 2597 | out.append({"path": legacy, "bytes": os.path.getsize(legacy), |
| 2598 | "reason": "superseded by the library copy"}) |
| 2599 | except OSError: |
| 2600 | pass |
| 2601 | return out |
| 2602 | |
| 2603 | |
| 2604 | def _reclaim() -> dict: |
| 2605 | """Remove everything _reclaimable() currently vouches for. |
| 2606 | |
| 2607 | The set is RE-DERIVED here rather than taken from the request. A caller does |
| 2608 | not get to name a path for deletion: the only paths that can ever be removed |
| 2609 | are ones this function independently proved were derived, a moment ago. |
| 2610 | """ |
| 2611 | freed, removed = 0, [] |
| 2612 | for item in _reclaimable(): |
| 2613 | try: |
| 2614 | os.remove(item["path"]) |
| 2615 | except OSError as exc: |
| 2616 | log.warning("reclaim: %s (%s)", item["path"], exc) |
| 2617 | continue |
| 2618 | freed += item["bytes"] |
| 2619 | removed.append(item["path"]) |
| 2620 | log.info("reclaimed %s (%s, %.0f MB)", os.path.basename(item["path"]), |
| 2621 | item["reason"], item["bytes"] / 1e6) |
| 2622 | return {"removed": removed, "freed": freed} |
| 2623 | |
| 2624 | |
| 2485 | 2625 | # ------------------------------ library audit ------------------------------- |
| 2486 | 2626 | # The truncation this exists for was invisible for five days and was found by |
| 2487 | 2627 | # watching an episode, not by any check. Every number needed to spot it was |
| … | … | @@ -2562,9 +2702,45 @@ tr.missing,tr.unknown{background:#fff8e1} |
| 2562 | 2702 | .tag.ok{background:#2e7d32;color:#fff} |
| 2563 | 2703 | button{padding:.3rem .7rem;cursor:pointer} |
| 2564 | 2704 | td.num{text-align:right;font-variant-numeric:tabular-nums} |
| 2705 | p.disk{color:#555} |
| 2706 | p.low{color:#b71c1c;font-weight:600} |
| 2565 | 2707 | """ |
| 2566 | 2708 | |
| 2567 | 2709 | |
| 2710 | def _gb(n): |
| 2711 | return f"{n / 1024 ** 3:.0f} GB" |
| 2712 | |
| 2713 | |
| 2714 | def _disk_html() -> str: |
| 2715 | """The budget, and whatever is provably safe to remove. |
| 2716 | |
| 2717 | Shown even when there is nothing to reclaim: the number that matters is the |
| 2718 | headroom, and the point of putting it here is that the season button can |
| 2719 | spend it twelve gigabytes at a time. |
| 2720 | """ |
| 2721 | disk = _disk() |
| 2722 | out = [] |
| 2723 | if disk: |
| 2724 | total, _used, free = disk |
| 2725 | low = free < LIBRARY_MIN_FREE_BYTES |
| 2726 | out.append( |
| 2727 | f"<p class='{'low' if low else 'disk'}'>" |
| 2728 | f"{_gb(free)} free of {_gb(total)}" |
| 2729 | f" · floor {_gb(LIBRARY_MIN_FREE_BYTES)}" |
| 2730 | + (" · <b>ingests are declining until this clears</b>" |
| 2731 | if low else "") + "</p>") |
| 2732 | items = _reclaimable() |
| 2733 | if not items: |
| 2734 | return "".join(out) + "<p class=disk>Nothing to reclaim.</p>" |
| 2735 | freed = sum(i["bytes"] for i in items) |
| 2736 | out.append(f"<form method=post action=/library/reclaim><p class=disk>" |
| 2737 | f"{len(items)} reclaimable ({freed / 1e6:.0f} MB): " |
| 2738 | + "; ".join(f"{html.escape(os.path.basename(i['path']))} " |
| 2739 | f"— {html.escape(i['reason'])}" for i in items) |
| 2740 | + " <button>Reclaim</button></p></form>") |
| 2741 | return "".join(out) |
| 2742 | |
| 2743 | |
| 2568 | 2744 | def _audit_html(rows: list[dict]) -> str: |
| 2569 | 2745 | def mins(seconds): |
| 2570 | 2746 | return "—" if not seconds else f"{seconds / 60:.0f}m" |
| … | … | @@ -2573,6 +2749,7 @@ def _audit_html(rows: list[dict]) -> str: |
| 2573 | 2749 | short = sum(1 for r in rows if r["status"] == AUDIT_SHORT) |
| 2574 | 2750 | body.append(f"<h1>Library ({len(rows)} fetched" |
| 2575 | 2751 | + (f", <b>{short} incomplete</b>" if short else "") + ")</h1>") |
| 2752 | body.append(_disk_html()) |
| 2576 | 2753 | body.append("<table><tr><th>Title</th><th></th><th>On disk</th>" |
| 2577 | 2754 | "<th>Expected</th><th></th><th>Checked</th><th></th></tr>") |
| 2578 | 2755 | for r in rows: |
| … | … | @@ -2640,6 +2817,20 @@ def library_reingest(): |
| 2640 | 2817 | return redirect("/library", code=303) |
| 2641 | 2818 | |
| 2642 | 2819 | |
| 2820 | @app.route("/library/reclaim", methods=["POST"]) |
| 2821 | def library_reclaim(): |
| 2822 | """POST only, like every other action here. |
| 2823 | |
| 2824 | Takes no arguments on purpose. There is no path parameter to get wrong, |
| 2825 | because the caller does not choose what goes -- _reclaim re-derives the set |
| 2826 | itself and will only ever remove what it can prove is derived. |
| 2827 | """ |
| 2828 | result = _reclaim() |
| 2829 | log.info("library: reclaimed %d file(s), %.0f MB", |
| 2830 | len(result["removed"]), result["freed"] / 1e6) |
| 2831 | return redirect("/library", code=303) |
| 2832 | |
| 2833 | |
| 2643 | 2834 | @app.route("/library/season", methods=["POST"]) |
| 2644 | 2835 | def library_season(): |
| 2645 | 2836 | """POST only, for the same reason as the re-ingest route.""" |