Automated daily backup: 2026-10-13 03:08

downloader.py

@@ -1852,7 +1852,7 @@ def _tickzoo_extract_video_links_from_html(html: str) -> list:
     return video_links
 
 
-async def _tickzoo_scrape_tag_playlist(playlist_url: str, scraper, queue, counter, overall_bar):
+async def _tickzoo_scrape_tag_playlist(playlist_url: str, cookies, scraper, queue, counter, overall_bar):
     """Scrape all video URLs across pages from a tickzoo.tv tag or category playlist."""
     seen_videos = set()
     curr_url = playlist_url
@@ -1860,7 +1860,7 @@ async def _tickzoo_scrape_tag_playlist(playlist_url: str, scraper, queue, counte
 
     while curr_url:
         log_info(f"Scraping tag page {page_num}: {curr_url}", indent=1)
-        html = await asyncio.to_thread(scraper_core.curl_fetch, curr_url, referer="https://tickzoo.tv/")
+        html = await asyncio.to_thread(scraper_core.curl_fetch, curr_url, cookies=cookies, referer="https://tickzoo.tv/")
         if not html:
             log_warn(f"Static fetch returned empty/blocked. Trying browser for tag page {page_num}...", indent=2)
             page = await scraper.new_page()
@@ -1907,13 +1907,13 @@ async def _tickzoo_scrape_tag_playlist(playlist_url: str, scraper, queue, counte
             break
 
 
-async def _tickzoo_producer(urls, scraper, queue, counter, overall_bar):
+async def _tickzoo_producer(urls, cookies, scraper, queue, counter, overall_bar):
     """Feed direct video URLs or scrape tag playlists dynamically into the queue."""
     for url in urls:
         if _tickzoo_is_tag_or_playlist(url):
             log_info(f"Scraping tag archive: {url}", indent=1)
             try:
-                await _tickzoo_scrape_tag_playlist(url, scraper, queue, counter, overall_bar)
+                await _tickzoo_scrape_tag_playlist(url, cookies, scraper, queue, counter, overall_bar)
             except Exception as e:
                 log_error(f"Error scraping tag {url}: {e}", indent=1)
         elif _tickzoo_is_video_link(url):
@@ -1928,7 +1928,7 @@ async def _tickzoo_producer(urls, scraper, queue, counter, overall_bar):
         await asyncio.sleep(random.uniform(0.3, 0.8))
 
 
-async def _tickzoo_worker(queue, scraper, download_dir, skip_existing, overall_bar, bar_pool):
+async def _tickzoo_worker(queue, cookies, scraper, download_dir, skip_existing, overall_bar, bar_pool):
     """Worker processing video URLs sequentially or concurrently with shared browser."""
     while True:
         url = await queue.get()
@@ -1936,7 +1936,7 @@ async def _tickzoo_worker(queue, scraper, download_dir, skip_existing, overall_b
             queue.task_done()
             break
         try:
-            await _tickzoo_process_one(scraper, url, download_dir, skip_existing, overall_bar, bar_pool)
+            await _tickzoo_process_one(scraper, url, cookies, download_dir, skip_existing, overall_bar, bar_pool)
         except Exception as e:
             log_error(f"Error processing {url}: {e}", indent=1)
             overall_bar.update(1)
@@ -1949,15 +1949,18 @@ async def _tickzoo_worker(queue, scraper, download_dir, skip_existing, overall_b
 def _tickzoo_extract_embed(html: str) -> str:
     """Pull the video iframe src out of the <textarea id="iframe"> embed box."""
     m = re.search(r"<textarea[^>]*id=[\"']iframe[\"'][^>]*>(.*?)</textarea>", html, re.S | re.I)
+    if not m:
+        # Also check any textarea containing iframe
+        m = re.search(r"<textarea[^>]*>(.*?&lt;iframe.*?)</textarea>", html, re.S | re.I)
     if not m:
         return ""
-    inner = m.group(1)
+    inner = unescape(m.group(1))
     m = re.search(r"<iframe\b[^>]*\bsrc\s*=\s*[\"']([^\"']+)[\"']", inner, re.I)
     if not m:
         m = re.search(r"<iframe\b[^>]*\bsrc\s*=\s*([^\"'>\s]+)", inner, re.I)
     if not m:
         return ""
-    src = unescape(m.group(1).strip())
+    src = m.group(1).strip()
     if any(ad in src.lower() for ad in _TICKZOO_AD_HOSTS):
         return ""
     return src
@@ -2003,13 +2006,13 @@ async def _tickzoo_resolve_media(scraper, embed_url: str, referer_url: str) -> s
     """Resolve the direct media URL from a tickzoo embed host via headless Chrome.
 
     Handles every embed host seen on tickzoo.tv:
-      * firestream.to  -> POST /api/videos/<id>/resolve returns JSON
-                          {signedVideoUrl: "<m3u8>", ...}
-      * hgcloud.to     -> HLS master manifest fires as a network response (jwplayer)
-      * rubyvidhub.com -> HLS master manifest fires as a network response
-      * veev.to        -> the mp4 URL lives in <video>.currentSrc after muted
-                          play(); its CDN is TLS/network gated, so the download
-                          itself may fail from some networks.
+      * firestream.to / firestream.site -> POST /api/videos/<id>/resolve returns JSON
+                                           {signedVideoUrl: "<m3u8>", ...}
+      * hgcloud.to                     -> HLS master manifest fires as a network response (jwplayer)
+      * rubyvidhub.com                 -> HLS master manifest fires as a network response
+      * veev.to                        -> the mp4 URL lives in <video>.currentSrc after muted
+                                          play(); its CDN is TLS/network gated, so the download
+                                          itself may fail from some networks.
     Returns the best scoring URL, or "" if nothing resolvable.
     """
     page = await scraper.new_page()
@@ -2049,41 +2052,47 @@ async def _tickzoo_resolve_media(scraper, embed_url: str, referer_url: str) -> s
     page.on("response", lambda r: asyncio.create_task(on_response(r)))
     try:
         log_info(f"Resolving embed {embed_url}", indent=2)
-        await page.goto(embed_url, wait_until="domcontentloaded", timeout=45000, referer=referer_url)
-        await page.wait_for_timeout(4000)
-
-        # Trigger playback so the player actually requests the media.
-        for selector in ("button.vjs-big-play-button", ".play-button", "[class*='play']", "video"):
-            try:
-                el = await page.query_selector(selector)
-                if el:
-                    await el.click(timeout=1200)
-                    await page.wait_for_timeout(1500)
-                    break
-            except Exception:
-                continue
+        # firestream resolves its signed token via client-side JS POST after full bundle load
+        wait_cond = "networkidle" if ("firestream" in embed_url.lower()) else "domcontentloaded"
         try:
-            await page.evaluate(
-                """() => { const v = document.querySelector('video');
-                           if (v) { v.muted = true; v.play().catch(()=>{}); } }"""
-            )
+            await page.goto(embed_url, wait_until=wait_cond, timeout=30000, referer=referer_url)
         except Exception:
             pass
+        await page.wait_for_timeout(2000)
 
-        # Some players (firestream) answer late / intermittently - poll a while.
+        # Trigger playback if candidates not yet found so other players (jwplayer, plyr, veev) request the media.
+        if not candidates:
+            for selector in ("button.vjs-big-play-button", ".play-button", "button[class*='play']", "video"):
+                try:
+                    el = await page.query_selector(selector)
+                    if el:
+                        await el.click(timeout=1200)
+                        await page.wait_for_timeout(1500)
+                        break
+                except Exception:
+                    continue
+            try:
+                await page.evaluate(
+                    """() => { const v = document.querySelector('video');
+                               if (v) { v.muted = true; v.play().catch(()=>{}); } }"""
+                )
+            except Exception:
+                pass
+
+        # Poll DOM video.src if stream URL was set directly in video element
         dom_js = """() => { const v = document.querySelector('video');
                            if (!v) return '';
                            return (v.currentSrc || v.src || ''); }"""
         for _ in range(8):
-            await page.wait_for_timeout(4000)
+            if candidates:
+                break
+            await page.wait_for_timeout(2000)
             try:
                 src = await page.evaluate(dom_js)
                 if src:
                     note(src, _tickzoo_score_media_url(src))
             except Exception:
                 pass
-            if candidates:
-                break
 
         good = {u: s for u, s in candidates.items() if s > 0}
         if not good:
@@ -2099,7 +2108,7 @@ async def _tickzoo_resolve_media(scraper, embed_url: str, referer_url: str) -> s
             pass
 
 
-async def _tickzoo_process_one(scraper, url, download_dir, skip_existing, overall_bar, bar_pool):
+async def _tickzoo_process_one(scraper, url, cookies, download_dir, skip_existing, overall_bar, bar_pool):
     """Fetch the tickzoo page, resolve its embed, download the video."""
     if is_filtered(url):
         log_filtered(url, indent=2)
@@ -2107,7 +2116,7 @@ async def _tickzoo_process_one(scraper, url, download_dir, skip_existing, overal
         return
 
     log_processing(url, indent=1)
-    html = await asyncio.to_thread(scraper_core.curl_fetch, url, referer="https://tickzoo.tv/")
+    html = await asyncio.to_thread(scraper_core.curl_fetch, url, cookies=cookies, referer="https://tickzoo.tv/")
     if not html:
         log_warn("Static fetch returned empty/blocked. Trying browser for video page...", indent=2)
         page = await scraper.new_page()
@@ -2200,19 +2209,46 @@ async def tickzoo_process_urls(download_dir, urls, concurrency, skip_existing):
 
     scraper = scraper_core.PlaywrightScraper()
     await scraper.start()
+
+    # Pre-warm cookies using browser on first target
+    cookies = {}
+    warmup_url = valid_targets[0]
+    log_info(f"Gathering initial session cookies from {warmup_url}...", indent=1)
     try:
-        # Launch workers (usually concurrency 1 or 2 for tickzoo embed stability)
+        page = await scraper.new_page()
+        try:
+            await scraper.resolve_page(page, warmup_url)
+            raw = await scraper.context.cookies()
+            cookies = {
+                c["name"]: c["value"]
+                for c in raw
+                if "tickzoo.tv" in c.get("domain", "") and c["name"] != "cf_clearance"
+            }
+        finally:
+            await page.close()
+    except Exception as e:
+        log_warn(f"Session warmup failed: {e}", indent=1)
+
+    log_info(f"Got cookies: {list(cookies.keys())}", indent=1)
+
+    try:
+        # Launch workers (they immediately begin consuming from queue)
         workers = [
-            asyncio.create_task(_tickzoo_worker(queue, scraper, download_dir, skip_existing, overall_bar, bar_pool))
+            asyncio.create_task(_tickzoo_worker(queue, cookies, scraper, download_dir, skip_existing, overall_bar, bar_pool))
             for _ in range(concurrency)
         ]
 
-        await _tickzoo_producer(valid_targets, scraper, queue, counter, overall_bar)
+        # Run producer concurrently with workers
+        async def run_producer():
+            try:
+                await _tickzoo_producer(valid_targets, cookies, scraper, queue, counter, overall_bar)
+            finally:
+                for _ in range(concurrency):
+                    await queue.put(None)
 
-        for _ in range(concurrency):
-            await queue.put(None)
+        producer_task = asyncio.create_task(run_producer())
 
-        await asyncio.gather(*workers)
+        await asyncio.gather(producer_task, *workers)
     finally:
         try:
             await scraper.close()