import os
import re
import sys
import time
import asyncio
import logging
import threading
import random
from pathlib import Path
from urllib.parse import urlparse
import requests
from tqdm import tqdm

import colorama
colorama.init(autoreset=False)

logging.basicConfig(
    level=logging.INFO,
    format="%(asctime)s [%(levelname)s] %(message)s",
    datefmt="%H:%M:%S",
)
log = logging.getLogger(__name__)

TAG_INFO = f"{colorama.Fore.CYAN}[INFO]{colorama.Style.RESET_ALL}"
TAG_WARN = f"{colorama.Fore.YELLOW}{colorama.Style.BRIGHT}[WARN]{colorama.Style.RESET_ALL}"
TAG_ERROR = f"{colorama.Fore.RED}{colorama.Style.BRIGHT}[ERROR]{colorama.Style.RESET_ALL}"
TAG_CURL = f"{colorama.Fore.BLUE}[curl_fetch]{colorama.Style.RESET_ALL}"

USER_AGENT = (
    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
    "AppleWebKit/537.36 (KHTML, like Gecko) "
    "Chrome/125.0.0.0 Safari/537.36"
)

HEADERS = {
    "User-Agent": USER_AGENT,
    "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8",
    "Accept-Language": "en-US,en;q=0.5",
    "Connection": "keep-alive",
}

STEALTH_JS = """() => {
    Object.defineProperty(navigator, 'webdriver', { get: () => false });
    Object.defineProperty(navigator, 'plugins', { get: () => [1, 2, 3] });
    Object.defineProperty(navigator, 'languages', { get: () => ['en-US', 'en'] });
}"""

def clean_write(msg: str):
    """Write text ensuring previous progress bar line fragments are cleanly overwritten."""
    tqdm.write(f"\r\x1b[2K{msg}")
    sys.stdout.flush()

class BarPositionPool:
    """Manages console line offsets for multiple concurrent tqdm progress bars."""
    def __init__(self, size):
        self.lock = threading.Lock()
        self.available = list(range(1, size + 1))

    def acquire(self):
        with self.lock:
            if self.available:
                return self.available.pop(0)
            return None

    def release(self, pos):
        with self.lock:
            if pos is not None and pos not in self.available:
                self.available.append(pos)
                self.available.sort()


def clean_filename(name: str) -> str:
    return re.sub(r'[^a-zA-Z0-9_.-]', '_', name)


def download_file(url: str, dest_path: Path, headers: dict = None, bar_position: int = 1, referer: str = None, cookies: dict = None) -> bool:
    """Download a file with requests showing a progress bar at a specific terminal line."""
    dest_path.parent.mkdir(parents=True, exist_ok=True)
    label = f"{dest_path.parent.name}/{dest_path.name}"

    req_headers = dict(HEADERS)
    if headers:
        req_headers.update(headers)
    if referer:
        req_headers["Referer"] = referer

    try:
        if ".m3u8" in url or "manifest" in url:
            import yt_dlp

            pbar = tqdm(
                desc=label[:25],
                unit="B",
                unit_scale=True,
                unit_divisor=1024,
                position=bar_position,
                leave=False,
                dynamic_ncols=True,
                colour="green",
            )

            class YtdlpTqdmHook:
                def __init__(self, pbar):
                    self.pbar = pbar
                    self.last_bytes = 0

                def __call__(self, d):
                    if d['status'] == 'downloading':
                        downloaded = d.get('downloaded_bytes', 0)
                        total = d.get('total_bytes') or d.get('total_bytes_estimate') or 0
                        if total and self.pbar.total is None:
                            self.pbar.total = total
                        delta = downloaded - self.last_bytes
                        if delta > 0:
                            self.pbar.update(delta)
                            self.last_bytes = downloaded
                    elif d['status'] == 'finished':
                        if self.pbar.total and self.last_bytes < self.pbar.total:
                            self.pbar.update(self.pbar.total - self.last_bytes)

            ydl_headers = dict(req_headers)
            if ydl_headers.get("User-Agent") == "stash-scraper/v1":
                ydl_headers.pop("User-Agent", None)

            ydl_opts = {
                'outtmpl': str(dest_path),
                'progress_hooks': [YtdlpTqdmHook(pbar)],
                'quiet': True,
                'no_warnings': True,
                'noprogress': True,
                'http_headers': ydl_headers,
                'downloader': {'default': 'ffmpeg'},
                'hls_use_mpegts': True,
                'hls_prefer_native': False,
            }
            try:
                with yt_dlp.YoutubeDL(ydl_opts) as ydl:
                    result = ydl.download([url])
                    if result != 0:
                        raise Exception("yt-dlp download failed with non-zero exit code")
            finally:
                pbar.close()
            return True

        # For direct MP4/media files, try curl_cffi first with TLS fingerprinting (bypasses CDN anti-bot SSL drops like veevcdn)
        curl_success = False
        try:
            from curl_cffi import requests as curl_req
            for attempt in range(3):
                try:
                    resp = curl_req.get(
                        url, impersonate="chrome", headers=req_headers,
                        cookies=cookies, stream=True, timeout=(15, 300),
                    )
                    resp.raise_for_status()
                    total = int(resp.headers.get("content-length", 0))

                    with tqdm(
                        total=total, unit="B", unit_scale=True, unit_divisor=1024,
                        desc=label[:25], position=bar_position, leave=False, dynamic_ncols=True, colour="green",
                    ) as pbar:
                        with open(dest_path, "wb") as f:
                            for chunk in resp.iter_content(chunk_size=65536):
                                if chunk:
                                    f.write(chunk)
                                    pbar.update(len(chunk))
                    curl_success = True
                    break
                except Exception as ce:
                    if attempt < 2:
                        time.sleep(1.5 * (attempt + 1))
                    else:
                        raise ce
            if curl_success:
                return True
        except ImportError:
            pass
        except Exception:
            # Fall back to standard requests if curl_cffi fails
            if dest_path.exists():
                try:
                    dest_path.unlink()
                except Exception:
                    pass

        # Fallback using requests
        for attempt in range(3):
            try:
                resp = requests.get(url, headers=req_headers, stream=True, timeout=(15, 300), cookies=cookies)
                resp.raise_for_status()

                if resp.raw._connection:
                    sock = resp.raw._connection.sock
                    if sock:
                        sock.settimeout(30.0)

                total = int(resp.headers.get("content-length", 0))

                with tqdm(
                    total=total, unit="B", unit_scale=True, unit_divisor=1024,
                    desc=label[:25], position=bar_position, leave=False, dynamic_ncols=True, colour="green",
                ) as pbar:
                    with open(dest_path, "wb") as f:
                        for chunk in resp.iter_content(chunk_size=65536):
                            if chunk:
                                f.write(chunk)
                                pbar.update(len(chunk))
                return True
            except Exception as re_err:
                if attempt < 2:
                    time.sleep(1.5 * (attempt + 1))
                else:
                    raise re_err
    except Exception as e:
        err_msg = str(e)
        if "UNEXPECTED_EOF_WHILE_READING" in err_msg or "EOF occurred" in err_msg or "Connection closed abruptly" in err_msg:
            clean_err = "Host CDN dropped connection (TLS/SSL gated or IP blocked by host)"
        elif "Max retries exceeded" in err_msg and "(" in err_msg:
            clean_err = err_msg.split("(")[0].strip()
        else:
            clean_err = err_msg
        clean_write(f"  {TAG_ERROR} Download failed ({label}): {clean_err}")
        if dest_path.exists():
            try:
                dest_path.unlink()
            except Exception:
                pass
        return False


def curl_fetch(url: str, cookies: dict = None, referer: str = None, retries: int = 5) -> str:
    """Fetch a page using curl_cffi with browser TLS impersonation. Returns HTML or empty string."""
    try:
        from curl_cffi import requests as curl_req
    except ImportError:
        return ""

    headers = dict(HEADERS)
    if referer:
        headers["Referer"] = referer

    for attempt in range(retries):
        try:
            req_cookies = dict(cookies) if cookies else None
            resp = curl_req.get(
                url, impersonate="chrome", headers=headers,
                cookies=req_cookies, timeout=30,
            )
            if resp.status_code == 200:
                # Ensure the page is not an anti-bot challenge interstitial (which returns HTTP 200 or 403)
                if any(ind in resp.text for ind in ("<title>One moment, please", "<title>Just a moment", "challenges.cloudflare.com", "challenge-platform")):
                    clean_write(f"    {TAG_CURL} {colorama.Fore.YELLOW}Interstitial bot-check page returned (attempt {attempt+1}/{retries}): {url}{colorama.Style.RESET_ALL}")
                else:
                    return resp.text
            if resp.status_code == 403:
                # If cookies had cf_clearance, Cloudflare 403s because the TLS fingerprint of curl_cffi
                # doesn't match the browser that generated cf_clearance. Retry without cf_clearance.
                if req_cookies and "cf_clearance" in req_cookies:
                    alt_cookies = {k: v for k, v in req_cookies.items() if k != "cf_clearance"}
                    try:
                        resp2 = curl_req.get(
                            url, impersonate="chrome", headers=headers,
                            cookies=alt_cookies or None, timeout=30,
                        )
                        if resp2.status_code == 200:
                            return resp2.text
                    except Exception:
                        pass
                clean_write(f"    {TAG_CURL} {colorama.Fore.YELLOW}403 Forbidden (attempt {attempt+1}/{retries}): {url}{colorama.Style.RESET_ALL}")
            elif resp.status_code == 429:
                wait = (2 ** attempt) * 8 + random.uniform(2.0, 6.0)
                clean_write(f"    {TAG_CURL} {colorama.Fore.YELLOW}429 Rate limited, waiting {wait:.1f}s (attempt {attempt+1}/{retries}): {url}{colorama.Style.RESET_ALL}")
                time.sleep(wait)
            else:
                clean_write(f"    {TAG_CURL} HTTP {resp.status_code} (attempt {attempt+1}/{retries}): {url}")
        except Exception as e:
            clean_write(f"    {TAG_CURL} {colorama.Fore.RED}Error (attempt {attempt+1}/{retries}): {e}{colorama.Style.RESET_ALL}")

        if attempt < retries - 1:
            time.sleep(2 * (attempt + 1) + random.uniform(0.5, 2.0))

    return ""


def extract_video_info_from_html(html: str) -> tuple:
    """Extract (video_src, uploader) from raw video page HTML."""
    src = ""
    uploader = ""

    source_match = re.search(r'<source\s+src="([^"]+)"', html)
    if source_match:
        src = source_match.group(1)

    if not src:
        video_match = re.search(r'<video[^>]+src="([^"]+)"', html)
        if video_match:
            src = video_match.group(1)

    if not src:
        ld_match = re.search(r'"contentUrl"\s*:\s*"([^"]+)"', html)
        if ld_match:
            src = ld_match.group(1)

    # Strip comments section to prevent extracting commenter profiles as the uploader
    main_html = html
    for marker in ['class="ul-comments"', 'class="videobas"', 'id="ul-comments"', 'class="comments"']:
        if marker in main_html:
            main_html = main_html.split(marker, 1)[0]
            break

    uploader_match = re.search(r'Uploader:\s*([^<\s]+)', main_html)
    if uploader_match:
        uploader = uploader_match.group(1)

    if not uploader:
        profile_match = re.search(r'<a[^>]*href="[^"]*profiles/[^"]*"[^>]*>\s*<img[^>]*alt="([^"]+)"', main_html)
        if profile_match:
            uploader = profile_match.group(1)

    return src.strip(), clean_filename(uploader) if uploader else "unknown"


def extract_video_links_from_html(html: str) -> list:
    """Extract unique video page URLs from a playlist page HTML."""
    links = re.findall(r'href="(https?://[^"]*?/videos/[^"]*?\.html)"', html)
    seen = set()
    unique = []
    for link in links:
        if link not in seen:
            seen.add(link)
            unique.append(link)
    return unique


def has_next_page(html: str) -> bool:
    """Check if a playlist page has a 'Next' pagination link."""
    return bool(re.search(r'<a[^>]*>[^<]*[Nn]ext[^<]*</a>', html))


class PlaywrightScraper:
    """Manages browser instance and page operations headlessly."""
    def __init__(self):
        self.playwright = None
        self.browser = None
        self.context = None

    async def start(self, headless=True):
        from playwright.async_api import async_playwright
        self.playwright = await async_playwright().start()
        
        launch_args = [
            "--disable-blink-features=AutomationControlled",
            "--no-sandbox",
            "--disable-dev-shm-usage",
        ]
        if headless:
            launch_args.append("--headless=new")
            
        try:
            self.browser = await self.playwright.chromium.launch(
                headless=headless,
                channel="chrome",
                args=launch_args
            )
        except Exception:
            self.browser = await self.playwright.chromium.launch(
                headless=headless,
                args=launch_args
            )
        self.context = await self.browser.new_context(
            user_agent=USER_AGENT,
            viewport={"width": 1920, "height": 1080},
        )

    async def new_page(self):
        page = await self.context.new_page()
        await page.add_init_script(STEALTH_JS)
        return page

    async def resolve_page(self, page, url: str, retries=3):
        for attempt in range(retries):
            try:
                await page.goto(url, wait_until="domcontentloaded", timeout=45000)
                await page.wait_for_timeout(4000)
                
                # Automatically handle age-gate overlays if present
                try:
                    overlay = await page.query_selector(".age-verification-overlay")
                    if overlay:
                        confirm_btn = await overlay.query_selector("button.btn-confirm, button:has-text('Enter'), button:has-text('Yes'), button:has-text('Agree'), button:has-text('18')")
                        if confirm_btn:
                            await confirm_btn.click()
                            # Wait for the overlay to disappear
                            await page.wait_for_selector(".age-verification-overlay", state="hidden", timeout=5000)
                except Exception:
                    pass

                title = await page.title()
                if not any(t in title for t in ("Just a moment", "One moment, please", "Attention Required")):
                    return
                clean_write("  Cloudflare challenge detected, waiting ...")
                
                # Check for Cloudflare Turnstile challenge container
                for loop in range(12): # Wait up to 60 seconds (12 * 5s) per attempt
                    await page.wait_for_timeout(5000)
                    title = await page.title()
                    if not any(t in title for t in ("Just a moment", "One moment, please", "Attention Required")):
                        return
                    
                    # Try to locate the challenge iframe and click near the center of the checkbox
                    try:
                        iframe = page.frame_locator('iframe[src*="challenges.cloudflare.com"]')
                        # Look for checkbox inside iframe if it's there
                        checkbox = iframe.locator('input[type="checkbox"], #challenge-stage, .ctp-checkbox-label')
                        if await checkbox.count() > 0:
                            # Try to click/focus to trigger validation
                            await checkbox.first.click(timeout=1000)
                            clean_write("    Attempted interaction with Turnstile checkbox...")
                    except Exception:
                        pass
            except Exception as e:
                clean_write(f"  Attempt {attempt + 1}/{retries} failed for {url}: {e}")
            if attempt < retries - 1:
                await page.wait_for_timeout(5000)
        raise RuntimeError(f"Could not resolve {url} after {retries} attempts")

    async def extract_media_source(self, page, url: str, video_selector="video"):
        """Navigates to a page, intercepts media requests, and extracts the direct video source."""
        media_urls = []
        
        # Intercept network requests for media files
        def handle_response(response):
            req = response.request
            content_type = response.headers.get("content-type", "").lower()
            url_lower = response.url.lower()
            
            # Skip static assets containing '.ts' or other extensions
            if any(x in content_type for x in ["text/css", "javascript", "image/", "font/"]):
                return
            if any(url_lower.endswith(ext) or f"{ext}?" in url_lower for ext in [".css", ".js", ".png", ".jpg", ".jpeg", ".gif", ".woff", ".svg", ".ico"]):
                return

            is_media = (
                req.resource_type in ["media", "video"] or
                "video/" in content_type or
                "application/x-mpegurl" in content_type or
                any(url_lower.endswith(ext) or f"{ext}?" in url_lower for ext in [".mp4", ".m3u8", ".webm", ".ts"])
            )
            if is_media and response.url not in media_urls:
                media_urls.append(response.url)

        page.on("response", handle_response)
        
        try:
            await self.resolve_page(page, url)
            
            # Try to trigger playback if the video player exists to force network requests
            try:
                play_buttons = [
                    "button.vjs-big-play-button", 
                    ".play-button", 
                    ".video-player", 
                    "video",
                    ".fp-play"
                ]
                for selector in play_buttons:
                    el = await page.query_selector(selector)
                    if el:
                        await el.click(timeout=1000)
                        await page.wait_for_timeout(2000)
                        break
            except Exception:
                pass
                
            # Allow some time for media requests to fire
            await page.wait_for_timeout(3000)
            
            if media_urls:
                # Return the largest/best URL found (prefer .mp4 or .m3u8 over small chunks)
                non_ts = [u for u in media_urls if not u.endswith(".ts")]
                if non_ts:
                    return non_ts[0]
                return media_urls[0]
                
            # Fallback to DOM inspection
            src = await page.evaluate(f"""(sel) => {{
                // 1. Check video tag
                const v = document.querySelector(sel);
                if (v && v.currentSrc) return v.currentSrc;
                if (v && v.src) return v.src;
                
                // 2. Check source tags
                const sources = document.querySelectorAll('video source');
                for (const s of sources) {{
                    if (s.src) return s.src;
                }}
                
                // 3. JSON-LD fallback
                const scripts = document.querySelectorAll('script[type="application/ld+json"]');
                for (const s of scripts) {{
                    try {{
                        const data = JSON.parse(s.innerText);
                        if (data.contentUrl) return data.contentUrl;
                        if (data.embedUrl) return data.embedUrl;
                    }} catch (e) {{}}
                }}
                return '';
            }}""", video_selector)
            
            return src.strip() if src else ""
        finally:
            page.remove_listener("response", handle_response)

    async def close(self):
        if self.context:
            await self.context.close()
        if self.browser:
            await self.browser.close()
        if self.playwright:
            await self.playwright.stop()


def append_log(log_path: Path, line: str):
    log_path.parent.mkdir(parents=True, exist_ok=True)
    with open(log_path, "a", encoding="utf-8") as f:
        f.write(line + "\n")


def is_already_downloaded(dest_path: Path) -> bool:
    return dest_path.exists() and dest_path.stat().st_size > 0