import os
import requests
import time
import sys
from concurrent.futures import ThreadPoolExecutor
import threading

# Root directory where folders with links.txt are located
root_dir = r'V:\smalldata\niggers\redgifs\videos'

# Configuration
MAX_WORKERS = 10  # Number of simultaneous downloads
HEADERS = {
    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
    'Accept': 'video/webm,video/ogg,video/*;q=0.9,application/ogg;q=0.7,audio/*;q=0.6,*/*;q=0.5',
    'Referer': 'https://www.redgifs.com/',
    'Connection': 'keep-alive',
}

# Thread-safe progress reporting
progress_lock = threading.Lock()
completed_count = 0

def progress_bar(current, total, bar_length=40):
    fraction = current / total if total > 0 else 1
    arrow = int(fraction * bar_length - 1) * '=' + '>'
    padding = int(bar_length - len(arrow)) * ' '
    ending = '\n' if current == total else '\r'
    print(f'Progress: [{arrow}{padding}] {current}/{total} ({fraction*100:.2f}%)', end=ending)

def download_url(url, download_dir, folder_failed_log, total):
    global completed_count
    filename = os.path.join(download_dir, os.path.basename(url))
    
    # Skip if already downloaded
    if os.path.exists(filename) and os.path.getsize(filename) > 0:
        with progress_lock:
            completed_count += 1
            progress_bar(completed_count, total)
        return

    retries = 3
    success = False
    
    while retries > 0 and not success:
        try:
            response = requests.get(url, headers=HEADERS, stream=True, timeout=30)
            if response.status_code == 200:
                with open(filename, 'wb') as f:
                    for chunk in response.iter_content(chunk_size=8192):
                        f.write(chunk)
                success = True
                time.sleep(0.1) # Minimal sleep for parallel threads
            elif response.status_code == 429:
                # If rate limited, we wait longer but it affects all threads
                time.sleep(10)
                retries -= 1
            else:
                retries = 0
        except Exception:
            retries -= 1
            time.sleep(2)
            
    if not success:
        with progress_lock:
            with open(folder_failed_log, 'a') as fail_log:
                fail_log.write(f"{url}\n")
    
    with progress_lock:
        completed_count += 1
        progress_bar(completed_count, total)

def download_from_file(file_path, download_dir):
    global completed_count
    if not os.path.exists(file_path):
        return

    folder_failed_log = os.path.join(download_dir, 'failed_downloads.txt')

    with open(file_path, 'r', encoding='utf-8') as f:
        urls = [line.strip() for line in f if line.strip() and not line.strip().startswith('#')]

    total = len(urls)
    folder_name = os.path.basename(download_dir)
    print(f"\nProcessing folder: {folder_name} ({total} files)")
    
    completed_count = 0
    with ThreadPoolExecutor(max_workers=MAX_WORKERS) as executor:
        for url in urls:
            executor.submit(download_url, url, download_dir, folder_failed_log, total)

# Main execution
if not os.path.exists(root_dir):
    print(f"Root directory not found: {root_dir}")
    sys.exit(1)

subfolders = [f.path for f in os.scandir(root_dir) if f.is_dir()]
print(f"Found {len(subfolders)} folders. Starting parallel batch download...")

for folder in subfolders:
    links_file = os.path.join(folder, 'links.txt')
    if os.path.exists(links_file):
        download_from_file(links_file, folder)

print("\nBatch download process complete.")