Automated daily backup: 2026-10-12 03:30
This commit is contained in:
gooner committed
1 parent
824cdedd7e
commit
2b31e92673
949 files changed
+381680
-1825
No files matched your search
+122
-49
@@ -11,6 +11,9 @@ from urllib.parse import urlparse
|
||||
import requests
|
||||
from tqdm import tqdm
|
||||
|
||||
import colorama
|
||||
colorama.init(autoreset=False)
|
||||
|
||||
logging.basicConfig(
|
||||
level=logging.INFO,
|
||||
format="%(asctime)s [%(levelname)s] %(message)s",
|
||||
@@ -18,6 +21,11 @@ logging.basicConfig(
|
||||
)
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
TAG_INFO = f"{colorama.Fore.CYAN}[INFO]{colorama.Style.RESET_ALL}"
|
||||
TAG_WARN = f"{colorama.Fore.YELLOW}{colorama.Style.BRIGHT}[WARN]{colorama.Style.RESET_ALL}"
|
||||
TAG_ERROR = f"{colorama.Fore.RED}{colorama.Style.BRIGHT}[ERROR]{colorama.Style.RESET_ALL}"
|
||||
TAG_CURL = f"{colorama.Fore.BLUE}[curl_fetch]{colorama.Style.RESET_ALL}"
|
||||
|
||||
USER_AGENT = (
|
||||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
|
||||
"AppleWebKit/537.36 (KHTML, like Gecko) "
|
||||
@@ -37,6 +45,11 @@ STEALTH_JS = """() => {
|
||||
Object.defineProperty(navigator, 'languages', { get: () => ['en-US', 'en'] });
|
||||
}"""
|
||||
|
||||
def clean_write(msg: str):
|
||||
"""Write text ensuring previous progress bar line fragments are cleanly overwritten."""
|
||||
tqdm.write(f"\r\x1b[2K{msg}")
|
||||
sys.stdout.flush()
|
||||
|
||||
class BarPositionPool:
|
||||
"""Manages console line offsets for multiple concurrent tqdm progress bars."""
|
||||
def __init__(self, size):
|
||||
@@ -82,7 +95,8 @@ def download_file(url: str, dest_path: Path, headers: dict = None, bar_position:
|
||||
unit_divisor=1024,
|
||||
position=bar_position,
|
||||
leave=False,
|
||||
ncols=80,
|
||||
dynamic_ncols=True,
|
||||
colour="green",
|
||||
)
|
||||
|
||||
class YtdlpTqdmHook:
|
||||
@@ -112,10 +126,12 @@ def download_file(url: str, dest_path: Path, headers: dict = None, bar_position:
|
||||
'outtmpl': str(dest_path),
|
||||
'progress_hooks': [YtdlpTqdmHook(pbar)],
|
||||
'quiet': True,
|
||||
'no_warnings': True,
|
||||
'noprogress': True,
|
||||
'http_headers': ydl_headers,
|
||||
'downloader': 'ffmpeg',
|
||||
'downloader': {'default': 'ffmpeg'},
|
||||
'hls_use_mpegts': True,
|
||||
'hls_prefer_native': False,
|
||||
}
|
||||
try:
|
||||
with yt_dlp.YoutubeDL(ydl_opts) as ydl:
|
||||
@@ -126,19 +142,63 @@ def download_file(url: str, dest_path: Path, headers: dict = None, bar_position:
|
||||
pbar.close()
|
||||
return True
|
||||
|
||||
if cookies:
|
||||
# For direct MP4/media files, try curl_cffi first with TLS fingerprinting (bypasses CDN anti-bot SSL drops like veevcdn)
|
||||
curl_success = False
|
||||
try:
|
||||
from curl_cffi import requests as curl_req
|
||||
for attempt in range(3):
|
||||
try:
|
||||
resp = curl_req.get(
|
||||
url, impersonate="chrome", headers=req_headers,
|
||||
cookies=cookies, stream=True, timeout=(15, 300),
|
||||
)
|
||||
resp.raise_for_status()
|
||||
total = int(resp.headers.get("content-length", 0))
|
||||
|
||||
with tqdm(
|
||||
total=total, unit="B", unit_scale=True, unit_divisor=1024,
|
||||
desc=label[:25], position=bar_position, leave=False, dynamic_ncols=True, colour="green",
|
||||
) as pbar:
|
||||
with open(dest_path, "wb") as f:
|
||||
for chunk in resp.iter_content(chunk_size=65536):
|
||||
if chunk:
|
||||
f.write(chunk)
|
||||
pbar.update(len(chunk))
|
||||
curl_success = True
|
||||
break
|
||||
except Exception as ce:
|
||||
if attempt < 2:
|
||||
time.sleep(1.5 * (attempt + 1))
|
||||
else:
|
||||
raise ce
|
||||
if curl_success:
|
||||
return True
|
||||
except ImportError:
|
||||
pass
|
||||
except Exception:
|
||||
# Fall back to standard requests if curl_cffi fails
|
||||
if dest_path.exists():
|
||||
try:
|
||||
dest_path.unlink()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
# Fallback using requests
|
||||
for attempt in range(3):
|
||||
try:
|
||||
from curl_cffi import requests as curl_req
|
||||
resp = curl_req.get(
|
||||
url, impersonate="chrome", headers=req_headers,
|
||||
cookies=cookies, stream=True, timeout=(15, 300),
|
||||
)
|
||||
resp = requests.get(url, headers=req_headers, stream=True, timeout=(15, 300), cookies=cookies)
|
||||
resp.raise_for_status()
|
||||
|
||||
if resp.raw._connection:
|
||||
sock = resp.raw._connection.sock
|
||||
if sock:
|
||||
sock.settimeout(30.0)
|
||||
|
||||
total = int(resp.headers.get("content-length", 0))
|
||||
|
||||
with tqdm(
|
||||
total=total, unit="B", unit_scale=True, unit_divisor=1024,
|
||||
desc=label[:25], position=bar_position, leave=False, ncols=80,
|
||||
desc=label[:25], position=bar_position, leave=False, dynamic_ncols=True, colour="green",
|
||||
) as pbar:
|
||||
with open(dest_path, "wb") as f:
|
||||
for chunk in resp.iter_content(chunk_size=65536):
|
||||
@@ -146,31 +206,20 @@ def download_file(url: str, dest_path: Path, headers: dict = None, bar_position:
|
||||
f.write(chunk)
|
||||
pbar.update(len(chunk))
|
||||
return True
|
||||
except ImportError:
|
||||
pass
|
||||
|
||||
resp = requests.get(url, headers=req_headers, stream=True, timeout=(15, 300), cookies=cookies)
|
||||
resp.raise_for_status()
|
||||
|
||||
if resp.raw._connection:
|
||||
sock = resp.raw._connection.sock
|
||||
if sock:
|
||||
sock.settimeout(30.0)
|
||||
|
||||
total = int(resp.headers.get("content-length", 0))
|
||||
|
||||
with tqdm(
|
||||
total=total, unit="B", unit_scale=True, unit_divisor=1024,
|
||||
desc=label[:25], position=bar_position, leave=False, ncols=80,
|
||||
) as pbar:
|
||||
with open(dest_path, "wb") as f:
|
||||
for chunk in resp.iter_content(chunk_size=65536):
|
||||
if chunk:
|
||||
f.write(chunk)
|
||||
pbar.update(len(chunk))
|
||||
return True
|
||||
except Exception as re_err:
|
||||
if attempt < 2:
|
||||
time.sleep(1.5 * (attempt + 1))
|
||||
else:
|
||||
raise re_err
|
||||
except Exception as e:
|
||||
tqdm.write(f" [ERROR] Download failed ({label}): {e}")
|
||||
err_msg = str(e)
|
||||
if "UNEXPECTED_EOF_WHILE_READING" in err_msg or "EOF occurred" in err_msg or "Connection closed abruptly" in err_msg:
|
||||
clean_err = "Host CDN dropped connection (TLS/SSL gated or IP blocked by host)"
|
||||
elif "Max retries exceeded" in err_msg and "(" in err_msg:
|
||||
clean_err = err_msg.split("(")[0].strip()
|
||||
else:
|
||||
clean_err = err_msg
|
||||
clean_write(f" {TAG_ERROR} Download failed ({label}): {clean_err}")
|
||||
if dest_path.exists():
|
||||
try:
|
||||
dest_path.unlink()
|
||||
@@ -192,22 +241,40 @@ def curl_fetch(url: str, cookies: dict = None, referer: str = None, retries: int
|
||||
|
||||
for attempt in range(retries):
|
||||
try:
|
||||
req_cookies = dict(cookies) if cookies else None
|
||||
resp = curl_req.get(
|
||||
url, impersonate="chrome", headers=headers,
|
||||
cookies=cookies, timeout=30,
|
||||
cookies=req_cookies, timeout=30,
|
||||
)
|
||||
if resp.status_code == 200:
|
||||
return resp.text
|
||||
# Ensure the page is not an anti-bot challenge interstitial (which returns HTTP 200 or 403)
|
||||
if any(ind in resp.text for ind in ("<title>One moment, please", "<title>Just a moment", "challenges.cloudflare.com", "challenge-platform")):
|
||||
clean_write(f" {TAG_CURL} {colorama.Fore.YELLOW}Interstitial bot-check page returned (attempt {attempt+1}/{retries}): {url}{colorama.Style.RESET_ALL}")
|
||||
else:
|
||||
return resp.text
|
||||
if resp.status_code == 403:
|
||||
tqdm.write(f" [curl_fetch] 403 Forbidden (attempt {attempt+1}/{retries}): {url}")
|
||||
# If cookies had cf_clearance, Cloudflare 403s because the TLS fingerprint of curl_cffi
|
||||
# doesn't match the browser that generated cf_clearance. Retry without cf_clearance.
|
||||
if req_cookies and "cf_clearance" in req_cookies:
|
||||
alt_cookies = {k: v for k, v in req_cookies.items() if k != "cf_clearance"}
|
||||
try:
|
||||
resp2 = curl_req.get(
|
||||
url, impersonate="chrome", headers=headers,
|
||||
cookies=alt_cookies or None, timeout=30,
|
||||
)
|
||||
if resp2.status_code == 200:
|
||||
return resp2.text
|
||||
except Exception:
|
||||
pass
|
||||
clean_write(f" {TAG_CURL} {colorama.Fore.YELLOW}403 Forbidden (attempt {attempt+1}/{retries}): {url}{colorama.Style.RESET_ALL}")
|
||||
elif resp.status_code == 429:
|
||||
wait = (2 ** attempt) * 8 + random.uniform(2.0, 6.0)
|
||||
tqdm.write(f" [curl_fetch] 429 Rate limited, waiting {wait:.1f}s (attempt {attempt+1}/{retries}): {url}")
|
||||
clean_write(f" {TAG_CURL} {colorama.Fore.YELLOW}429 Rate limited, waiting {wait:.1f}s (attempt {attempt+1}/{retries}): {url}{colorama.Style.RESET_ALL}")
|
||||
time.sleep(wait)
|
||||
else:
|
||||
tqdm.write(f" [curl_fetch] HTTP {resp.status_code} (attempt {attempt+1}/{retries}): {url}")
|
||||
clean_write(f" {TAG_CURL} HTTP {resp.status_code} (attempt {attempt+1}/{retries}): {url}")
|
||||
except Exception as e:
|
||||
tqdm.write(f" [curl_fetch] Error (attempt {attempt+1}/{retries}): {e}")
|
||||
clean_write(f" {TAG_CURL} {colorama.Fore.RED}Error (attempt {attempt+1}/{retries}): {e}{colorama.Style.RESET_ALL}")
|
||||
|
||||
if attempt < retries - 1:
|
||||
time.sleep(2 * (attempt + 1) + random.uniform(0.5, 2.0))
|
||||
@@ -289,11 +356,17 @@ class PlaywrightScraper:
|
||||
if headless:
|
||||
launch_args.append("--headless=new")
|
||||
|
||||
self.browser = await self.playwright.chromium.launch(
|
||||
headless=headless,
|
||||
channel="chrome",
|
||||
args=launch_args
|
||||
)
|
||||
try:
|
||||
self.browser = await self.playwright.chromium.launch(
|
||||
headless=headless,
|
||||
channel="chrome",
|
||||
args=launch_args
|
||||
)
|
||||
except Exception:
|
||||
self.browser = await self.playwright.chromium.launch(
|
||||
headless=headless,
|
||||
args=launch_args
|
||||
)
|
||||
self.context = await self.browser.new_context(
|
||||
user_agent=USER_AGENT,
|
||||
viewport={"width": 1920, "height": 1080},
|
||||
@@ -323,15 +396,15 @@ class PlaywrightScraper:
|
||||
pass
|
||||
|
||||
title = await page.title()
|
||||
if "Just a moment" not in title:
|
||||
if not any(t in title for t in ("Just a moment", "One moment, please", "Attention Required")):
|
||||
return
|
||||
tqdm.write(" Cloudflare challenge detected, waiting ...")
|
||||
clean_write(" Cloudflare challenge detected, waiting ...")
|
||||
|
||||
# Check for Cloudflare Turnstile challenge container
|
||||
for loop in range(12): # Wait up to 60 seconds (12 * 5s) per attempt
|
||||
await page.wait_for_timeout(5000)
|
||||
title = await page.title()
|
||||
if "Just a moment" not in title:
|
||||
if not any(t in title for t in ("Just a moment", "One moment, please", "Attention Required")):
|
||||
return
|
||||
|
||||
# Try to locate the challenge iframe and click near the center of the checkbox
|
||||
@@ -342,11 +415,11 @@ class PlaywrightScraper:
|
||||
if await checkbox.count() > 0:
|
||||
# Try to click/focus to trigger validation
|
||||
await checkbox.first.click(timeout=1000)
|
||||
tqdm.write(" Attempted interaction with Turnstile checkbox...")
|
||||
clean_write(" Attempted interaction with Turnstile checkbox...")
|
||||
except Exception:
|
||||
pass
|
||||
except Exception as e:
|
||||
tqdm.write(f" Attempt {attempt + 1}/{retries} failed for {url}: {e}")
|
||||
clean_write(f" Attempt {attempt + 1}/{retries} failed for {url}: {e}")
|
||||
if attempt < retries - 1:
|
||||
await page.wait_for_timeout(5000)
|
||||
raise RuntimeError(f"Could not resolve {url} after {retries} attempts")
|
||||
|
||||
Reference in new issue
Block a user