import requests import time import os import re import urllib.parse from bs4 import BeautifulSoup from sqlalchemy.orm import Session from ..database import Program, Episode, Config from ..diagnostics import ScraperError, invalid_response MAX_REQUESTS_LIMIT = 50 BASE_URL = "https://nowyswiat.online" def get_cookie_for_url(cookie_jar, name, url): """Wybiera najbardziej szczegółowe cookie, gdy jar zawiera duplikaty nazwy.""" parsed_url = urllib.parse.urlparse(url) host = parsed_url.hostname.lower() path = parsed_url.path or "/" candidates = [] for cookie in cookie_jar: if cookie.name != name or (cookie.secure and parsed_url.scheme != "https"): continue domain = cookie.domain.lstrip(".").lower() if domain and not (host == domain or host.endswith(f".{domain}")): continue cookie_path = cookie.path or "/" if not path.startswith(cookie_path.rstrip("/") or "/"): continue candidates.append(cookie) if not candidates: return None selected = max(candidates, key=lambda cookie: (len(cookie.path or "/"), len(cookie.domain or ""))) for cookie in candidates: if cookie is not selected: cookie_jar.clear(cookie.domain, cookie.path, cookie.name) return selected.value def parse_polish_date(date_str): months = { "stycznia": "01", "lutego": "02", "marca": "03", "kwietnia": "04", "maja": "05", "czerwca": "06", "lipca": "07", "sierpnia": "08", "września": "09", "października": "10", "listopada": "11", "grudnia": "12", "styczeń": "01", "luty": "02", "marzec": "03", "kwiecień": "04", "maj": "05", "czerwiec": "06", "lipiec": "07", "sierpień": "08", "wrzesień": "09", "październik": "10", "listopad": "11", "grudzień": "12" } parts = date_str.lower().split() if len(parts) == 3: day = parts[0].zfill(2) month = months.get(parts[1], "01") year = parts[2] return f"{year}-{month}-{day}" return date_str class RNScraper: def __init__(self, db: Session, logger=print, progress_callback=None, warning_callback=None, stop_flag=None): self.db = db self.logger = logger self.progress_callback = progress_callback self.warning_callback = warning_callback self.stop_flag = stop_flag self.requests_made = 0 self.start_time = time.time() self.max_execution_time = 900 # 15 minut max na cały sync limit_conf = self.db.query(Config).filter_by(key="rns_limit").first() self.backfill_limit = int(limit_conf.value) if limit_conf and limit_conf.value.isdigit() else 50 hard_limit_conf = self.db.query(Config).filter_by(key="rns_hard_limit").first() self.hard_limit = int(hard_limit_conf.value) if hard_limit_conf and hard_limit_conf.value.isdigit() else 500 self.session = requests.Session() self.session.headers.update({"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64)"}) if self.progress_callback: self.progress_callback(self.requests_made, self.hard_limit) def check_timeout(self): if self.stop_flag and self.stop_flag(): raise Exception("Zatrzymano na żądanie użytkownika.") if time.time() - self.start_time > self.max_execution_time: raise Exception(f"Przekroczono limit czasu wykonywania skryptu ({self.max_execution_time // 60} min). Zatrzymano awaryjnie.") def _get_html(self, url): self.check_timeout() if self.requests_made >= self.hard_limit: return None for attempt in range(1, 4): if self.requests_made >= self.hard_limit: return None self.requests_made += 1 if self.progress_callback: self.progress_callback(self.requests_made, self.hard_limit) try: r = self.session.get(url, timeout=15) if r.status_code == 200: return r.text invalid_response("rns", "HTTP_ERROR", "Błąd pobierania strony", url, r) except requests.exceptions.Timeout as e: if attempt == 3: raise ScraperError("rns", "NETWORK_TIMEOUT", "Trzy próby pobrania strony zakończyły się timeoutem", url) from e self.logger(f"RNŚ: Timeout dla {url}, próba {attempt}/3. Ponawiam za 30 sekund.") self.check_timeout() time.sleep(30) except Exception as e: if isinstance(e, ScraperError): raise raise ScraperError("rns", "NETWORK_ERROR", f"Błąd zapytania: {e}", url) from e return None def perform_login(self): email = self.db.query(Config).filter_by(key="rns_email").first() password = self.db.query(Config).filter_by(key="rns_password").first() if not email or not password: self.logger("RNŚ: Brak danych logowania w bazie.") return False self.logger("RNŚ: Inicjalizacja logowania (pobieranie CSRF)...") try: r1 = self.session.get("https://nowyswiat.online/konto/zaloguj", timeout=15) except Exception as e: self.logger(f"RNŚ: Sieć zablokowała pobieranie CSRF: {e}") return False csrf_token = get_cookie_for_url( self.session.cookies, "csrf_cookie_neocms", "https://nowyswiat.online/konto/zaloguj", ) if not csrf_token: self.logger("RNŚ: Nie udało się pobrać tokenu CSRF.") return False self.logger("RNŚ: Wysyłanie formularza...") payload = {"csrf_neocms": csrf_token, "login": email.value, "password": password.value, "ufd_data": "{}"} try: r2 = self.session.post("https://nowyswiat.online/konto/zaloguj", data=payload, headers={ "X-Requested-With": "XMLHttpRequest" }, timeout=15) except Exception as e: self.logger(f"RNŚ: Błąd sieci przy wysyłaniu formularza: {e}") return False # Check if login succeeded by looking for a session cookie or a success response if '"status":"OK"' in r2.text or "logowanie udane" in r2.text.lower(): # Save cookies to DB (serialize) cookies_dict = requests.utils.dict_from_cookiejar(self.session.cookies) import json cookie_conf = self.db.query(Config).filter_by(key="rns_cookies").first() if not cookie_conf: self.db.add(Config(key="rns_cookies", value=json.dumps(cookies_dict))) else: cookie_conf.value = json.dumps(cookies_dict) self.db.commit() return True else: self.logger(f"RNŚ: Błędne dane logowania (lub zmiana mechanizmu). Odpowiedź: {r2.text[:100]}") return False def ensure_auth(self): cookie_conf = self.db.query(Config).filter_by(key="rns_cookies").first() if cookie_conf: import json try: cookies_dict = json.loads(cookie_conf.value) self.session.cookies = requests.utils.cookiejar_from_dict(cookies_dict) html = self._get_html(f"{BASE_URL}/") if html and "wyloguj" in html.lower(): return True self.logger("RNŚ: Zapisana sesja wygasła, wykonuję ponowne logowanie.") except: pass return self.perform_login() def check_auth_status(self): cookie_conf = self.db.query(Config).filter_by(key="rns_cookies").first() if not cookie_conf: return False, "Brak zapisanych ciasteczek." import json try: self.session.cookies = requests.utils.cookiejar_from_dict(json.loads(cookie_conf.value)) html = self._get_html("https://nowyswiat.online/") if html and "wyloguj" in html.lower(): return True, "Ciasteczka aktywne, sesja poprawna." return False, "Ciasteczka nieaktywne lub wygasły (brak dostępu do profilu)." except Exception as e: return False, f"Błąd: {e}" def update_programs(self): self.logger("RNŚ: Aktualizacja listy programów...") html = self._get_html("https://nowyswiat.online/podcasty") if not html: return soup = BeautifulSoup(html, "html.parser") links = soup.find_all("a", href=True) program_links = [link for link in links if "rbroadcast=" in link["href"]] if not program_links: invalid_response("rns", "HTML_SCHEMA_CHANGED", "Nie znaleziono programów w stronie podcastów", "https://nowyswiat.online/podcasty", content=html) for link in program_links: href = link["href"] if "rbroadcast=" in href: parsed = urllib.parse.urlparse(href) slug = urllib.parse.parse_qs(parsed.query).get("rbroadcast", [None])[0] if not slug: continue title_el = link.find("h2", class_="rns-search-dropdown-title") title = title_el.text.strip() if title_el else slug img_el = link.find("img") img_url = img_el["src"] if img_el and img_el.has_attr("src") else "" if img_url and not img_url.startswith("http"): img_url = f"{BASE_URL}/{img_url.lstrip('/')}" prog = self.db.query(Program).filter_by(station="rns", slug=slug).first() if not prog: self.db.add(Program( station="rns", slug=slug, name=title, image=img_url, description=f"Radio Nowy Świat: {title}", backfill_page=2, backfill_complete=False )) self.db.commit() def _verify_audio_teaser(self, ep: Episode): """Wykrywa 1-minutowy teaser (rozmiar mniejszy niż ~2MB, mimo że audycja trwa > 5 min).""" self.check_timeout() if not ep.url: return False if self.requests_made >= self.hard_limit: return False self.requests_made += 1 if self.progress_callback: self.progress_callback(self.requests_made, self.hard_limit) try: r = self.session.head(ep.url, timeout=5) cl = int(r.headers.get("Content-Length", 0)) if cl > 0 and cl < 2_500_000 and ep.duration_secs > 300: self.logger(f"RNŚ: Wykryto uszkodzony link (Teaser 1-min) dla {ep.title}. Usuwam URL.") ep.url = None ep.is_broken = True return True except Exception as e: pass return False def fetch_program_page(self, program_slug, page): url = f"https://nowyswiat.online/podcasty?rbroadcast={program_slug}&page={page}" html = self._get_html(url) if not html: raise Exception(f"Błąd sieci podczas pobierania strony {page} audycji {program_slug}") soup = BeautifulSoup(html, "html.parser") cards = soup.find_all("a", class_="rns-grid-podcast-card") known_episode_count = self.db.query(Episode).filter_by( station="rns", program_slug=program_slug ).count() page_has_podcast_content = bool(soup.find(string=re.compile(r"podcast|podkast", re.IGNORECASE))) if not cards and page > 1: return 0, 0, page - 1 if not cards and (known_episode_count or not page_has_podcast_content): invalid_response("rns", "HTML_SCHEMA_CHANGED", "Nie znaleziono kart odcinków dla istniejącego programu", url, content=html) new_found = 0 missing_player_count = 0 valid_cards = 0 for card in cards: href = card.get("href", "") if not href or "/podcasty/" not in href: continue valid_cards += 1 ep_id = href.split("/podcasty/")[-1].split("?")[0] player_box = card.find("div", class_="rns-play-btn-box") audio_url = player_box.get("data-neo-player-src") if player_box else None if not player_box: missing_player_count += 1 if audio_url and not audio_url.startswith("http"): audio_url = f"{BASE_URL}/{audio_url.lstrip('/')}" title_el = card.find("p", class_="rns-post-title") raw_title = player_box.get("data-neo-player-title", "") if player_box else (title_el.text.strip() if title_el else ep_id) title = BeautifulSoup(raw_title, "html.parser").text.strip() if raw_title else ep_id raw_subtitle = player_box.get("data-neo-player-subtitle", "") if player_box else "" authors = BeautifulSoup(raw_subtitle, "html.parser").text.strip() if raw_subtitle else "Radio Nowy Świat" date_el = card.find("p", class_="rns-podcast-details-date") pub_date = parse_polish_date(date_el.text.strip()) if date_el else "" time_el = card.find("p", class_="rns-podcast-details-long") duration_str = time_el.text.strip() if time_el else "0:00" duration_secs = 0 if ":" in duration_str: parts = duration_str.split(":") if len(parts) == 3: duration_secs = int(parts[0])*3600 + int(parts[1])*60 + int(parts[2]) elif len(parts) == 2: duration_secs = int(parts[0])*60 + int(parts[1]) img_url = player_box.get("data-neo-player-img", "") if player_box else "" if img_url and not img_url.startswith("http"): img_url = f"{BASE_URL}/{img_url.lstrip('/')}" desc_el = card.find("p", class_="rns-post-card-desc") description = desc_el.get_text(separator="\n").strip() if desc_el else "" ep = self.db.query(Episode).filter_by(station="rns", ep_id=ep_id).first() if not ep: ep = Episode( station="rns", program_slug=program_slug, ep_id=ep_id, title=title, authors=authors, url=audio_url, image=img_url, pub_date=pub_date, duration_secs=duration_secs, description=description, is_broken=False ) self.db.add(ep) new_found += 1 else: if not ep.url and audio_url: ep.url = audio_url ep.is_broken = False # Celowo nie zwiększamy new_found, aby Daily Catchup nie wchodził w tryb głębokiego archiwum. # Łataniem starych dziur zajmie się faza Backfill. if cards and not valid_cards: invalid_response("rns", "HTML_SCHEMA_CHANGED", "Znaleziono karty podcastów bez oczekiwanych identyfikatorów", url, content=html) if missing_player_count: warning = f"{missing_player_count} odcinków bez dostępnego playera audio; pomijam ich URL-e." if self.warning_callback: self.warning_callback(warning) else: self.logger(f"RNŚ: WARNING: {warning}") self.db.commit() page_links = soup.find_all("a", href=re.compile(r"page=\d+")) max_page = page for link in page_links: href = link.get("href") m = re.search(r"page=(\d+)", href) if m: max_page = max(max_page, int(m.group(1))) return new_found, len(cards), max_page def _compute_next_catchup(self, prog: Program) -> float: """Wylicza kiedy najwcześniej warto znowu sprawdzać tę audycję.""" episodes = self.db.query(Episode).filter_by( station="rns", program_slug=prog.slug ).order_by(Episode.pub_date.desc()).limit(20).all() if len(episodes) < 3: # Za mało danych – sprawdzamy przy każdym uruchomieniu return 0.0 # Wylicz średnią przerwę między odcinkami w dniach import re dates = [] for ep in episodes: if ep.pub_date and re.match(r'\d{4}-\d{2}-\d{2}', ep.pub_date): try: import datetime dates.append(datetime.date.fromisoformat(ep.pub_date[:10])) except ValueError: pass if len(dates) < 3: return 0.0 dates.sort(reverse=True) gaps = [(dates[i] - dates[i+1]).days for i in range(len(dates)-1)] avg_gap = sum(gaps) / len(gaps) if avg_gap < 2: # Codziennie lub częściej → zawsze sprawdzamy (brak cooldownu) return 0.0 elif avg_gap <= 8: # Tygodniowo → cooldown = 70% cyklu (żeby sprawdzić przed następnym odcinkiem) cooldown_days = avg_gap * 0.7 else: # Rzadziej niż tygodniowo → max 7 dni cooldown_days = 7.0 return time.time() + cooldown_days * 86400 def phase_1_catchup(self, specific_program_slug=None): self.logger("RNŚ: Phase 1 (Daily Catchup)...") query = self.db.query(Program).filter_by(station="rns") if specific_program_slug: query = query.filter_by(slug=specific_program_slug) else: query = query.order_by(Program.last_catchup.asc()) catchup_limit = max(1, self.hard_limit - self.backfill_limit) skipped = 0 timeout_program = None for prog in query.all(): if self.requests_made >= catchup_limit: self.logger(f"RNŚ: [{self.requests_made}/{self.hard_limit}] Limit Catchup osiągnięty. Zostawiam resztę dla Backfill.") break # Pomiń jeśli za wcześnie (adaptive cooldown) if not specific_program_slug and prog.next_catchup_after and time.time() < prog.next_catchup_after: skipped += 1 continue is_first_sync = (prog.last_catchup == 0.0) page = 1 catchup_succeeded = False while True: if self.requests_made >= catchup_limit: break self.logger(f"RNŚ: [{self.requests_made}/{self.hard_limit}] Sprawdzam {prog.slug} (strona {page})") try: new_found, total_cards, max_page = self.fetch_program_page(prog.slug, page) except ScraperError as exc: if exc.code != "NETWORK_TIMEOUT": raise if timeout_program: raise ScraperError( "rns", "NETWORK_TIMEOUT", f"Timeout także dla kolejnej audycji {prog.slug}; poprzednia: {timeout_program}", exc.url, ) from exc timeout_program = prog.slug self.logger(f"RNŚ: Pomijam {prog.slug} po trzech timeoutach i sprawdzam następną audycję.") break if timeout_program: warning = f"Poprzednia audycja {timeout_program} miała trzy timeouty; kolejna audycja odpowiada poprawnie." if self.warning_callback: self.warning_callback(warning) else: self.logger(f"RNŚ: WARNING: {warning}") timeout_program = None if max_page > prog.total_pages: prog.total_pages = max_page if total_cards == 0 or new_found == 0: catchup_succeeded = True break if is_first_sync: catchup_succeeded = True break if page >= max_page: catchup_succeeded = True break page += 1 time.sleep(0.5) if catchup_succeeded: prog.last_catchup = time.time() prog.next_catchup_after = self._compute_next_catchup(prog) self.db.commit() if skipped: self.logger(f"RNŚ: Pominięto {skipped} audycji (cooldown adaptacyjny).") def phase_2_backfill(self, specific_program_slug=None): backfill_count = 0 if specific_program_slug: prog = self.db.query(Program).filter_by(station="rns", slug=specific_program_slug).first() if prog: page = prog.backfill_page self.logger(f"RNŚ: [{self.requests_made}/{self.hard_limit}] Backfill manualny dla {prog.slug} (od strony {page})") while self.requests_made < self.hard_limit and backfill_count < self.backfill_limit: new_found, total, max_p = self.fetch_program_page(prog.slug, page) backfill_count += 1 if max_p > prog.total_pages: prog.total_pages = max_p self.logger(f"RNŚ: [{self.requests_made}/{self.hard_limit}] Backfill {prog.slug} strona {page} z {prog.total_pages or '?'} ({new_found} nowych)") if total == 0 or page >= max_p: prog.backfill_complete = True self.logger(f"RNŚ: Backfill {prog.slug} – zakończono.") break prog.backfill_page = page + 1 page += 1 self.db.commit() time.sleep(0.5) return while self.requests_made < self.hard_limit and backfill_count < self.backfill_limit: prog = self.db.query(Program).filter_by(station="rns", backfill_complete=False).order_by(Program.backfill_page.asc()).first() if not prog: self.logger("RNŚ: Wszystkie audycje w pełni uzupełnione!") break page = prog.backfill_page self.logger(f"RNŚ: [{self.requests_made}/{self.hard_limit}] Backfill dla {prog.slug} (strona {page} z {prog.total_pages or '?'})") new_found, total_cards, max_p = self.fetch_program_page(prog.slug, page) backfill_count += 1 if max_p > prog.total_pages: prog.total_pages = max_p if total_cards == 0 or page >= max_p: prog.backfill_complete = True else: prog.backfill_page = page + 1 self.db.commit() time.sleep(0.5) def run_full_sync(self, specific_program_slug=None): if not self.ensure_auth(): self.logger("RNŚ: Błąd logowania przed startem!") raise Exception("RNS Login Failed") self.update_programs() self.phase_1_catchup(specific_program_slug) if self.requests_made < self.hard_limit: self.phase_2_backfill(specific_program_slug) self.logger(f"RNŚ: Koniec. Wysłano zapytania: {self.requests_made}/{self.hard_limit} (w tym backfill ograniczony do {self.backfill_limit}).")