import json import os import re import hashlib from datetime import datetime class ScraperError(Exception): def __init__(self, station, code, message, url=None, status_code=None): self.station = station self.code = code self.url = url self.status_code = status_code details = f"[{code}] {message}" if status_code is not None: details += f" (HTTP {status_code})" if url: details += f" URL: {url}" super().__init__(details) def save_response_snapshot(station, code, url, content, status_code=None): directory = os.path.join("data", "diagnostics") try: os.makedirs(directory, exist_ok=True) digest = hashlib.sha256(f"{station}:{code}:{url}".encode("utf-8")).hexdigest()[:16] safe_station = re.sub(r"[^a-zA-Z0-9_-]", "_", station) base_path = os.path.join(directory, f"{safe_station}-{code}-{digest}") with open(f"{base_path}.json", "w", encoding="utf-8") as metadata_file: json.dump({"station": station, "code": code, "url": url, "status_code": status_code, "updated_at": datetime.now().isoformat()}, metadata_file, ensure_ascii=False, indent=2) with open(f"{base_path}.html", "w", encoding="utf-8") as response_file: response_file.write((content or "")[:2_000_000]) return base_path except OSError as exc: return f"snapshot unavailable: {exc}" def invalid_response(station, code, message, url, response=None, content=None): snapshot = save_response_snapshot( station, url, response.text if response is not None else content, response.status_code if response is not None else None, ) raise ScraperError(station, code, f"{message}. Snapshot: {snapshot}", url, response.status_code if response is not None else None)