48 lines
1.9 KiB
Python
48 lines
1.9 KiB
Python
import json
|
|
import os
|
|
import re
|
|
import hashlib
|
|
from datetime import datetime
|
|
|
|
|
|
class ScraperError(Exception):
|
|
def __init__(self, station, code, message, url=None, status_code=None):
|
|
self.station = station
|
|
self.code = code
|
|
self.url = url
|
|
self.status_code = status_code
|
|
details = f"[{code}] {message}"
|
|
if status_code is not None:
|
|
details += f" (HTTP {status_code})"
|
|
if url:
|
|
details += f" URL: {url}"
|
|
super().__init__(details)
|
|
|
|
|
|
def save_response_snapshot(station, code, url, content, status_code=None):
|
|
directory = os.path.join("data", "diagnostics")
|
|
try:
|
|
os.makedirs(directory, exist_ok=True)
|
|
digest = hashlib.sha256(f"{station}:{code}:{url}".encode("utf-8")).hexdigest()[:16]
|
|
safe_station = re.sub(r"[^a-zA-Z0-9_-]", "_", station)
|
|
base_path = os.path.join(directory, f"{safe_station}-{code}-{digest}")
|
|
|
|
with open(f"{base_path}.json", "w", encoding="utf-8") as metadata_file:
|
|
json.dump({"station": station, "code": code, "url": url, "status_code": status_code,
|
|
"updated_at": datetime.now().isoformat()}, metadata_file, ensure_ascii=False, indent=2)
|
|
with open(f"{base_path}.html", "w", encoding="utf-8") as response_file:
|
|
response_file.write((content or "")[:2_000_000])
|
|
return base_path
|
|
except OSError as exc:
|
|
return f"snapshot unavailable: {exc}"
|
|
|
|
|
|
def invalid_response(station, code, message, url, response=None, content=None):
|
|
snapshot = save_response_snapshot(
|
|
station,
|
|
url,
|
|
response.text if response is not None else content,
|
|
response.status_code if response is not None else None,
|
|
)
|
|
raise ScraperError(station, code, f"{message}. Snapshot: {snapshot}", url,
|
|
response.status_code if response is not None else None) |