wehub-resource-sync
247153575d
Tests / tests (map[TOXENV:py310], macos-latest, 3.10) (push) Has been cancelled
Tests / tests (map[TOXENV:py311], macos-latest, 3.11) (push) Has been cancelled
Tests / tests (map[TOXENV:py312], macos-latest, 3.12) (push) Has been cancelled
Tests / tests (map[TOXENV:py313], macos-latest, 3.13) (push) Has been cancelled
80 lines
2.9 KiB
Python
80 lines
2.9 KiB
Python
from base64 import b64encode, b64decode
|
|
from pathlib import Path
|
|
|
|
import orjson
|
|
import anyio
|
|
from anyio import Path as AsyncPath
|
|
|
|
from scrapling.core.utils import log
|
|
from scrapling.core._types import Dict, Optional, Any
|
|
from scrapling.engines.toolbelt.custom import Response
|
|
|
|
|
|
class ResponseCacheManager:
|
|
"""Caches HTTP responses to disk for replay during spider development."""
|
|
|
|
def __init__(self, cache_dir: str | Path):
|
|
self._cache_dir = AsyncPath(cache_dir)
|
|
|
|
def _cache_path(self, fingerprint: bytes) -> AsyncPath:
|
|
return self._cache_dir / f"{fingerprint.hex()}.json"
|
|
|
|
async def get(self, fingerprint: bytes) -> Optional[Response]:
|
|
path = self._cache_path(fingerprint)
|
|
if not await path.exists():
|
|
return None
|
|
|
|
try:
|
|
async with await anyio.open_file(path, "rb") as f:
|
|
data: Dict[str, Any] = orjson.loads(await f.read())
|
|
|
|
return Response(
|
|
url=data["url"],
|
|
content=b64decode(data["content"]),
|
|
status=data["status"],
|
|
reason=data["reason"],
|
|
encoding=data["encoding"],
|
|
cookies=data["cookies"],
|
|
headers=data["headers"],
|
|
request_headers=data["request_headers"],
|
|
method=data["method"],
|
|
)
|
|
except Exception as e:
|
|
log.warning(f"Failed to read cached response for {fingerprint.hex()}: {e}")
|
|
return None
|
|
|
|
async def put(self, fingerprint: bytes, response: Response, method: str = "GET") -> None:
|
|
await self._cache_dir.mkdir(parents=True, exist_ok=True)
|
|
temp_path = self._cache_path(fingerprint).with_suffix(".tmp")
|
|
|
|
try:
|
|
serialized = orjson.dumps(
|
|
{
|
|
"url": response.url,
|
|
"content": b64encode(response.body).decode("ascii"),
|
|
"status": response.status,
|
|
"reason": response.reason,
|
|
"encoding": response.encoding,
|
|
"cookies": dict(response.cookies) if isinstance(response.cookies, dict) else {},
|
|
"headers": dict(response.headers),
|
|
"request_headers": dict(response.request_headers),
|
|
"method": method,
|
|
}
|
|
)
|
|
async with await anyio.open_file(temp_path, "wb") as f:
|
|
await f.write(serialized)
|
|
|
|
await temp_path.replace(self._cache_path(fingerprint))
|
|
except Exception as e:
|
|
if await temp_path.exists():
|
|
await temp_path.unlink()
|
|
log.warning(f"Failed to cache response for {fingerprint.hex()}: {e}")
|
|
|
|
async def clear(self) -> None:
|
|
if not await self._cache_dir.exists():
|
|
return
|
|
async for entry in self._cache_dir.iterdir():
|
|
if entry.suffix == ".json":
|
|
await entry.unlink()
|
|
log.info(f"Cleared response cache at {self._cache_dir}")
|