chore: import upstream snapshot with attribution

2026-07-13 12:12:13 +08:00
commit 0446c45d8e
898 changed files with 328024 additions and 0 deletions
@@ -0,0 +1,130 @@
+from typing import Any, List, Optional, Dict
+from enum import Enum
+from pydantic import BaseModel, Field, HttpUrl, field_validator
+from utils import FilterType
+
+
+class CrawlRequest(BaseModel):
+    urls: List[str] = Field(min_length=1, max_length=100)
+    browser_config: Optional[Dict] = Field(default_factory=dict)
+    crawler_config: Optional[Dict] = Field(default_factory=dict)
+    crawler_configs: Optional[List[Dict]] = Field(
+        default=None,
+        description=(
+            "List of per-URL CrawlerRunConfig dicts for arun_many(). "
+            "When provided, each config can include a 'url_matcher' pattern "
+            "to match against specific URLs. Takes precedence over crawler_config."
+        ),
+    )
+
+
+class HookSpec(BaseModel):
+    """A single declarative hook: a fixed action plus schema-validated params.
+
+    Arbitrary Python (the old `code` map) is no longer accepted - it was an
+    exec()-based RCE surface. Available actions are enumerated by GET /hooks/info
+    and validated server-side by hook_registry.py.
+    """
+    action: str = Field(..., description="One of the registered hook actions")
+    params: Dict[str, Any] = Field(default_factory=dict, description="Action parameters")
+
+
+class HookConfig(BaseModel):
+    """Configuration for declarative hooks."""
+    hooks: List[HookSpec] = Field(
+        default_factory=list,
+        max_length=10,
+        description="Declarative hook specs (action + params), max 10",
+    )
+    timeout: int = Field(
+        default=30,
+        ge=1,
+        le=120,
+        description="Timeout in seconds for each hook execution",
+    )
+
+    class Config:
+        json_schema_extra = {
+            "example": {
+                "hooks": [
+                    {"action": "block_resources", "params": {"resource_types": ["image", "font"]}},
+                    {"action": "scroll_to_bottom", "params": {"max_steps": 10, "delay_ms": 500}},
+                ],
+                "timeout": 30,
+            }
+        }
+
+
+class CrawlRequestWithHooks(CrawlRequest):
+    """Extended crawl request with hooks support"""
+    hooks: Optional[HookConfig] = Field(
+        default=None,
+        description="Optional user-provided hook functions"
+    )
+
+class MarkdownRequest(BaseModel):
+    """Request body for the /md endpoint."""
+    url: str                    = Field(...,  description="Absolute http/https URL to fetch")
+    f:   FilterType             = Field(FilterType.FIT, description="Content‑filter strategy: fit, raw, bm25, or llm")
+    q:   Optional[str] = Field(None,  description="Query string used by BM25/LLM filters")
+    c:   Optional[str] = Field("0",   description="Cache‑bust / revision counter")
+    provider: Optional[str] = Field(None, description="LLM provider override (e.g., 'anthropic/claude-3-opus')")
+    temperature: Optional[float] = Field(None, description="LLM temperature override (0.0-2.0)")
+    # base_url removed: a request-supplied LLM endpoint was a credential-exfil
+    # vector. The endpoint is derived server-side from the provider name.
+
+
+class RawCode(BaseModel):
+    code: str
+
+class HTMLRequest(BaseModel):
+    url: str
+    
+class ScreenshotRequest(BaseModel):
+    url: str
+    screenshot_wait_for: Optional[float] = 2
+    wait_for_images: Optional[bool] = False
+    # output_path removed: callers never name a filesystem path (it was an
+    # arbitrary-write -> RCE vector). The server writes to the sandboxed
+    # artifact store and returns an opaque artifact_id.
+
+
+class PDFRequest(BaseModel):
+    url: str
+    # output_path removed (see ScreenshotRequest).
+
+
+class JSEndpointRequest(BaseModel):
+    url: str
+    scripts: List[str] = Field(
+        ...,
+        description="List of separated JavaScript snippets to execute"
+    )
+
+
+class WebhookConfig(BaseModel):
+    """Configuration for webhook notifications."""
+    webhook_url: HttpUrl
+    webhook_data_in_payload: bool = False
+    webhook_headers: Optional[Dict[str, str]] = None
+
+    @field_validator("webhook_headers")
+    @classmethod
+    def _validate_headers(cls, v):
+        # Reject unsafe outbound headers early (422). Mirrors
+        # webhook.sanitize_webhook_headers; kept inline to avoid an import cycle.
+        if not v:
+            return v
+        from webhook import sanitize_webhook_headers
+        return sanitize_webhook_headers(v)
+
+
+class WebhookPayload(BaseModel):
+    """Payload sent to webhook endpoints."""
+    task_id: str
+    task_type: str  # "crawl", "llm_extraction", etc.
+    status: str  # "completed" or "failed"
+    timestamp: str  # ISO 8601 format
+    urls: List[str]
+    error: Optional[str] = None
+    data: Optional[Dict] = None  # Included only if webhook_data_in_payload=True