wehub-resource-sync
247153575d
Tests / tests (map[TOXENV:py310], macos-latest, 3.10) (push) Has been cancelled
Tests / tests (map[TOXENV:py311], macos-latest, 3.11) (push) Has been cancelled
Tests / tests (map[TOXENV:py312], macos-latest, 3.12) (push) Has been cancelled
Tests / tests (map[TOXENV:py313], macos-latest, 3.13) (push) Has been cancelled
119 lines
3.5 KiB
Python
119 lines
3.5 KiB
Python
from scrapling.core._types import (
|
|
Any,
|
|
Dict,
|
|
List,
|
|
Set,
|
|
Tuple,
|
|
Sequence,
|
|
Callable,
|
|
Optional,
|
|
SetCookieParam,
|
|
SelectorWaitStates,
|
|
FollowRedirects,
|
|
)
|
|
|
|
# Parameter definitions for shell function signatures (defined once at module level)
|
|
# Mirrors TypedDict definitions from _types.py but runtime-accessible for IPython introspection
|
|
_REQUESTS_PARAMS = {
|
|
"params": Optional[Dict | List | Tuple],
|
|
"cookies": Any,
|
|
"auth": Optional[Tuple[str, str]],
|
|
"impersonate": Any,
|
|
"http3": Optional[bool],
|
|
"stealthy_headers": Optional[bool],
|
|
"proxies": Any,
|
|
"proxy": Optional[str],
|
|
"proxy_auth": Optional[Tuple[str, str]],
|
|
"timeout": Optional[int | float],
|
|
"headers": Any,
|
|
"retries": Optional[int],
|
|
"retry_delay": Optional[int],
|
|
"follow_redirects": Optional[FollowRedirects],
|
|
"max_redirects": Optional[int],
|
|
"verify": Optional[bool],
|
|
"cert": Optional[str | Tuple[str, str]],
|
|
"selector_config": Optional[Dict],
|
|
}
|
|
|
|
_FETCH_PARAMS = {
|
|
"headless": bool,
|
|
"disable_resources": bool,
|
|
"network_idle": bool,
|
|
"load_dom": bool,
|
|
"wait_selector": Optional[str],
|
|
"wait_selector_state": SelectorWaitStates,
|
|
"cookies": Sequence[SetCookieParam],
|
|
"google_search": bool,
|
|
"wait": int | float,
|
|
"timezone_id": str | None,
|
|
"page_action": Optional[Callable],
|
|
"page_setup": Optional[Callable],
|
|
"proxy": Optional[str | Dict[str, str] | Tuple],
|
|
"extra_headers": Optional[Dict[str, str]],
|
|
"timeout": int | float,
|
|
"init_script": Optional[str],
|
|
"user_data_dir": str,
|
|
"selector_config": Optional[Dict],
|
|
"additional_args": Optional[Dict],
|
|
"locale": Optional[str],
|
|
"real_chrome": bool,
|
|
"cdp_url": Optional[str],
|
|
"useragent": Optional[str],
|
|
"extra_flags": Optional[List[str]],
|
|
"blocked_domains": Optional[Set[str]],
|
|
"block_ads": bool,
|
|
"retries": int,
|
|
"retry_delay": int | float,
|
|
"capture_xhr": str | None,
|
|
"executable_path": Optional[str],
|
|
"dns_over_https": bool,
|
|
}
|
|
|
|
_STEALTHY_FETCH_PARAMS = {
|
|
"headless": bool,
|
|
"disable_resources": bool,
|
|
"network_idle": bool,
|
|
"load_dom": bool,
|
|
"wait_selector": Optional[str],
|
|
"wait_selector_state": SelectorWaitStates,
|
|
"cookies": Sequence[SetCookieParam],
|
|
"google_search": bool,
|
|
"wait": int | float,
|
|
"timezone_id": str | None,
|
|
"page_action": Optional[Callable],
|
|
"page_setup": Optional[Callable],
|
|
"proxy": Optional[str | Dict[str, str] | Tuple],
|
|
"extra_headers": Optional[Dict[str, str]],
|
|
"timeout": int | float,
|
|
"init_script": Optional[str],
|
|
"user_data_dir": str,
|
|
"selector_config": Optional[Dict],
|
|
"additional_args": Optional[Dict],
|
|
"locale": Optional[str],
|
|
"real_chrome": bool,
|
|
"cdp_url": Optional[str],
|
|
"useragent": Optional[str],
|
|
"extra_flags": Optional[List[str]],
|
|
"blocked_domains": Optional[Set[str]],
|
|
"block_ads": bool,
|
|
"retries": int,
|
|
"retry_delay": int | float,
|
|
"capture_xhr": str | None,
|
|
"executable_path": Optional[str],
|
|
"dns_over_https": bool,
|
|
"allow_webgl": bool,
|
|
"hide_canvas": bool,
|
|
"block_webrtc": bool,
|
|
"solve_cloudflare": bool,
|
|
}
|
|
|
|
# Mapping of function names to their parameter definitions
|
|
Signatures_map = {
|
|
"get": _REQUESTS_PARAMS,
|
|
"post": {**_REQUESTS_PARAMS, "data": Optional[Dict | str], "json": Optional[Dict | List]},
|
|
"put": {**_REQUESTS_PARAMS, "data": Optional[Dict | str], "json": Optional[Dict | List]},
|
|
"delete": _REQUESTS_PARAMS,
|
|
"fetch": _FETCH_PARAMS,
|
|
"stealthy_fetch": _STEALTHY_FETCH_PARAMS,
|
|
}
|