wehub-resource-sync
43 lines
1.1 KiB
Python
43 lines
1.1 KiB
Python
"""Scrapling + CloakBrowser: adaptive web scraping with stealth fingerprints.
|
|
|
|
Scrapling handles parsing and element tracking,
|
|
CloakBrowser handles bot detection.
|
|
|
|
Requires: pip install scrapling[all] cloakbrowser
|
|
"""
|
|
|
|
import asyncio
|
|
import json
|
|
from urllib.request import urlopen
|
|
|
|
from scrapling.fetchers import StealthyFetcher
|
|
|
|
from cloakbrowser import launch_async
|
|
|
|
|
|
async def main():
|
|
# Launch CloakBrowser with remote debugging
|
|
cb_browser = await launch_async(
|
|
headless=True,
|
|
args=["--remote-debugging-port=9245", "--remote-debugging-address=127.0.0.1"],
|
|
)
|
|
|
|
# Get the WebSocket URL from Chrome (Scrapling requires ws:// scheme)
|
|
info = json.loads(urlopen("http://127.0.0.1:9245/json/version").read())
|
|
ws_url = info["webSocketDebuggerUrl"]
|
|
|
|
# Connect Scrapling to the stealth browser via CDP
|
|
page = await StealthyFetcher.async_fetch(
|
|
"https://example.com",
|
|
cdp_url=ws_url,
|
|
)
|
|
|
|
print(f"Title: {page.css('title::text').get()}")
|
|
print(f"Text: {page.css('p::text').getall()}")
|
|
|
|
await cb_browser.close()
|
|
|
|
|
|
if __name__ == "__main__":
|
|
asyncio.run(main())
|