chore: import upstream snapshot with attribution
This commit is contained in:
wehub-resource-sync
@@ -0,0 +1,12 @@
|
||||
[env]
|
||||
RUST_TEST_THREADS = "1"
|
||||
|
||||
# macOS 26 (Tahoe) moved libc++ headers inside the SDK; Apple clang 17 no longer
|
||||
# finds them at the old search path. This causes boring-sys's cmake step to fail
|
||||
# when building with --features stealth. Setting CXXFLAGS and SDKROOT here makes
|
||||
# them visible to all build scripts (including boring-sys's) so cmake can locate
|
||||
# the headers. force=false lets CI or developers override via their shell env.
|
||||
# Non-macOS platforms ignore SDKROOT; the -isystem path is silently skipped if
|
||||
# it doesn't exist. See: https://github.com/h4ckf0r0day/obscura/issues/136
|
||||
CXXFLAGS = { value = "-isystem /Library/Developer/CommandLineTools/SDKs/MacOSX.sdk/usr/include/c++/v1", force = false }
|
||||
SDKROOT = { value = "/Library/Developer/CommandLineTools/SDKs/MacOSX.sdk", force = false }
|
||||
@@ -0,0 +1,2 @@
|
||||
# Auto detect text files and perform LF normalization
|
||||
* text=auto
|
||||
@@ -0,0 +1,5 @@
|
||||
root: ./docs/
|
||||
|
||||
structure:
|
||||
readme: README.md
|
||||
summary: SUMMARY.md
|
||||
@@ -0,0 +1,94 @@
|
||||
name: Bug report
|
||||
description: Report a problem with obscura
|
||||
labels: ["bug"]
|
||||
body:
|
||||
- type: markdown
|
||||
attributes:
|
||||
value: |
|
||||
Thanks for filing a bug. Please fill in the fields below so we can reproduce it quickly.
|
||||
- type: checkboxes
|
||||
id: preflight
|
||||
attributes:
|
||||
label: Before you start
|
||||
options:
|
||||
- label: I searched existing issues and this is not a duplicate.
|
||||
required: true
|
||||
- label: I tested against the latest release or `main`.
|
||||
required: true
|
||||
- label: This is not a security vulnerability (those go through SECURITY.md, not a public issue).
|
||||
required: true
|
||||
- type: input
|
||||
id: version
|
||||
attributes:
|
||||
label: obscura version or commit
|
||||
placeholder: "e.g. v0.1.3 or commit a8358cc"
|
||||
validations:
|
||||
required: true
|
||||
- type: input
|
||||
id: platform
|
||||
attributes:
|
||||
label: OS and architecture
|
||||
placeholder: "e.g. Ubuntu 22.04 x86_64, macOS 15 arm64, Windows 11 x64"
|
||||
validations:
|
||||
required: true
|
||||
- type: dropdown
|
||||
id: surface
|
||||
attributes:
|
||||
label: How are you using obscura?
|
||||
options:
|
||||
- CLI (fetch / serve / scrape / mcp)
|
||||
- CDP server with Puppeteer or Playwright
|
||||
- Embedded Rust library (obscura crate)
|
||||
- Other
|
||||
validations:
|
||||
required: true
|
||||
- type: dropdown
|
||||
id: stealth
|
||||
attributes:
|
||||
label: Stealth mode
|
||||
options:
|
||||
- "Off (default build)"
|
||||
- "On (--stealth, default build)"
|
||||
- "On (--stealth, --features stealth build)"
|
||||
validations:
|
||||
required: false
|
||||
- type: textarea
|
||||
id: repro
|
||||
attributes:
|
||||
label: Reproduction
|
||||
description: A URL, an `--eval` snippet, or a short CDP sequence. Multi-statement `--eval` that starts with `const` returns null, so wrap it in an IIFE.
|
||||
placeholder: |
|
||||
obscura fetch https://example.com --eval "(function(){ return document.title; })()"
|
||||
validations:
|
||||
required: true
|
||||
- type: textarea
|
||||
id: expected
|
||||
attributes:
|
||||
label: Expected behavior
|
||||
validations:
|
||||
required: true
|
||||
- type: textarea
|
||||
id: actual
|
||||
attributes:
|
||||
label: Actual behavior
|
||||
validations:
|
||||
required: true
|
||||
- type: dropdown
|
||||
id: chrome
|
||||
attributes:
|
||||
label: Does headless Chrome behave the same?
|
||||
description: Anti-bot, CAPTCHA, and login walls block headless Chrome too from a datacenter IP, so those are not engine bugs.
|
||||
options:
|
||||
- "No: headless Chrome renders or behaves correctly and obscura does not"
|
||||
- "Yes: headless Chrome has the same problem"
|
||||
- "I did not check"
|
||||
validations:
|
||||
required: true
|
||||
- type: textarea
|
||||
id: logs
|
||||
attributes:
|
||||
label: Logs
|
||||
description: Output with `--verbose` if relevant.
|
||||
render: shell
|
||||
validations:
|
||||
required: false
|
||||
@@ -0,0 +1,8 @@
|
||||
blank_issues_enabled: false
|
||||
contact_links:
|
||||
- name: Questions and API proposals
|
||||
url: https://github.com/h4ckf0r0day/obscura/discussions
|
||||
about: Ask questions or propose larger API designs in Discussions.
|
||||
- name: Report a security vulnerability
|
||||
url: https://github.com/h4ckf0r0day/obscura/security/advisories/new
|
||||
about: Report security issues privately, not as a public issue. See SECURITY.md.
|
||||
@@ -0,0 +1,37 @@
|
||||
name: Feature request
|
||||
description: Suggest a capability or API for obscura
|
||||
labels: ["enhancement"]
|
||||
body:
|
||||
- type: markdown
|
||||
attributes:
|
||||
value: |
|
||||
For larger API proposals, consider opening a Discussion first so the design can be talked through before you build it.
|
||||
- type: textarea
|
||||
id: problem
|
||||
attributes:
|
||||
label: Problem
|
||||
description: What are you trying to do that obscura does not support today?
|
||||
validations:
|
||||
required: true
|
||||
- type: textarea
|
||||
id: solution
|
||||
attributes:
|
||||
label: Proposed solution
|
||||
validations:
|
||||
required: true
|
||||
- type: textarea
|
||||
id: alternatives
|
||||
attributes:
|
||||
label: Alternatives considered
|
||||
validations:
|
||||
required: false
|
||||
- type: checkboxes
|
||||
id: scope
|
||||
attributes:
|
||||
label: Scope
|
||||
description: Obscura targets web scraping and AI-agent automation, with performance and stability as hard constraints.
|
||||
options:
|
||||
- label: This should not regress performance (obscura's speed and memory footprint are a hard constraint).
|
||||
required: false
|
||||
- label: This is not detection-evasion for abusive purposes.
|
||||
required: false
|
||||
@@ -0,0 +1,45 @@
|
||||
name: Docker
|
||||
|
||||
on:
|
||||
push:
|
||||
tags:
|
||||
- 'v*'
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
build-and-push:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- name: Set up QEMU
|
||||
uses: docker/setup-qemu-action@v3
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@v3
|
||||
|
||||
- name: Login to Docker Hub
|
||||
uses: docker/login-action@v3
|
||||
with:
|
||||
username: ${{ secrets.DOCKERHUB_USERNAME }}
|
||||
password: ${{ secrets.DOCKERHUB_TOKEN }}
|
||||
|
||||
- name: Extract version from tag
|
||||
id: version
|
||||
run: echo "version=${GITHUB_REF#refs/tags/v}" >> $GITHUB_OUTPUT
|
||||
|
||||
- name: Build and push
|
||||
uses: docker/build-push-action@v6
|
||||
with:
|
||||
context: .
|
||||
push: true
|
||||
build-args: |
|
||||
OBSCURA_VERSION=${{ steps.version.outputs.version }}
|
||||
tags: |
|
||||
${{ secrets.DOCKERHUB_USERNAME }}/obscura:${{ steps.version.outputs.version }}
|
||||
${{ secrets.DOCKERHUB_USERNAME }}/obscura:latest
|
||||
platforms: linux/amd64,linux/arm64
|
||||
cache-from: type=gha
|
||||
cache-to: type=gha,mode=max
|
||||
@@ -0,0 +1,90 @@
|
||||
name: Release
|
||||
|
||||
on:
|
||||
push:
|
||||
tags:
|
||||
- 'v*'
|
||||
|
||||
permissions:
|
||||
contents: write
|
||||
|
||||
jobs:
|
||||
build:
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
include:
|
||||
- target: x86_64-unknown-linux-gnu
|
||||
# Build on Ubuntu 22.04 so the Linux release binary works on
|
||||
# common LTS servers with glibc 2.35 instead of requiring the
|
||||
# newer glibc shipped by ubuntu-latest.
|
||||
os: ubuntu-22.04
|
||||
name: obscura-x86_64-linux
|
||||
- target: aarch64-unknown-linux-gnu
|
||||
# Native build on GitHub's ARM runner.
|
||||
os: ubuntu-22.04-arm
|
||||
name: obscura-aarch64-linux
|
||||
- target: aarch64-apple-darwin
|
||||
os: macos-latest
|
||||
name: obscura-aarch64-macos
|
||||
- target: x86_64-apple-darwin
|
||||
# Must run on a native Intel runner. obscura-js bakes a V8 startup
|
||||
# snapshot at build time, and that snapshot is architecture specific.
|
||||
# macos-latest is Apple Silicon, so building x86_64 there embeds an
|
||||
# arm64 snapshot into the Intel binary and V8 crashes deserializing
|
||||
# it at isolate init (issue #290). macos-15-intel is the last x86_64
|
||||
# image (available until Aug 2027); there is no cross-build that
|
||||
# produces a correct snapshot.
|
||||
os: macos-15-intel
|
||||
name: obscura-x86_64-macos
|
||||
- target: x86_64-pc-windows-msvc
|
||||
os: windows-latest
|
||||
name: obscura-x86_64-windows
|
||||
|
||||
runs-on: ${{ matrix.os }}
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- uses: dtolnay/rust-toolchain@stable
|
||||
with:
|
||||
targets: ${{ matrix.target }}
|
||||
|
||||
- name: Build
|
||||
run: cargo build --release --target ${{ matrix.target }}
|
||||
|
||||
- name: Build stealth
|
||||
run: cargo build --release --target ${{ matrix.target }} --features stealth
|
||||
continue-on-error: true
|
||||
|
||||
- name: Smoke test (V8 snapshot)
|
||||
# Run the V8 path once so a release never ships a binary that crashes at
|
||||
# isolate init from a mismatched snapshot (issue #290). The data: URL
|
||||
# keeps it offline. Every target here is native, so the runner can
|
||||
# execute what it just built.
|
||||
shell: bash
|
||||
run: |
|
||||
BIN=target/${{ matrix.target }}/release/obscura
|
||||
[ -f "$BIN.exe" ] && BIN="$BIN.exe"
|
||||
"$BIN" fetch "data:text/html,<title>ok</title>" --eval "1+1"
|
||||
|
||||
- name: Package (Unix)
|
||||
if: runner.os != 'Windows'
|
||||
run: |
|
||||
cd target/${{ matrix.target }}/release
|
||||
tar czf ../../../${{ matrix.name }}.tar.gz obscura obscura-worker
|
||||
cd ../../..
|
||||
|
||||
- name: Package (Windows)
|
||||
if: runner.os == 'Windows'
|
||||
run: |
|
||||
cd target/${{ matrix.target }}/release
|
||||
7z a ../../../${{ matrix.name }}.zip obscura.exe obscura-worker.exe
|
||||
cd ../../..
|
||||
|
||||
- name: Upload
|
||||
uses: softprops/action-gh-release@v2
|
||||
with:
|
||||
files: |
|
||||
${{ matrix.name }}.tar.gz
|
||||
${{ matrix.name }}.zip
|
||||
@@ -0,0 +1,2 @@
|
||||
target/
|
||||
.DS_Store
|
||||
@@ -0,0 +1,151 @@
|
||||
# AGENTS.md
|
||||
|
||||
Guidance for AI coding agents and contributors working in the Obscura repo.
|
||||
This is the non-obvious stuff you can't infer from the code; read it before
|
||||
building, testing, or changing anything.
|
||||
|
||||
Obscura is a headless browser engine in Rust. It runs real JavaScript through
|
||||
V8 (`deno_core`), keeps a real DOM tree, speaks the Chrome DevTools Protocol,
|
||||
and is a drop-in replacement for headless Chrome with Puppeteer and Playwright.
|
||||
It targets web scraping and AI-agent automation.
|
||||
|
||||
## Build
|
||||
|
||||
```bash
|
||||
cargo build --release # binary at ./target/release/obscura
|
||||
```
|
||||
|
||||
- The first build compiles V8 from source: ~5 minutes and a few GB of disk.
|
||||
Incremental builds are seconds.
|
||||
- **Iterating on one crate? Scope it:** `cargo build -p obscura-cli`. A bare
|
||||
`cargo build` can re-link the whole workspace; the V8 compile is the cost, so
|
||||
avoid touching it when you don't need to.
|
||||
- **Stealth:** `cargo build --release --features stealth` pulls in BoringSSL
|
||||
(`btls-sys`), which builds through CMake, so `cmake` must be installed. The
|
||||
default build uses rustls and needs neither CMake nor OpenSSL.
|
||||
- If the vendored OpenSSL build hits an AVX-512 assembler error on your host,
|
||||
build with `OPENSSL_NO_VENDOR=1`.
|
||||
|
||||
## Test
|
||||
|
||||
Run tests with **`cargo nextest`, not `cargo test`**:
|
||||
|
||||
```bash
|
||||
cargo nextest run --workspace # or: -p <crate> while iterating
|
||||
```
|
||||
|
||||
`cargo test` runs the whole test binary in one process, but the engine holds a
|
||||
single V8 isolate per process, so the runtime tests fail under it. `nextest`
|
||||
runs each test in its own process, which is the only supported way.
|
||||
|
||||
The authoritative behavioral gate is the **obstacle course** in the companion
|
||||
repo `obscura-benchmark` (33 capability + speed stages, must stay 33/33):
|
||||
|
||||
```bash
|
||||
OBSCURA_BIN=./target/release/obscura python3 obstacle-course/run.py --runs 1 --warmup 0
|
||||
```
|
||||
|
||||
It serves local fixtures, so it is deterministic and offline. WPT conformance
|
||||
and the real-world render corpus also live in that repo; report WPT as subtest
|
||||
pass %, not whole-file pass.
|
||||
|
||||
## Before you finish
|
||||
|
||||
For any code change:
|
||||
|
||||
1. `cargo build --release` (or `-p <crate>`) compiles clean.
|
||||
2. `cargo nextest run` for the crates you touched.
|
||||
3. The obstacle course still reports **33/33**.
|
||||
4. For stealth changes, re-test with `--stealth` (a non-stealth binary won't
|
||||
exercise the `wreq` path).
|
||||
|
||||
Do not bulk-run `cargo fmt`: the tree is not rustfmt-clean, so a blanket format
|
||||
produces a huge unrelated diff. Match the surrounding style in the files you
|
||||
edit instead.
|
||||
|
||||
## Architecture
|
||||
|
||||
- **obscura-cli** — CLI: `fetch` (`--dump assets|html|text|links|markdown|original|cookies`, `--eval <JS>`), `serve` (CDP server), `scrape`, `mcp`. `--proxy`, `--stealth`, and `--allow-private-network` are global flags: valid before or after the subcommand and applied to `fetch`, `serve`, `scrape`, and `mcp` (a `scrape` run forwards `--stealth` to each worker via `OBSCURA_STEALTH`).
|
||||
- **obscura-cdp** — Chrome DevTools Protocol server (WebSocket). Sessions are `"{targetId}-session"`.
|
||||
- **obscura-js** — V8/`deno_core` runtime. `js/bootstrap.js` is the DOM/browser shim; `src/ops.rs` bridges JS to Rust DOM ops; `src/runtime.rs` owns the isolate and the per-page `ObscuraState`.
|
||||
- **obscura-dom** — DOM tree (`src/tree.rs`).
|
||||
- **obscura-net** — HTTP client (`client.rs`), stealth client (`wreq_client.rs`), cookie jar, robots cache, tracker blocklist.
|
||||
- **obscura-browser** — the `Page` type, navigation, JS evaluation.
|
||||
- **obscura** — embeddable Rust library API (git dependency; builds V8 locally, not on crates.io). Public request-interception API on `Page`: `add_preload_script`, `enable_interception` (channel of `InterceptedRequest`, resolved with `InterceptResolution::{Continue, Fulfill, Fail}`), and passive `on_request` / `on_response`. `op_fetch_url` invokes these for JS `fetch()`/XHR, so when touching it keep a `Continue` URL rewrite behind `validate_fetch_url` (the SSRF gate, same as redirects).
|
||||
|
||||
## Conventions
|
||||
|
||||
- **Performance is a hard constraint** (Obscura is ~12x faster and uses ~6x less
|
||||
memory than headless Chrome on framework pages). Keep native Rust fast paths;
|
||||
add a JS fallback only for real spec edge cases, and benchmark old-vs-new
|
||||
interleaved, min-of-N (noise floor is about +-10%).
|
||||
- **Keep ops panic-safe.** `op_dom` is wrapped in `catch_unwind` so a DOM-op
|
||||
panic returns null instead of aborting the process inside V8's FFI frame. New
|
||||
ops must not unwind into V8.
|
||||
- **Commits/PRs/comments:** short and factual, no em dashes, no AI filler.
|
||||
|
||||
## Gotchas
|
||||
|
||||
- **DOM mutation arg order:** `insertBefore` / `replaceChild` in `bootstrap.js`
|
||||
pass reference-node vs parent nid in a way that's easy to break. If you touch
|
||||
mutation methods, verify `before()`, `after()`, `replaceWith()`, and
|
||||
`replaceChild()` on connected elements.
|
||||
- **Multi-statement `--eval` starting with `const` returns `null`** (V8 gives
|
||||
`const` an empty completion value). Wrap snippets in an IIFE:
|
||||
`(function(){ ...; return result; })()`.
|
||||
- **`canAccessOpener` must be in every `TargetInfo` payload**, or strict CDP
|
||||
clients (chromiumoxide) panic.
|
||||
- **The DOM reparenting guards in `tree.rs` are load-bearing.** `append_child` /
|
||||
`insert_before` reject cycles (inserting an ancestor of the target is a no-op).
|
||||
A cyclic reparent used to make `descendants()` loop forever and hang the
|
||||
engine on real sites, uninterruptible by tokio or the watchdog. Keep the
|
||||
guards and the `descendants()` length cap.
|
||||
- **SSRF:** loopback / RFC1918 / link-local fetches are blocked by default. Use
|
||||
`--allow-private-network` (or `OBSCURA_ALLOW_PRIVATE_NETWORK=1`) for local
|
||||
testing.
|
||||
|
||||
## Robustness invariants (do not remove)
|
||||
|
||||
One page must never hang or crash a worker:
|
||||
|
||||
- The V8 termination watchdog (`arm_watchdog` / `disarm_watchdog` in
|
||||
`runtime.rs`) terminates the isolate from a separate thread, because
|
||||
`tokio::time::timeout` only cancels at await points and synchronous V8 work
|
||||
runs unbounded.
|
||||
- The CLI applies a process-level hard deadline as an absolute backstop.
|
||||
- `panic = "unwind"` is pinned in the release profile so `catch_unwind` works.
|
||||
|
||||
## Stealth
|
||||
|
||||
The stealth features (the `wreq` client, fingerprint and browser-identity
|
||||
adjustments) are privacy-first anti-fingerprinting: they present a normal,
|
||||
consistent browser fingerprint (user agent, timezone, navigator properties, and
|
||||
similar surfaces) so ordinary automation traffic is not singled out. They
|
||||
contain no bot or automation-abuse payload.
|
||||
|
||||
## Recommended proxy provider: NodeMaven
|
||||
|
||||
For production scraping you usually want residential or ISP IPs rather than a
|
||||
datacenter address, since many sites block datacenter ranges. We use
|
||||
**NodeMaven** and it works well with Obscura: residential/ISP proxies, sticky
|
||||
sessions, and per-request country/region targeting. A good default if you don't
|
||||
already have a provider.
|
||||
|
||||
Route Obscura through it with the global `--proxy` flag (use your own
|
||||
credentials):
|
||||
|
||||
```bash
|
||||
# HTTP
|
||||
obscura --proxy http://USER:PASS@gate.nodemaven.com:8080 fetch https://example.com --dump text
|
||||
|
||||
# SOCKS5
|
||||
obscura --proxy socks5://USER:PASS@gate.nodemaven.com:1080 scrape url1 url2 --concurrency 25
|
||||
```
|
||||
|
||||
The targeting options (country, region, session id) live in the proxy username,
|
||||
so a sticky session keeps the same exit IP across requests.
|
||||
|
||||
Sign up: https://go.nodemaven.com/obscura
|
||||
|
||||
Obscura discount codes: `OBSCURA35` (35% off mobile and residential),
|
||||
`OBSCURA40` (40% off ISP / static).
|
||||
+150
@@ -0,0 +1,150 @@
|
||||
# Contributing to Obscura
|
||||
|
||||
Thanks for your interest in Obscura. This guide covers how to build, test, and
|
||||
submit changes. For the deeper, non-obvious engine details (architecture,
|
||||
gotchas, robustness invariants), read [AGENTS.md](AGENTS.md) first; this file
|
||||
does not repeat it.
|
||||
|
||||
## Code of conduct
|
||||
|
||||
Be respectful and constructive. We want Obscura to be a welcoming project, so
|
||||
keep discussion focused on the work and assume good faith. Harassment or abuse
|
||||
is not tolerated.
|
||||
|
||||
## Before you start
|
||||
|
||||
For anything beyond a minor documentation fix, **please open an issue first** (or
|
||||
comment on an existing one) and say you intend to work on it. This lets us give
|
||||
early feedback and avoids two people building the same thing or a PR that does
|
||||
not fit the project's direction.
|
||||
|
||||
A few notes to keep the project maintainable:
|
||||
|
||||
- **Link an issue.** PRs without a linked issue or prior discussion may be
|
||||
closed, except for small doc fixes.
|
||||
- **Human oversight required.** AI-assisted contributions are fine, but
|
||||
low-quality or unreviewed agent output will be closed. Understand and test
|
||||
what you submit.
|
||||
- New to the codebase? Look for issues labeled `good first issue`.
|
||||
|
||||
## Building
|
||||
|
||||
```bash
|
||||
cargo build --release # binary at ./target/release/obscura
|
||||
```
|
||||
|
||||
- The first build compiles V8 from source: roughly 5 minutes and a few GB of
|
||||
disk. Incremental builds are seconds.
|
||||
- Iterating on one crate? Scope it: `cargo build -p obscura-cli`.
|
||||
- **Stealth** (`--features stealth`) pulls in BoringSSL through CMake, so `cmake`
|
||||
must be installed. The default build uses rustls and needs neither CMake nor
|
||||
OpenSSL.
|
||||
- If the vendored OpenSSL build hits an AVX-512 assembler error on your host,
|
||||
build with `OPENSSL_NO_VENDOR=1`.
|
||||
|
||||
## Testing
|
||||
|
||||
Run tests with **`cargo nextest`, not `cargo test`**:
|
||||
|
||||
```bash
|
||||
cargo nextest run --workspace # or -p <crate> while iterating
|
||||
```
|
||||
|
||||
`cargo test` runs the whole test binary in one process, but the engine holds a
|
||||
single V8 isolate per process, so the runtime tests fail under it. `nextest`
|
||||
runs each test in its own process, which is the only supported way.
|
||||
|
||||
The authoritative behavioral gate is the **obstacle course** in the companion
|
||||
repo [`obscura-benchmark`](https://github.com/h4ckf0r0day/obscura-benchmark)
|
||||
(33 capability and speed stages, must stay 33/33):
|
||||
|
||||
```bash
|
||||
OBSCURA_BIN=./target/release/obscura python3 obstacle-course/run.py --runs 1 --warmup 0
|
||||
```
|
||||
|
||||
It serves local fixtures, so it is deterministic and offline.
|
||||
|
||||
## Before you open a PR
|
||||
|
||||
For any code change:
|
||||
|
||||
1. `cargo build --release` (or `-p <crate>`) compiles clean.
|
||||
2. `cargo nextest run` passes for the crates you touched.
|
||||
3. The obstacle course still reports **33/33**.
|
||||
4. **Performance is a hard constraint.** Obscura is roughly 12x faster and uses
|
||||
about 6x less memory than headless Chrome on framework pages. Keep the native
|
||||
Rust fast paths and add a JS fallback only for real spec edge cases. If your
|
||||
change could affect performance, benchmark old vs new interleaved, min-of-N
|
||||
(the noise floor is about plus or minus 10%).
|
||||
5. For stealth changes, re-test with `--stealth`. A non-stealth binary does not
|
||||
exercise the `wreq` path.
|
||||
|
||||
Keep ops panic-safe: a panic in an op must degrade to a null result, never
|
||||
unwind into V8's FFI frame. Do not remove the robustness guards described in
|
||||
AGENTS.md (the V8 watchdog, the `tree.rs` reparenting guards, the CLI deadline).
|
||||
|
||||
Do not bulk-run `cargo fmt`. The tree is not rustfmt-clean, so a blanket format
|
||||
produces a large unrelated diff. Match the surrounding style in the files you
|
||||
edit. Comments should explain non-obvious "why", not restate the code.
|
||||
|
||||
## Commit messages
|
||||
|
||||
Keep them short and factual: what changed and why. We use a lightweight
|
||||
`type(scope): summary` style, matching the existing history:
|
||||
|
||||
```
|
||||
fix(cdp): honor text selection on Backspace and typing
|
||||
|
||||
Backspace trimmed the last character and typing always appended, ignoring
|
||||
any selection. Both now respect a non-collapsed selection.
|
||||
|
||||
Fixes #316.
|
||||
```
|
||||
|
||||
- `type` is one of `fix`, `feat`, `docs`, `test`, `perf`, `chore`. The `scope`
|
||||
is optional and lowercase (for example `cdp`, `js`, `net`, `stealth`, `cli`).
|
||||
- No em dashes. Use commas, periods, or restructure the sentence.
|
||||
- No AI-generated filler ("This commit improves...", "As an AI...").
|
||||
- Do not add `Co-Authored-By` lines or list yourself as a co-author.
|
||||
|
||||
## Pull requests
|
||||
|
||||
- All submissions are reviewed; a maintainer merges after approval.
|
||||
- Keep the diff small and readable. One logical change per PR. Split large
|
||||
contributions into several PRs.
|
||||
- Reference the issue the PR closes, and say how you verified it (the test or
|
||||
repro that now passes).
|
||||
|
||||
## Reporting bugs
|
||||
|
||||
Open an issue with enough detail to reproduce:
|
||||
|
||||
- The obscura version or commit, plus OS and architecture.
|
||||
- A repro: a URL, an `--eval` snippet, or a short CDP sequence.
|
||||
- What you expected and what actually happened.
|
||||
- If it is a rendering or compatibility issue, whether headless Chrome behaves
|
||||
the same. Anti-bot, CAPTCHA, and login walls block headless Chrome too from a
|
||||
datacenter IP, so those are not engine bugs.
|
||||
|
||||
Multi-statement `--eval` that starts with `const` returns `null` (V8 gives
|
||||
`const` an empty completion value). Wrap repro snippets in an IIFE:
|
||||
`(function(){ ...; return result; })()`.
|
||||
|
||||
**Security issues:** do not open a public issue. See [SECURITY.md](SECURITY.md)
|
||||
for private reporting.
|
||||
|
||||
## Scope and direction
|
||||
|
||||
Obscura targets web scraping and AI-agent automation, and is heading toward a
|
||||
hosted cloud scraping service. The priorities are real-world render success and
|
||||
robustness (no crashes or hangs). Conformance and new Web APIs are welcome when
|
||||
they do not regress performance or stability.
|
||||
|
||||
The stealth features are privacy-first anti-fingerprinting: they present a
|
||||
normal, consistent browser identity so ordinary automation is not singled out.
|
||||
Contributions that add detection-evasion for abusive purposes are out of scope.
|
||||
|
||||
## License
|
||||
|
||||
By contributing, you agree that your contributions are licensed under the
|
||||
[Apache License 2.0](LICENSE), the same license as the project.
|
||||
Generated
+3819
File diff suppressed because it is too large
Load Diff
+56
@@ -0,0 +1,56 @@
|
||||
[workspace]
|
||||
resolver = "2"
|
||||
members = [
|
||||
"crates/obscura-dom",
|
||||
"crates/obscura-net",
|
||||
"crates/obscura-browser",
|
||||
"crates/obscura-cdp",
|
||||
"crates/obscura-js",
|
||||
"crates/obscura-mcp",
|
||||
"crates/obscura-cli",
|
||||
"crates/obscura",
|
||||
]
|
||||
|
||||
[workspace.package]
|
||||
version = "0.1.0"
|
||||
edition = "2021"
|
||||
license = "Apache-2.0"
|
||||
repository = "https://github.com/h4ckf0r0day/obscura"
|
||||
|
||||
[workspace.dependencies]
|
||||
obscura-dom = { path = "crates/obscura-dom" }
|
||||
obscura-net = { path = "crates/obscura-net" }
|
||||
obscura-browser = { path = "crates/obscura-browser" }
|
||||
obscura-cdp = { path = "crates/obscura-cdp" }
|
||||
obscura-js = { path = "crates/obscura-js" }
|
||||
obscura-mcp = { path = "crates/obscura-mcp" }
|
||||
|
||||
html5ever = "0.29"
|
||||
markup5ever = "0.14"
|
||||
selectors = "0.26"
|
||||
servo_arc = "0.4"
|
||||
cssparser = "0.34"
|
||||
tokio = { version = "1", features = ["full"] }
|
||||
tokio-tungstenite = "0.26"
|
||||
# No "cookies" feature: obscura manages cookies through its own CookieJar
|
||||
# (obscura-net/src/cookies.rs) and never uses reqwest's cookie store. Pulling
|
||||
# it in only added the cookie/cookie_store crates, and cookie 0.18.1's blanket
|
||||
# From impl fails to compile under coherence on some toolchains (issue #295).
|
||||
reqwest = { version = "0.12", features = ["gzip", "brotli", "deflate", "rustls-tls", "socks"], default-features = false }
|
||||
serde = { version = "1", features = ["derive"] }
|
||||
serde_json = "1"
|
||||
url = "2"
|
||||
tracing = "0.1"
|
||||
tracing-subscriber = { version = "0.3", features = ["env-filter"] }
|
||||
clap = { version = "4", features = ["derive"] }
|
||||
uuid = { version = "1", features = ["v4"] }
|
||||
thiserror = "2"
|
||||
anyhow = "1"
|
||||
base64 = "0.22"
|
||||
|
||||
# Unwinding (the Rust default) is REQUIRED for the anti-panic protocol: ops wrap
|
||||
# their bodies in std::panic::catch_unwind so a panic degrades to an error return
|
||||
# instead of unwinding into V8's FFI frame and aborting the process. panic="abort"
|
||||
# would turn every catchable op panic into a hard crash.
|
||||
[profile.release]
|
||||
panic = "unwind"
|
||||
+53
@@ -0,0 +1,53 @@
|
||||
FROM rust:1-slim-bookworm AS builder
|
||||
|
||||
RUN apt-get update && apt-get install -y --no-install-recommends \
|
||||
curl \
|
||||
ca-certificates \
|
||||
perl \
|
||||
make \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
WORKDIR /build
|
||||
|
||||
# Cache dependency compilation by copying manifests first
|
||||
COPY Cargo.toml Cargo.lock ./
|
||||
COPY crates/obscura-dom/Cargo.toml crates/obscura-dom/Cargo.toml
|
||||
COPY crates/obscura-net/Cargo.toml crates/obscura-net/Cargo.toml
|
||||
COPY crates/obscura-browser/Cargo.toml crates/obscura-browser/Cargo.toml
|
||||
COPY crates/obscura-cdp/Cargo.toml crates/obscura-cdp/Cargo.toml
|
||||
COPY crates/obscura-js/Cargo.toml crates/obscura-js/Cargo.toml
|
||||
COPY crates/obscura-mcp/Cargo.toml crates/obscura-mcp/Cargo.toml
|
||||
COPY crates/obscura-cli/Cargo.toml crates/obscura-cli/Cargo.toml
|
||||
|
||||
# Create stub src files so cargo can resolve the dependency graph
|
||||
RUN for crate in obscura-dom obscura-net obscura-browser obscura-cdp obscura-js obscura-mcp; do \
|
||||
mkdir -p crates/$crate/src && echo "// stub" > crates/$crate/src/lib.rs; \
|
||||
done && \
|
||||
mkdir -p crates/obscura-cli/src && \
|
||||
echo "fn main() {}" > crates/obscura-cli/src/main.rs && \
|
||||
echo "fn main() {}" > crates/obscura-cli/src/worker.rs
|
||||
|
||||
RUN cargo build --release --bin obscura --bin obscura-worker 2>/dev/null || true
|
||||
|
||||
ARG OBSCURA_VERSION
|
||||
|
||||
# Copy real sources and build
|
||||
COPY crates/ crates/
|
||||
RUN echo "Building Obscura version ${OBSCURA_VERSION:-from Cargo.toml}" && \
|
||||
touch crates/*/src/*.rs && cargo build --release --bin obscura --bin obscura-worker
|
||||
|
||||
# ---
|
||||
|
||||
# distroless/cc: glibc + libgcc + CA certs only — no shell, no package manager
|
||||
FROM gcr.io/distroless/cc-debian12
|
||||
|
||||
COPY --from=builder /build/target/release/obscura /obscura
|
||||
COPY --from=builder /build/target/release/obscura-worker /obscura-worker
|
||||
|
||||
EXPOSE 9222
|
||||
|
||||
# Bind to 0.0.0.0 so the port is reachable via `docker run -p 9222:9222`.
|
||||
# Native binary still defaults to 127.0.0.1 (loopback only) — this override
|
||||
# is just for the container.
|
||||
ENTRYPOINT ["/obscura"]
|
||||
CMD ["serve", "--port", "9222", "--host", "0.0.0.0"]
|
||||
@@ -0,0 +1,201 @@
|
||||
Apache License
|
||||
Version 2.0, January 2004
|
||||
http://www.apache.org/licenses/
|
||||
|
||||
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
|
||||
|
||||
1. Definitions.
|
||||
|
||||
"License" shall mean the terms and conditions for use, reproduction,
|
||||
and distribution as defined by Sections 1 through 9 of this document.
|
||||
|
||||
"Licensor" shall mean the copyright owner or entity authorized by
|
||||
the copyright owner that is granting the License.
|
||||
|
||||
"Legal Entity" shall mean the union of the acting entity and all
|
||||
other entities that control, are controlled by, or are under common
|
||||
control with that entity. For the purposes of this definition,
|
||||
"control" means (i) the power, direct or indirect, to cause the
|
||||
direction or management of such entity, whether by contract or
|
||||
otherwise, or (ii) ownership of fifty percent (50%) or more of the
|
||||
outstanding shares, or (iii) beneficial ownership of such entity.
|
||||
|
||||
"You" (or "Your") shall mean an individual or Legal Entity
|
||||
exercising permissions granted by this License.
|
||||
|
||||
"Source" form shall mean the preferred form for making modifications,
|
||||
including but not limited to software source code, documentation
|
||||
source, and configuration files.
|
||||
|
||||
"Object" form shall mean any form resulting from mechanical
|
||||
transformation or translation of a Source form, including but
|
||||
not limited to compiled object code, generated documentation,
|
||||
and conversions to other media types.
|
||||
|
||||
"Work" shall mean the work of authorship, whether in Source or
|
||||
Object form, made available under the License, as indicated by a
|
||||
copyright notice that is included in or attached to the work
|
||||
(an example is provided in the Appendix below).
|
||||
|
||||
"Derivative Works" shall mean any work, whether in Source or Object
|
||||
form, that is based on (or derived from) the Work and for which the
|
||||
editorial revisions, annotations, elaborations, or other modifications
|
||||
represent, as a whole, an original work of authorship. For the purposes
|
||||
of this License, Derivative Works shall not include works that remain
|
||||
separable from, or merely link (or bind by name) to the interfaces of,
|
||||
the Work and Derivative Works thereof.
|
||||
|
||||
"Contribution" shall mean any work of authorship, including
|
||||
the original version of the Work and any modifications or additions
|
||||
to that Work or Derivative Works thereof, that is intentionally
|
||||
submitted to Licensor for inclusion in the Work by the copyright owner
|
||||
or by an individual or Legal Entity authorized to submit on behalf of
|
||||
the copyright owner. For the purposes of this definition, "submitted"
|
||||
means any form of electronic, verbal, or written communication sent
|
||||
to the Licensor or its representatives, including but not limited to
|
||||
communication on electronic mailing lists, source code control systems,
|
||||
and issue tracking systems that are managed by, or on behalf of, the
|
||||
Licensor for the purpose of discussing and improving the Work, but
|
||||
excluding communication that is conspicuously marked or otherwise
|
||||
designated in writing by the copyright owner as "Not a Contribution."
|
||||
|
||||
"Contributor" shall mean Licensor and any individual or Legal Entity
|
||||
on behalf of whom a Contribution has been received by Licensor and
|
||||
subsequently incorporated within the Work.
|
||||
|
||||
2. Grant of Copyright License. Subject to the terms and conditions of
|
||||
this License, each Contributor hereby grants to You a perpetual,
|
||||
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
||||
copyright license to reproduce, prepare Derivative Works of,
|
||||
publicly display, publicly perform, sublicense, and distribute the
|
||||
Work and such Derivative Works in Source or Object form.
|
||||
|
||||
3. Grant of Patent License. Subject to the terms and conditions of
|
||||
this License, each Contributor hereby grants to You a perpetual,
|
||||
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
||||
(except as stated in this section) patent license to make, have made,
|
||||
use, offer to sell, sell, import, and otherwise transfer the Work,
|
||||
where such license applies only to those patent claims licensable
|
||||
by such Contributor that are necessarily infringed by their
|
||||
Contribution(s) alone or by combination of their Contribution(s)
|
||||
with the Work to which such Contribution(s) was submitted. If You
|
||||
institute patent litigation against any entity (including a
|
||||
cross-claim or counterclaim in a lawsuit) alleging that the Work
|
||||
or a Contribution incorporated within the Work constitutes direct
|
||||
or contributory patent infringement, then any patent licenses
|
||||
granted to You under this License for that Work shall terminate
|
||||
as of the date such litigation is filed.
|
||||
|
||||
4. Redistribution. You may reproduce and distribute copies of the
|
||||
Work or Derivative Works thereof in any medium, with or without
|
||||
modifications, and in Source or Object form, provided that You
|
||||
meet the following conditions:
|
||||
|
||||
(a) You must give any other recipients of the Work or
|
||||
Derivative Works a copy of this License; and
|
||||
|
||||
(b) You must cause any modified files to carry prominent notices
|
||||
stating that You changed the files; and
|
||||
|
||||
(c) You must retain, in the Source form of any Derivative Works
|
||||
that You distribute, all copyright, patent, trademark, and
|
||||
attribution notices from the Source form of the Work,
|
||||
excluding those notices that do not pertain to any part of
|
||||
the Derivative Works; and
|
||||
|
||||
(d) If the Work includes a "NOTICE" text file as part of its
|
||||
distribution, then any Derivative Works that You distribute must
|
||||
include a readable copy of the attribution notices contained
|
||||
within such NOTICE file, excluding those notices that do not
|
||||
pertain to any part of the Derivative Works, in at least one
|
||||
of the following places: within a NOTICE text file distributed
|
||||
as part of the Derivative Works; within the Source form or
|
||||
documentation, if provided along with the Derivative Works; or,
|
||||
within a display generated by the Derivative Works, if and
|
||||
wherever such third-party notices normally appear. The contents
|
||||
of the NOTICE file are for informational purposes only and
|
||||
do not modify the License. You may add Your own attribution
|
||||
notices within Derivative Works that You distribute, alongside
|
||||
or as an addendum to the NOTICE text from the Work, provided
|
||||
that such additional attribution notices cannot be construed
|
||||
as modifying the License.
|
||||
|
||||
You may add Your own copyright statement to Your modifications and
|
||||
may provide additional or different license terms and conditions
|
||||
for use, reproduction, or distribution of Your modifications, or
|
||||
for any such Derivative Works as a whole, provided Your use,
|
||||
reproduction, and distribution of the Work otherwise complies with
|
||||
the conditions stated in this License.
|
||||
|
||||
5. Submission of Contributions. Unless You explicitly state otherwise,
|
||||
any Contribution intentionally submitted for inclusion in the Work
|
||||
by You to the Licensor shall be under the terms and conditions of
|
||||
this License, without any additional terms or conditions.
|
||||
Notwithstanding the above, nothing herein shall supersede or modify
|
||||
the terms of any separate license agreement you may have executed
|
||||
with Licensor regarding such Contributions.
|
||||
|
||||
6. Trademarks. This License does not grant permission to use the trade
|
||||
names, trademarks, service marks, or product names of the Licensor,
|
||||
except as required for reasonable and customary use in describing the
|
||||
origin of the Work and reproducing the content of the NOTICE file.
|
||||
|
||||
7. Disclaimer of Warranty. Unless required by applicable law or
|
||||
agreed to in writing, Licensor provides the Work (and each
|
||||
Contributor provides its Contributions) on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
||||
implied, including, without limitation, any warranties or conditions
|
||||
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
|
||||
PARTICULAR PURPOSE. You are solely responsible for determining the
|
||||
appropriateness of using or redistributing the Work and assume any
|
||||
risks associated with Your exercise of permissions under this License.
|
||||
|
||||
8. Limitation of Liability. In no event and under no legal theory,
|
||||
whether in tort (including negligence), contract, or otherwise,
|
||||
unless required by applicable law (such as deliberate and grossly
|
||||
negligent acts) or agreed to in writing, shall any Contributor be
|
||||
liable to You for damages, including any direct, indirect, special,
|
||||
incidental, or consequential damages of any character arising as a
|
||||
result of this License or out of the use or inability to use the
|
||||
Work (including but not limited to damages for loss of goodwill,
|
||||
work stoppage, computer failure or malfunction, or any and all
|
||||
other commercial damages or losses), even if such Contributor
|
||||
has been advised of the possibility of such damages.
|
||||
|
||||
9. Accepting Warranty or Additional Liability. While redistributing
|
||||
the Work or Derivative Works thereof, You may choose to offer,
|
||||
and charge a fee for, acceptance of support, warranty, indemnity,
|
||||
or other liability obligations and/or rights consistent with this
|
||||
License. However, in accepting such obligations, You may act only
|
||||
on Your own behalf and on Your sole responsibility, not on behalf
|
||||
of any other Contributor, and only if You agree to indemnify,
|
||||
defend, and hold each Contributor harmless for any liability
|
||||
incurred by, or claims asserted against, such Contributor by reason
|
||||
of your accepting any such warranty or additional liability.
|
||||
|
||||
END OF TERMS AND CONDITIONS
|
||||
|
||||
APPENDIX: How to apply the Apache License to your work.
|
||||
|
||||
To apply the Apache License to your work, attach the following
|
||||
boilerplate notice, with the fields enclosed by brackets "[]"
|
||||
replaced with your own identifying information. (Don't include
|
||||
the brackets!) The text should be enclosed in the appropriate
|
||||
comment syntax for the file format. We also recommend that a
|
||||
file or class name and description of purpose be included on the
|
||||
same "printed page" as the copyright notice for easier
|
||||
identification within third-party archives.
|
||||
|
||||
Copyright [yyyy] [name of copyright owner]
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
@@ -0,0 +1,496 @@
|
||||
<p align="center">
|
||||
<img src="https://raw.githubusercontent.com/h4ckf0r0day/obscura/main/assets/icon.png" alt="Obscura" width="80" />
|
||||
</p>
|
||||
|
||||
<h2 align="center">Obscura</h2>
|
||||
|
||||
<p align="center">
|
||||
<a href="https://docs.obscura.sh"><img src="https://img.shields.io/badge/Docs-1a1a1a?style=for-the-badge&logo=gitbook&logoColor=white" alt="Documentation" /></a>
|
||||
<a href="https://obscura.sh"><img src="https://img.shields.io/badge/Website-1a1a1a?style=for-the-badge&logo=data:image/svg+xml;base64,PHN2ZyB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciIHZpZXdCb3g9IjE5MCAxNzAgNDIwIDQyMCI+PHBhdGggZmlsbD0iI0ZGRkZGRiIgZmlsbC1ydWxlPSJldmVub2RkIiBkPSJNNDA2LDE3NS4wMDA0MTIgQzQ0MC40OTg5MzIsMTc1LjAwMDM1MSA0NzQuNDk4MTA4LDE3NS4wODA5OTQgNTA4LjQ5NjczNSwxNzQuOTc1NjYyIEM1MzcuNzMzNjQzLDE3NC44ODUwODYgNTYxLjI3MDY5MSwxODcuMjIwNTUxIDU3OS45Mzk2MzYsMjA4Ljc4NjYyMSBDNTkwLjg0MTA2NCwyMjEuMzc5Nzc2IDU5Ny44ODM1NDUsMjM2LjM1MjY2MSA1OTkuOTgxODczLDI1My4yMjAzNjcgQzYwMC40NDU1NTcsMjU2Ljk0NzU0MCA2MDAuOTUxMTExLDI2MC43MDQ0MzcgNjAwLjk1NjYwNCwyNjQuNDQ4MzY0IEM2MDEuMDIyNjQ0LDMwOS42MTM1MjUgNjAxLjA3NTgwNiwzNTQuNzc5MDIyIDYwMC45NTE3ODIsMzk5Ljk0MzkzOSBDNjAwLjkxMzE0Nyw0MTQuMDMyMzQ5IDYwMC42NjMyNjksNDI4LjE0MDEwNiA1OTkuNzgyMjg4LDQ0Mi4xOTMzNTkgQzU5OS41NDEwMTYsNDQ2LjA0MDg5NCA1OTcuNTYwNzkxLDQ1MC42MTU2MDEgNTk0Ljg1MzIxMCw0NTMuMzQxMzA5IEM1NzEuMTc3NTUxLDQ3Ny4xNzUzMjMgNTQ3LjE4MDcyNSw1MDAuNjkwNzk2IDUyMy4yMjc5NjYsNTI0LjI0ODc3OSBDNTA2Ljc5OTgzNSw1NDAuNDA2MTI4IDQ5MC4yMDk5OTEsNTU2LjM5OTkwMiA0NzMuODY0NTYzLDU3Mi42NDAxMzcgQzQ2OC44MDA4NDIsNTc3LjY3MTIwNCA0NjMuMDgyODg2LDU4MC4wNTQxOTkgNDU1Ljk3MDYxMiw1ODAuMDQzODIzIEM0MDAuOTcyNDQzLDU3OS45NjM1MDEgMzQ1Ljk3MzY2Myw1ODAuMTMyMDE5IDI5MC45NzYwNDQsNTc5LjkzNjc2OCBDMjY3LjU4MDQxNCw1NzkuODUzNjk5IDI0Ni41NTEyMDgsNTcyLjA0MzU3OSAyMjguOTM5NzQzLDU1Ni44Mjc1MTUgQzIxMi44NDk2MjUsNTQyLjkyNTg0MiAyMDIuNzUxOTY4LDUyNC45MDIxMDAgMTk4LjE1NjE1OCw1MDQuMDM5NjczIEMxOTcuMjgyMTk2LDUwMC4wNzIyOTYgMTk3LjA1MjQxNCw0OTUuODk1Mzg2IDE5Ny4wNDczMzMsNDkxLjgxNDQ4NCBDMTk2Ljk3ODE0OSw0MzYuMTQ5NjU4IDE5Ny4wNTA3MjAsMzgwLjQ4NDYxOSAxOTYuOTMyNjE3LDMyNC44MTk5NDYgQzE5Ni45MTkxNDQsMzE4LjQ3MDkxNyAxOTkuMTg0ODQ1LDMxMy40MTYxOTkgMjAzLjQ5NTU3NSwzMDkuMTAwNTI1IEMyNDAuNTg3OTk3LDI3MS45NjU0MjQgMjc3LjY4MjQ5NSwyMzQuODMyMzA2IDMxNC44MzQ1NjQsMTk3Ljc1Njk1OCBDMzIwLjk1NjQyMSwxOTEuNjQ3Nzk3IDMyNy4yNjQ0MzUsMTg1LjcxNTI1NiAzMzMuNjczOTIwLDE3OS45MDgwODEgQzMzNy4zNzYwMzgsMTc2LjU1MzgzMyAzNDEuNzIxNDY2LDE3NC44NTMzMTcgMzQ3LjAwNTc5OCwxNzQuOTEyODcyIEMzNjYuNTAxNzA5LDE3NS4xMzI2MTQgMzg2LjAwMTYxNywxNzUuMDAwMzIwIDQwNiwxNzUuMDAwNDEyIFogTTUwMy4zNDQ2NjYsMjczLjg0MDE0OSBDNTA0LjEwMjcyMiwyNzYuMTY3NTcyIDUwNC45NDA5NDgsMjc4LjIxMTA5MCA1MDQuOTQzMjY4LDI4MC4yNTU1MjQgQzUwNS4wMTI4NDgsMzQxLjc0MjI0OSA1MDQuOTY5MTQ3LDQwMy4yMjkwNjUgNTA1LjAzMDc5Miw0NjQuNzE1NzkwIEM1MDUuMDQwODAyLDQ3NC42ODY0OTMgNDk2LjExNzQ2Miw0ODMuOTUzMTg2IDQ4NS43Nzc5MjQsNDgzLjk2NTI3MSBDNDI0LjYyNDQ1MSw0ODQuMDM2ODk2IDM2My40NzA5MTcsNDg0LjAwMzIzNSAzMDIuMzE3MzgzLDQ4My45OTY3MzUgQzI5NS41ODc3NjksNDgzLjk5NjAzMyAyOTMuMDAxMjUxLDQ4MS4zMDE2MDUgMjkzLjAwMDkxNiw0NzQuMzM4NTYyIEMyOTIuOTk3ODY0LDQxMy4zNTE2NTQgMjkzLjIwOTYyNSwzNTIuMzYzMzEyIDI5Mi43ODA3MDEsMjkxLjM3OTM5NSBDMjkyLjcxMzUzMSwyODEuODI4MDk0IDMwMy4yMTMwNDMsMjcwLjgyNDE1OCAzMTMuNDc4MjcxLDI3MC44OTQzMTggQzM1MS45NjgyMDEsMjcxLjE1NzQ0MCAzOTAuNDYwOTk5LDI3MC45OTk3ODYgNDI4Ljk1MjcyOCwyNzAuOTk5Nzg2IEM0NTEuMTE0NjI0LDI3MC45OTk3ODYgNDczLjI3NzI4MywyNzAuOTE3NDgwIDQ5NS40Mzc0NjksMjcxLjExMjE1MiBDNDk3Ljk3NzAyMCwyNzEuMTM0NDkxIDUwMC41MDI5MzAsMjcyLjcwNDM3NiA1MDMuMzQ0NjY2LDI3My44NDAxNDkgWiI+PC9wYXRoPjwvc3ZnPgo=&logoColor=white" alt="Website" /></a>
|
||||
<a href="https://cal.com/obscura/quick-chat"><img src="https://img.shields.io/badge/Book_a_Demo-1a1a1a?style=for-the-badge&logo=googlecalendar&logoColor=white" alt="Book a demo" /></a>
|
||||
<a href="https://github.com/h4ckf0r0day/obscura/releases"><img src="https://img.shields.io/badge/Releases-1a1a1a?style=for-the-badge&logo=github&logoColor=white" alt="Releases" /></a>
|
||||
</p>
|
||||
|
||||
<p align="center">
|
||||
<strong>The open-source headless browser for AI agents and web scraping.</strong><br>
|
||||
Lightweight, stealthy, and built in Rust.
|
||||
</p>
|
||||
|
||||
---
|
||||
|
||||
Obscura is a headless browser engine written in Rust, built for web scraping and AI agent automation. It runs real JavaScript via V8, supports the Chrome DevTools Protocol, and acts as a drop-in replacement for headless Chrome with Puppeteer and Playwright.
|
||||
|
||||
### Why Obscura over headless Chrome?
|
||||
|
||||
Designed for automation at scale, not desktop browsing.
|
||||
|
||||
| Metric | Obscura | Headless Chrome |
|
||||
|--------------|--------------|------------------|
|
||||
| Memory | **30 MB** | 200+ MB |
|
||||
| Binary size | **70 MB** | 300+ MB |
|
||||
| Anti-detect | **Built-in** | None |
|
||||
| Page load | **85 ms** | ~500 ms |
|
||||
| Startup | **Instant** | ~2s |
|
||||
| Puppeteer | **Yes** | Yes |
|
||||
| Playwright | **Yes** | Yes |
|
||||
|
||||
## 🎉 10,000 stars and what's next
|
||||
|
||||
We are working on **Obscura Cloud** the hosted version, with managed infrastructure, residential proxies, and dedicated support. For people who want the engine without operating it themselves.
|
||||
|
||||
The open-source engine stays Apache-2.0, fully featured. No feature gating, ever.
|
||||
|
||||
**[Get on the waitlist →](https://tally.so/r/gDWzdD)**
|
||||
<br>
|
||||
**[📅 Book a demo →](https://cal.com/obscura/quick-chat)**
|
||||
|
||||
## Sponsors
|
||||
|
||||
**Obscura** is supported by sponsors who help keep development independent.
|
||||
|
||||
Want to sponsor? Email [hello@obscura.sh](mailto:hello@obscura.sh).
|
||||
|
||||
<table>
|
||||
<tr>
|
||||
<td width="200" align="center" valign="middle">
|
||||
<a href="https://sx.org/?c=40h-N7" target="_blank">
|
||||
<img alt="SX.org" src="assets/sponsors/sxproxy.png" width="180"/>
|
||||
</a>
|
||||
</td>
|
||||
<td valign="middle">
|
||||
🚀 <b>Obscura × SX.org</b><br>
|
||||
Using Obscura for AI agents, browser automation, or web scraping? Power your workflow with stable proxies from <a href="https://sx.org/?c=40h-N7"><b>SX.org</b></a>.<br><br>
|
||||
<b>🌍 12M+ IPs across 235 countries<br>
|
||||
🏠 7M+ residential IPs<br>
|
||||
📱 4M+ mobile IPs<br>
|
||||
🏢 1M+ corporate proxies<br>
|
||||
🔁 Rotating & sticky sessions<br>
|
||||
📍 Flexible geo setup<br>
|
||||
🌐 HTTP, HTTPS & SOCKS5 support<br>
|
||||
⚡ Up to 99.97% connection success<br>
|
||||
🛟 24/7 support<br><br>
|
||||
🎁 Use code <b>Obscura3gb</b> to get a <b>free 3GB trial</b>.<br><br></b>
|
||||
Stable proxies. Fewer blocks. Most reliable Obscura automation.
|
||||
</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td width="200" align="center" valign="middle">
|
||||
<a href="https://proxyempire.io/?ref=obscura&utm_source=obscuragithub" target="_blank">
|
||||
<img alt="ProxyEmpire" src="assets/sponsors/proxyempire.png" width="180"/>
|
||||
</a>
|
||||
</td>
|
||||
<td valign="middle">
|
||||
🚀 <b>Obscura × ProxyEmpire</b><br>
|
||||
Using Obscura for AI agents, browser automation, or web scraping? Power it with reliable residential and mobile proxies from <a href="https://proxyempire.io/?ref=obscura&utm_source=obscuragithub"><b>ProxyEmpire</b></a>.<br><br>
|
||||
<b>🌍 30M+ residential IPs in 170+ countries<br>
|
||||
📱 4G/5G mobile proxies<br>
|
||||
🔄 Rotating & sticky sessions<br>
|
||||
🎯 City, region & ISP targeting<br>
|
||||
🔐 HTTP, HTTPS & SOCKS5 support<br><br>
|
||||
🎁 Use code <b>OBSCURA35</b> for a <b>35% recurring discount</b>.<br><br></b>
|
||||
Better proxies. Fewer blocks. More scalable automation.
|
||||
</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td width="200" align="center" valign="middle">
|
||||
<a href="https://mangoproxy.com/?utm_source=github&utm_medium=partner&utm_campaign=h4ckf0r0day" target="_blank">
|
||||
<img alt="MangoProxy" src="assets/sponsors/mangoproxy.png" width="180"/>
|
||||
</a>
|
||||
</td>
|
||||
<td valign="middle">
|
||||
<a href="https://mangoproxy.com/?utm_source=github&utm_medium=partner&utm_campaign=h4ckf0r0day"><b>MangoProxy</b></a> provides residential, ISP, datacenter, and mobile proxies in 200+ countries. Trusted by businesses worldwide for stable connections, fast response times, and scalable proxy infrastructure.<br>
|
||||
Use Promo code <b>OBSCURA</b> for 8% off Static ISP Proxies.
|
||||
</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td width="200" align="center" valign="middle">
|
||||
<a href="https://9proxy.com/?utm_source=Github&utm_campaign=obscura" target="_blank">
|
||||
<img alt="9Proxy" src="assets/sponsors/9proxy.png" width="180"/>
|
||||
</a>
|
||||
</td>
|
||||
<td valign="middle">
|
||||
<a href="https://9proxy.com/?utm_source=Github&utm_campaign=obscura"><b>9Proxy</b></a> provides residential proxies from
|
||||
just $0.018/IP or $0.68/GB. 20M+ IPs across 90+ countries. Sticky or rotating sessions, managed from desktop or mobile app.
|
||||
</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td width="200" align="center" valign="middle">
|
||||
<a href="https://go.nodemaven.com/obscura" target="_blank">
|
||||
<img alt="NodeMaven" src="assets/sponsors/nodemaven.svg" width="180"/>
|
||||
</a>
|
||||
</td>
|
||||
<td valign="middle">
|
||||
<a href="https://go.nodemaven.com/obscura"><b>NodeMaven</b></a> — the most reliable proxy provider with the highest quality IPs on the market. Built for automation, web scraping, SEO research, and social media management.<br><br>
|
||||
<b>99.9% uptime<br>
|
||||
Sticky sessions up to 7 days<br>
|
||||
IP filtering on every proxy<br>
|
||||
No KYC required<br>
|
||||
Cashback on traffic — earn up to 10% back<br><br></b>
|
||||
🎁 Use code <b>OBSCURA35</b> for 35% off Mobile & Residential, or <b>OBSCURA40</b> for 40% off ISP (Static) proxies.
|
||||
</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td width="200" align="center" valign="middle">
|
||||
<a href="https://www.rapidproxy.io/?ref=obscura" target="_blank">
|
||||
<img alt="Rapidproxy" src="assets/sponsors/rapidproxy.png" width="180"/>
|
||||
</a>
|
||||
</td>
|
||||
<td valign="middle">
|
||||
<a href="https://www.rapidproxy.io/?ref=obscura"><b>Rapidproxy</b></a> — A residential proxy platform with 90M+ real IPs across 200+ countries. It supports rotation, geo-targeting, and high concurrency to improve scraping success. Start your free trial today!<br><br>
|
||||
<b>Flexible pricing starting from $0.65/GB<br>
|
||||
Traffic that never expires<br>
|
||||
Supports HTTP / HTTPS / SOCKS5 protocols<br>
|
||||
High-speed, low-latency network built for scale<br><br>
|
||||
🎁 Use discount code <b>RAPID10</b> to get <b>10% off</b>.<br><br></b>
|
||||
</td>
|
||||
</tr>
|
||||
</table>
|
||||
|
||||
## Install
|
||||
|
||||
### Download
|
||||
|
||||
Grab the latest binary from [Releases](https://github.com/h4ckf0r0day/obscura/releases):
|
||||
|
||||
```bash
|
||||
# Linux x86_64
|
||||
curl -LO https://github.com/h4ckf0r0day/obscura/releases/latest/download/obscura-x86_64-linux.tar.gz
|
||||
tar xzf obscura-x86_64-linux.tar.gz
|
||||
./obscura fetch https://example.com --eval "document.title"
|
||||
|
||||
# Linux ARM64 (aarch64)
|
||||
curl -LO https://github.com/h4ckf0r0day/obscura/releases/latest/download/obscura-aarch64-linux.tar.gz
|
||||
tar xzf obscura-aarch64-linux.tar.gz
|
||||
|
||||
# Arch Linux (AUR)
|
||||
yay -S obscura-browser
|
||||
|
||||
# macOS Apple Silicon
|
||||
curl -LO https://github.com/h4ckf0r0day/obscura/releases/latest/download/obscura-aarch64-macos.tar.gz
|
||||
tar xzf obscura-aarch64-macos.tar.gz
|
||||
|
||||
# macOS Intel
|
||||
curl -LO https://github.com/h4ckf0r0day/obscura/releases/latest/download/obscura-x86_64-macos.tar.gz
|
||||
tar xzf obscura-x86_64-macos.tar.gz
|
||||
|
||||
# Windows
|
||||
Download the `.zip` from the releases page and extract it manually.
|
||||
```
|
||||
|
||||
No Chrome, no Node.js, no dependencies. Release archives include both
|
||||
`obscura` and `obscura-worker`; keep them in the same directory for the
|
||||
parallel `scrape` command.
|
||||
|
||||
Linux release builds target Ubuntu 22.04 so the downloaded binary remains
|
||||
usable on common LTS servers with glibc 2.35+.
|
||||
|
||||
### Docker
|
||||
|
||||
```bash
|
||||
docker run -d --name obscura -p 127.0.0.1:9222:9222 h4ckf0r0day/obscura
|
||||
```
|
||||
|
||||
Image on [Docker Hub](https://hub.docker.com/r/h4ckf0r0day/obscura). Multi-stage build on `distroless/cc`, no shell, no package manager, ~57 MB compressed.
|
||||
|
||||
### Build from source
|
||||
|
||||
```bash
|
||||
git clone https://github.com/h4ckf0r0day/obscura.git
|
||||
cd obscura
|
||||
cargo build --release
|
||||
|
||||
# With stealth mode (anti-detection + tracker blocking)
|
||||
cargo build --release --features stealth
|
||||
```
|
||||
|
||||
Requires Rust 1.75+ ([rustup.rs](https://rustup.rs)). First build takes ~5 min (V8 compiles from source, cached after).
|
||||
|
||||
## Quick Start
|
||||
|
||||
### Fetch a page
|
||||
|
||||
```bash
|
||||
# Get the page title
|
||||
obscura fetch https://example.com --eval "document.title"
|
||||
|
||||
# Extract all links
|
||||
obscura fetch https://example.com --dump links
|
||||
|
||||
# Render JavaScript and dump HTML
|
||||
obscura fetch https://news.ycombinator.com --dump html
|
||||
|
||||
# Write dump or eval output to a file
|
||||
obscura fetch https://example.com --dump text --output page.txt
|
||||
|
||||
# Stream the raw response body verbatim (binary-safe; bypasses the JS/DOM layer).
|
||||
# Use this for images, JSON, JS, CSS, or any non-HTML resource.
|
||||
obscura fetch https://picsum.photos/200/300 --dump original > photo.jpg
|
||||
|
||||
# List every sub-resource URL the page would fetch (NDJSON; one record per asset)
|
||||
obscura fetch https://example.com --dump assets
|
||||
|
||||
# Fetch through an HTTP or SOCKS proxy
|
||||
obscura --proxy socks5://127.0.0.1:1080 fetch https://example.com --dump text
|
||||
|
||||
# Wait for dynamic content
|
||||
obscura fetch https://example.com --wait-until networkidle0
|
||||
|
||||
# Bound navigation time for slow or broken pages
|
||||
obscura fetch https://example.com --timeout 10
|
||||
```
|
||||
|
||||
### Start the CDP server
|
||||
|
||||
```bash
|
||||
obscura serve --port 9222
|
||||
|
||||
# With stealth mode (anti-detection + tracker blocking)
|
||||
obscura serve --port 9222 --stealth
|
||||
```
|
||||
|
||||
### Scrape in parallel
|
||||
|
||||
```bash
|
||||
obscura scrape url1 url2 url3 ... \
|
||||
--concurrency 25 \
|
||||
--eval "document.querySelector('h1').textContent" \
|
||||
--format json
|
||||
|
||||
# Suppress scrape progress on stderr for script-friendly output
|
||||
obscura scrape https://example.com --quiet --format json
|
||||
|
||||
# Scrape workers inherit the global proxy
|
||||
obscura --proxy http://127.0.0.1:8080 scrape https://example.com https://news.ycombinator.com
|
||||
```
|
||||
|
||||
## Puppeteer / Playwright
|
||||
|
||||
### Puppeteer
|
||||
|
||||
```bash
|
||||
npm install puppeteer-core
|
||||
```
|
||||
|
||||
```javascript
|
||||
import puppeteer from 'puppeteer-core';
|
||||
|
||||
const browser = await puppeteer.connect({
|
||||
browserWSEndpoint: 'ws://127.0.0.1:9222/devtools/browser',
|
||||
});
|
||||
|
||||
const page = await browser.newPage();
|
||||
await page.goto('https://news.ycombinator.com');
|
||||
|
||||
const stories = await page.evaluate(() =>
|
||||
Array.from(document.querySelectorAll('.titleline > a'))
|
||||
.map(a => ({ title: a.textContent, url: a.href }))
|
||||
);
|
||||
console.log(stories);
|
||||
|
||||
await browser.disconnect();
|
||||
```
|
||||
|
||||
### Playwright
|
||||
|
||||
```bash
|
||||
npm install playwright-core
|
||||
```
|
||||
|
||||
```javascript
|
||||
import { chromium } from 'playwright-core';
|
||||
|
||||
const browser = await chromium.connectOverCDP({
|
||||
endpointURL: 'ws://127.0.0.1:9222',
|
||||
});
|
||||
|
||||
const page = await browser.newContext().then(ctx => ctx.newPage());
|
||||
await page.goto('https://en.wikipedia.org/wiki/Web_scraping');
|
||||
console.log(await page.title());
|
||||
|
||||
await browser.close();
|
||||
```
|
||||
|
||||
### Form submission & login
|
||||
|
||||
```javascript
|
||||
await page.goto('https://quotes.toscrape.com/login');
|
||||
await page.evaluate(() => {
|
||||
document.querySelector('#username').value = 'admin';
|
||||
document.querySelector('#password').value = 'admin';
|
||||
document.querySelector('form').submit();
|
||||
});
|
||||
// Obscura handles the POST, follows the 302 redirect, maintains cookies
|
||||
```
|
||||
|
||||
## Benchmarks
|
||||
|
||||
Page load:
|
||||
|
||||
| Page | Obscura | Chrome |
|
||||
|------|---------|--------|
|
||||
| Static HTML | **51 ms** | ~500 ms |
|
||||
| JS + XHR + fetch | **84 ms** | ~800 ms |
|
||||
| Dynamic scripts | **78 ms** | ~700 ms |
|
||||
|
||||
The full benchmark suite (WPT conformance, obstacle course, real-world corpus, and vs-Chrome speed) lives in a separate repo: https://github.com/h4ckf0r0day/obscura-benchmark
|
||||
|
||||
## Stealth Mode
|
||||
|
||||
Enable with `--features stealth`.
|
||||
|
||||
### Anti-fingerprinting
|
||||
- Per-session fingerprint randomization (GPU, screen, canvas, audio, battery)
|
||||
- Realistic `navigator.userAgentData` (Chrome 145, high-entropy values)
|
||||
- `event.isTrusted = true` for dispatched events
|
||||
- Hidden internal properties (`Object.keys(window)` safe)
|
||||
- Native function masking (`Function.prototype.toString()` → `[native code]`)
|
||||
- `navigator.webdriver = undefined` (matches real Chrome)
|
||||
|
||||
### Tracker Blocking
|
||||
- 3,520 domains blocked
|
||||
- Blocks analytics, ads, telemetry, and fingerprinting scripts
|
||||
- Prevents trackers from loading entirely
|
||||
- Enabled automatically with `--stealth`
|
||||
|
||||
## CDP API
|
||||
|
||||
Obscura implements the Chrome DevTools Protocol for Puppeteer/Playwright compatibility.
|
||||
|
||||
| Domain | Methods |
|
||||
|--------|---------|
|
||||
| **Target** | createTarget, closeTarget, attachToTarget, createBrowserContext, disposeBrowserContext |
|
||||
| **Page** | navigate, getFrameTree, addScriptToEvaluateOnNewDocument, lifecycleEvents |
|
||||
| **Runtime** | evaluate, callFunctionOn, getProperties, addBinding |
|
||||
| **DOM** | getDocument, querySelector, querySelectorAll, getOuterHTML, resolveNode |
|
||||
| **Network** | enable, setCookies, getCookies, setExtraHTTPHeaders, setUserAgentOverride |
|
||||
| **Fetch** | enable, continueRequest, fulfillRequest, failRequest (live interception), takeResponseBodyAsStream |
|
||||
| **IO** | read, close (stream a large response body in chunks) |
|
||||
| **Storage** | getCookies, setCookies, deleteCookies |
|
||||
| **Input** | dispatchMouseEvent, dispatchKeyEvent |
|
||||
| **LP** | getMarkdown (DOM-to-Markdown conversion) |
|
||||
|
||||
To download a large resource without one giant `Network.getResponseBody` blob, call `Fetch.takeResponseBodyAsStream` then read it in chunks with `IO.read` / `IO.close`. Response bodies over the cache limit (`OBSCURA_NETWORK_BODY_BUFFER_BYTES`, default 2 MiB) are not retained, so raise that limit when you intend to stream large downloads.
|
||||
## CLI Reference
|
||||
|
||||
### Tuning V8
|
||||
|
||||
Obscura embeds V8 directly. Use `--v8-flags` to pass raw flags through to V8, same syntax as Chromium's `--js-flags` and Node's command-line flags. Most common use is raising the heap cap to fix `JavaScript heap out of memory` on JS-heavy pages:
|
||||
|
||||
```bash
|
||||
obscura --v8-flags "--max-old-space-size=4096" fetch <url>
|
||||
```
|
||||
|
||||
### Heavy SPAs (script execution budget)
|
||||
|
||||
Obscura caps the page's script-execution phase so one slow or hung page cannot stall a worker. The default budget is 30s; pages that finish sooner return immediately, so the cap only affects pages that keep running. A very heavy React/Vue/Angular SPA on a slow network can need more time to boot before it fires its data requests. Raise the budget with `OBSCURA_SCRIPT_DEADLINE_MS` (milliseconds), and pair it with a matching navigation timeout in your CDP client:
|
||||
|
||||
```bash
|
||||
OBSCURA_SCRIPT_DEADLINE_MS=60000 obscura serve --port 9222
|
||||
```
|
||||
|
||||
### `obscura serve`
|
||||
|
||||
Start a CDP WebSocket server.
|
||||
|
||||
| Flag | Default | Description |
|
||||
|------|---------|-------------|
|
||||
| `--port` | `9222` | WebSocket port |
|
||||
| `--proxy` | — | HTTP/SOCKS5 proxy URL |
|
||||
| `--stealth` | off | Enable anti-detection + tracker blocking |
|
||||
| `--workers` | `1` | Number of parallel worker processes |
|
||||
| `--obey-robots` | off | Respect robots.txt |
|
||||
|
||||
### `obscura fetch <URL>`
|
||||
|
||||
Fetch and render a single page.
|
||||
|
||||
| Flag | Default | Description |
|
||||
|------|---------|-------------|
|
||||
| `--dump` | `html` | Output: `html`, `text`, `links`, `markdown`, `assets` (NDJSON of every sub-resource URL the page references), or `original` (raw response body) |
|
||||
| `--eval` | — | JavaScript expression to evaluate |
|
||||
| `--wait-until` | `load` | Wait: `load`, `domcontentloaded`, `networkidle0` |
|
||||
| `--timeout` | `30` | Maximum navigation time in seconds |
|
||||
| `--selector` | — | Wait for CSS selector |
|
||||
| `--stealth` | off | Anti-detection mode |
|
||||
| `--output` | — | Write dump or eval output to a file |
|
||||
| `--quiet` | off | Suppress banner |
|
||||
| `--proxy` | — | Inherited global HTTP/SOCKS5 proxy URL |
|
||||
|
||||
### `obscura scrape <URL...>`
|
||||
|
||||
Scrape multiple URLs in parallel with worker processes.
|
||||
|
||||
| Flag | Default | Description |
|
||||
|------|---------|-------------|
|
||||
| `--concurrency` | `10` | Parallel workers |
|
||||
| `--eval` | — | JS expression per page |
|
||||
| `--format` | `json` | Output: `json` or `text` |
|
||||
| `--quiet` | off | Suppress scrape progress on stderr |
|
||||
| `--proxy` | — | Inherited global HTTP/SOCKS5 proxy URL for all workers |
|
||||
|
||||
## MCP (Model Context Protocol)
|
||||
|
||||
Obscura ships an MCP server that exposes browser automation tools to AI agents (Claude Desktop, Cursor, etc.).
|
||||
|
||||
### Start
|
||||
|
||||
**stdio** (default) — for Claude Desktop and MCP clients that launch a subprocess:
|
||||
|
||||
```bash
|
||||
obscura mcp
|
||||
```
|
||||
|
||||
**HTTP** — for clients that connect over the network:
|
||||
|
||||
```bash
|
||||
obscura mcp --http --port 8080
|
||||
# endpoint: http://127.0.0.1:8080/mcp
|
||||
```
|
||||
|
||||
Optional flags (both transports):
|
||||
|
||||
| Flag | Description |
|
||||
|------|-------------|
|
||||
| `--proxy <URL>` | HTTP/SOCKS5 proxy |
|
||||
| `--user-agent <UA>` | Custom User-Agent string |
|
||||
| `--stealth` | Enable anti-detection mode |
|
||||
|
||||
### Claude Desktop config
|
||||
|
||||
```json
|
||||
{
|
||||
"mcpServers": {
|
||||
"obscura": {
|
||||
"command": "obscura",
|
||||
"args": ["mcp"]
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### Tools
|
||||
|
||||
| Tool | Description |
|
||||
|------|-------------|
|
||||
| `browser_navigate` | Navigate to a URL (`url`, optional `waitUntil`: `load` / `domcontentloaded` / `networkidle0`) |
|
||||
| `browser_snapshot` | Return the current page URL, title, and body text |
|
||||
| `browser_click` | Click an element by CSS selector |
|
||||
| `browser_fill` | Set an input value (triggers `input` + `change` events) |
|
||||
| `browser_type` | Append text to an input |
|
||||
| `browser_press_key` | Dispatch a keyboard event (`key`, optional `selector`) |
|
||||
| `browser_select_option` | Select an `<option>` by value or text |
|
||||
| `browser_evaluate` | Evaluate a JavaScript expression and return the result |
|
||||
| `browser_wait_for` | Wait for a CSS selector to appear (`selector`, optional `timeout` in seconds) |
|
||||
| `browser_network_requests` | List network requests made by the current page |
|
||||
| `browser_console_messages` | Return console messages logged by the page |
|
||||
| `browser_close` | Close the page and reset browser state |
|
||||
|
||||
## Integrations
|
||||
|
||||
- **[Hermes agent plugin](https://github.com/SGavrl/hermes-plugin-obscura)**: run [Hermes](https://github.com/NousResearch/hermes-agent) agent browser tasks on Obscura. The plugin spawns `obscura serve` per session (or connects to an already running server) and drives it over CDP, with optional `--stealth`.
|
||||
|
||||
## License
|
||||
|
||||
Apache 2.0
|
||||
|
||||
---
|
||||
@@ -0,0 +1,7 @@
|
||||
# WeHub 来源说明
|
||||
|
||||
- 原始项目：`h4ckf0r0day/obscura`
|
||||
- 原始仓库：https://github.com/h4ckf0r0day/obscura
|
||||
- 导入方式：上游默认分支的最新快照
|
||||
- 原作者、版权和许可证信息以原始仓库及本仓库 LICENSE 为准
|
||||
- 本文件仅用于记录来源，不代表 WeHub 是原项目作者
|
||||
+109
@@ -0,0 +1,109 @@
|
||||
# Security Policy
|
||||
|
||||
The Obscura project takes security seriously. Obscura runs real, untrusted
|
||||
JavaScript from arbitrary web pages through V8, so we appreciate your efforts to
|
||||
responsibly disclose what you find and will work with you to address it.
|
||||
|
||||
## Reporting a vulnerability
|
||||
|
||||
**Please do not report security issues through public GitHub issues, pull
|
||||
requests, or discussions.**
|
||||
|
||||
Report a vulnerability privately through GitHub: go to the repository's
|
||||
**Security** tab and click [**Report a vulnerability**](https://github.com/h4ckf0r0day/obscura/security/advisories/new)
|
||||
to open a private advisory.
|
||||
|
||||
If you cannot use GitHub advisories, email **hello@obscura.sh** with "security"
|
||||
in the subject line.
|
||||
|
||||
### What to include
|
||||
|
||||
To help us triage quickly, please provide as much of the following as you can:
|
||||
|
||||
- Type of issue (SSRF, hang or crash / denial of service, memory safety,
|
||||
cross-session data exposure, etc.).
|
||||
- The obscura version or commit, plus OS and architecture.
|
||||
- The location of the affected code (file path and commit, or a direct link).
|
||||
- Any special configuration or flags required to reproduce.
|
||||
- Step-by-step instructions: a page, `--eval` snippet, or CDP sequence that
|
||||
triggers it.
|
||||
- Proof-of-concept code if you have it, and the impact you think it has.
|
||||
|
||||
## Our response
|
||||
|
||||
We will send a response indicating the next steps in handling your report,
|
||||
normally within 3 business days. We will keep you informed of progress toward a
|
||||
fix, and may ask for additional information. We practice coordinated disclosure:
|
||||
please give us a reasonable window to ship a fix before any public writeup, and
|
||||
we will credit you in the advisory unless you ask us not to.
|
||||
|
||||
If you do not receive an acknowledgement within 6 business days, please follow
|
||||
up by email to make sure we received the report.
|
||||
|
||||
## Scope
|
||||
|
||||
Obscura is a powerful tool for browser automation, scraping, and inspection. It
|
||||
is the responsibility of the calling code and the operator to use it safely. The
|
||||
security boundaries Obscura is meant to hold, and which are in scope:
|
||||
|
||||
- **Egress / SSRF control.** Page content reaching loopback, RFC 1918, or
|
||||
link-local addresses without `--allow-private-network` being set.
|
||||
- **Availability.** A page, script, or DOM structure that defeats the V8
|
||||
termination watchdog, the CLI hard deadline, or the panic guards and so hangs
|
||||
or aborts the process.
|
||||
- **Memory safety** in Obscura's own `unsafe` Rust or in an op that bridges JS
|
||||
to Rust.
|
||||
- **Process and data integrity.** A page that escapes the intended op surface,
|
||||
corrupts another page's state, or reads data across origins or sessions it
|
||||
should not (cookies, storage, response bodies).
|
||||
- **TLS / identity correctness** issues that weaken or misrepresent a connection
|
||||
in a way the user did not request.
|
||||
|
||||
The following are **not** vulnerabilities. We welcome feedback on them as
|
||||
feature requests, but they will not be treated as security issues:
|
||||
|
||||
- **Stealth / anti-fingerprinting behavior.** Presenting a normal, consistent
|
||||
browser fingerprint is the intended, privacy-first design of stealth mode.
|
||||
Requests to add detection-evasion for abusive purposes are out of scope.
|
||||
- **Anything behind an explicit opt-in,** such as reaching a private address
|
||||
when `--allow-private-network` (or `OBSCURA_ALLOW_PRIVATE_NETWORK=1`) is set,
|
||||
or behavior that requires local access to the machine running Obscura.
|
||||
- **Resource use from a page you chose to load** that stays within the watchdog
|
||||
and deadline limits. Slow pages are not a vulnerability.
|
||||
- **Findings against the companion benchmark repo fixtures** rather than the
|
||||
engine itself.
|
||||
|
||||
## Third-party dependencies
|
||||
|
||||
Report vulnerabilities in third-party crates to their respective maintainers (or
|
||||
the [RustSec advisory database](https://rustsec.org/)). The workspace is gated
|
||||
by `cargo deny` via `deny.toml`. If a dependency advisory affects Obscura's own
|
||||
behavior, let us know so we can pin or patch.
|
||||
|
||||
## Security model and operator responsibilities
|
||||
|
||||
Obscura executes untrusted page JavaScript **in process** through V8. Its
|
||||
in-process hardening reduces blast radius but is not a substitute for operating
|
||||
system isolation:
|
||||
|
||||
- The **V8 termination watchdog** terminates the isolate from a separate thread
|
||||
when synchronous script work overruns, because `tokio` timeouts only cancel at
|
||||
await points.
|
||||
- The **CLI process-level hard deadline** is an absolute backstop for a hang
|
||||
inside a Rust op that neither `tokio` nor `terminate_execution` can interrupt.
|
||||
- **Panic safety:** ops are wrapped so a panic degrades to a null result instead
|
||||
of aborting the process inside V8's FFI frame; `panic = "unwind"` is pinned in
|
||||
the release profile.
|
||||
- The **default network egress policy** blocks private and loopback ranges.
|
||||
|
||||
These measures protect availability and limit egress. They do **not** claim to
|
||||
contain a hostile page that achieves native code execution through a V8 exploit.
|
||||
If you run Obscura against untrusted or adversarial input at scale, run it under
|
||||
OS-level isolation (a container or VM) with a restricted network, the same way
|
||||
you would run headless Chrome. Per-user container isolation is planned for the
|
||||
hosted service so that one session cannot affect another.
|
||||
|
||||
## Supported versions
|
||||
|
||||
Security fixes land on `main` and ship in the next release. Please test reports
|
||||
against the latest release or `main`.
|
||||
Binary file not shown.
|
After Width: | Height: | Size: 33 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 14 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 2.2 MiB |
Binary file not shown.
|
After Width: | Height: | Size: 18 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 498 KiB |
File diff suppressed because one or more lines are too long
|
After Width: | Height: | Size: 29 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 8.6 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 16 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 328 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 7.6 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 2.9 KiB |
@@ -0,0 +1,21 @@
|
||||
[package]
|
||||
name = "obscura-browser"
|
||||
version.workspace = true
|
||||
edition.workspace = true
|
||||
|
||||
[features]
|
||||
default = []
|
||||
stealth = ["obscura-net/stealth", "obscura-js/stealth"]
|
||||
|
||||
[dependencies]
|
||||
obscura-dom = { workspace = true }
|
||||
obscura-net = { workspace = true }
|
||||
obscura-js = { workspace = true }
|
||||
tokio = { workspace = true }
|
||||
futures = "0.3"
|
||||
url = { workspace = true }
|
||||
tracing = { workspace = true }
|
||||
thiserror = { workspace = true }
|
||||
anyhow = { workspace = true }
|
||||
serde_json = { workspace = true }
|
||||
base64 = { workspace = true }
|
||||
@@ -0,0 +1,207 @@
|
||||
use std::path::PathBuf;
|
||||
use std::sync::Arc;
|
||||
|
||||
use obscura_net::{CookieJar, ObscuraHttpClient, RobotsCache};
|
||||
|
||||
pub struct BrowserContext {
|
||||
pub id: String,
|
||||
pub cookie_jar: Arc<CookieJar>,
|
||||
pub http_client: Arc<ObscuraHttpClient>,
|
||||
pub user_agent: String,
|
||||
pub platform: String,
|
||||
pub ua_platform: String,
|
||||
pub ua_platform_version: String,
|
||||
pub proxy_url: Option<String>,
|
||||
pub robots_cache: Arc<RobotsCache>,
|
||||
pub obey_robots: bool,
|
||||
pub stealth: bool,
|
||||
/// When true, CDP-driven navigation to file:// URLs is permitted.
|
||||
/// Default is false: a remote CDP client cannot point the browser
|
||||
/// at /etc/shadow even if Obscura is running as a privileged user.
|
||||
/// Flip on via `obscura serve --allow-file-access` for legitimate
|
||||
/// local-HTML testing workflows. The CLI's own `obscura fetch
|
||||
/// file://...` path is unaffected because it does not go through
|
||||
/// the CDP server.
|
||||
pub allow_file_access: bool,
|
||||
pub storage_dir: Option<PathBuf>,
|
||||
/// When true, the http client allows fetching localhost / RFC1918 /
|
||||
/// link-local addresses. Set via `--allow-private-network` (issue #33).
|
||||
/// Independent of `allow_file_access` because they cover different threat
|
||||
/// models: file:// is a local file-system read, while private-network is
|
||||
/// the broader SSRF gate from issue #4.
|
||||
pub allow_private_network: bool,
|
||||
}
|
||||
|
||||
impl BrowserContext {
|
||||
pub fn new(id: String) -> Self {
|
||||
Self::_new_inner(id, None, false, None, None, false)
|
||||
}
|
||||
|
||||
/// Create a BrowserContext with an optional storage directory.
|
||||
/// When `storage_dir` is set, cookies are automatically loaded from
|
||||
/// `{storage_dir}/cookies.json` on creation.
|
||||
pub fn with_storage(
|
||||
id: String,
|
||||
storage_dir: Option<PathBuf>,
|
||||
) -> Self {
|
||||
Self::_new_inner(id, None, false, None, storage_dir, false)
|
||||
}
|
||||
|
||||
/// Create a BrowserContext with full options including storage_dir.
|
||||
pub fn with_storage_full(
|
||||
id: String,
|
||||
proxy_url: Option<String>,
|
||||
stealth: bool,
|
||||
user_agent: Option<String>,
|
||||
storage_dir: Option<PathBuf>,
|
||||
) -> Self {
|
||||
Self::_new_inner(id, proxy_url, stealth, user_agent, storage_dir, false)
|
||||
}
|
||||
|
||||
/// Variant that also accepts the `allow_private_network` opt-in. All
|
||||
/// pre-existing constructors default it to `false`; callers that want the
|
||||
/// CLI's `--allow-private-network` (issue #33) behaviour go through here.
|
||||
pub fn with_storage_and_network(
|
||||
id: String,
|
||||
proxy_url: Option<String>,
|
||||
stealth: bool,
|
||||
user_agent: Option<String>,
|
||||
storage_dir: Option<PathBuf>,
|
||||
allow_private_network: bool,
|
||||
) -> Self {
|
||||
Self::_new_inner(id, proxy_url, stealth, user_agent, storage_dir, allow_private_network)
|
||||
}
|
||||
|
||||
fn _new_inner(
|
||||
id: String,
|
||||
proxy_url: Option<String>,
|
||||
stealth: bool,
|
||||
user_agent: Option<String>,
|
||||
storage_dir: Option<PathBuf>,
|
||||
allow_private_network: bool,
|
||||
) -> Self {
|
||||
let cookie_jar = Arc::new(CookieJar::new());
|
||||
|
||||
// Restore cookies from disk if storage_dir is configured
|
||||
if let Some(ref dir) = storage_dir {
|
||||
let cookie_path = dir.join("cookies.json");
|
||||
if cookie_path.exists() {
|
||||
match cookie_jar.load_from_file(&cookie_path) {
|
||||
Ok(n) if n > 0 => {
|
||||
tracing::info!("Loaded {} cookies from {}", n, cookie_path.display());
|
||||
}
|
||||
Ok(_) => {}
|
||||
Err(e) => {
|
||||
tracing::warn!("Failed to load cookies from {}: {}", cookie_path.display(), e);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let mut client = ObscuraHttpClient::with_full_options(
|
||||
cookie_jar.clone(),
|
||||
proxy_url.as_deref(),
|
||||
allow_private_network,
|
||||
);
|
||||
if stealth {
|
||||
client.block_trackers = true;
|
||||
}
|
||||
let profile = crate::profiles::select_profile();
|
||||
let resolved_ua = user_agent.unwrap_or_else(|| profile.user_agent.to_string());
|
||||
let platform = profile.platform.to_string();
|
||||
let ua_platform = profile.ua_platform.to_string();
|
||||
let ua_platform_version = profile.ua_platform_version.to_string();
|
||||
// Sync the http client's UA at construction so navigation requests pick it
|
||||
// up before any async setup runs. The lock has no other holders here, so
|
||||
// try_write always succeeds; we fall back silently if it ever fails.
|
||||
if let Ok(mut guard) = client.user_agent.try_write() {
|
||||
*guard = resolved_ua.clone();
|
||||
}
|
||||
let http_client = Arc::new(client);
|
||||
BrowserContext {
|
||||
id,
|
||||
cookie_jar,
|
||||
http_client,
|
||||
user_agent: resolved_ua,
|
||||
platform,
|
||||
ua_platform,
|
||||
ua_platform_version,
|
||||
proxy_url,
|
||||
robots_cache: Arc::new(RobotsCache::new()),
|
||||
obey_robots: false,
|
||||
stealth,
|
||||
allow_file_access: false,
|
||||
storage_dir,
|
||||
allow_private_network,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn with_options(id: String, proxy_url: Option<String>, stealth: bool) -> Self {
|
||||
Self::with_full_options(id, proxy_url, stealth, None)
|
||||
}
|
||||
|
||||
pub fn with_full_options(
|
||||
id: String,
|
||||
proxy_url: Option<String>,
|
||||
stealth: bool,
|
||||
user_agent: Option<String>,
|
||||
) -> Self {
|
||||
Self::_new_inner(id, proxy_url, stealth, user_agent, None, false)
|
||||
}
|
||||
|
||||
pub fn with_proxy(id: String, proxy_url: Option<String>) -> Self {
|
||||
Self::with_options(id, proxy_url, false)
|
||||
}
|
||||
|
||||
/// Persist cookies to disk if storage_dir is configured.
|
||||
/// Called during graceful shutdown.
|
||||
pub fn save_cookies(&self) {
|
||||
if let Some(ref dir) = self.storage_dir {
|
||||
let _ = std::fs::create_dir_all(dir);
|
||||
let cookie_path = dir.join("cookies.json");
|
||||
if let Err(e) = self.cookie_jar.save_to_file(&cookie_path) {
|
||||
tracing::warn!("Failed to save cookies to {}: {}", cookie_path.display(), e);
|
||||
} else {
|
||||
tracing::info!("Saved cookies to {}", cookie_path.display());
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[tokio::test(flavor = "current_thread")]
|
||||
async fn with_full_options_propagates_user_agent_to_http_client() {
|
||||
let ctx = BrowserContext::with_full_options(
|
||||
"test".to_string(),
|
||||
None,
|
||||
false,
|
||||
Some("Custom-UA/1.0".to_string()),
|
||||
);
|
||||
assert_eq!(ctx.user_agent, "Custom-UA/1.0");
|
||||
let client_ua = ctx.http_client.user_agent.read().await.clone();
|
||||
assert_eq!(client_ua, "Custom-UA/1.0");
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "current_thread")]
|
||||
async fn with_full_options_falls_back_to_chrome_default() {
|
||||
let ctx = BrowserContext::with_full_options(
|
||||
"test".to_string(),
|
||||
None,
|
||||
false,
|
||||
None,
|
||||
);
|
||||
assert!(ctx.user_agent.contains("Chrome"));
|
||||
let client_ua = ctx.http_client.user_agent.read().await.clone();
|
||||
assert!(client_ua.contains("Chrome"));
|
||||
assert_eq!(ctx.user_agent, client_ua);
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "current_thread")]
|
||||
async fn with_options_keeps_default_user_agent() {
|
||||
let ctx = BrowserContext::with_options("test".to_string(), None, false);
|
||||
assert!(ctx.user_agent.contains("Chrome"));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,12 @@
|
||||
pub mod page;
|
||||
pub mod context;
|
||||
pub mod lifecycle;
|
||||
pub mod profiles;
|
||||
|
||||
pub use page::{NetworkEvent, Page, PageError};
|
||||
pub use context::BrowserContext;
|
||||
pub use lifecycle::{LifecycleState, WaitUntil};
|
||||
pub use obscura_js::HTML_TO_MARKDOWN_JS;
|
||||
// Re-exported so the embeddable `obscura` crate (which depends on obscura-browser,
|
||||
// not obscura-js) can surface the interception channel types.
|
||||
pub use obscura_js::ops::{InterceptResolution, InterceptedRequest};
|
||||
@@ -0,0 +1,42 @@
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum LifecycleState {
|
||||
Idle,
|
||||
Loading,
|
||||
DomContentLoaded,
|
||||
Loaded,
|
||||
NetworkIdle,
|
||||
Failed,
|
||||
}
|
||||
|
||||
impl LifecycleState {
|
||||
pub fn is_loading(&self) -> bool {
|
||||
matches!(self, LifecycleState::Loading)
|
||||
}
|
||||
|
||||
pub fn is_loaded(&self) -> bool {
|
||||
matches!(self, LifecycleState::Loaded | LifecycleState::NetworkIdle)
|
||||
}
|
||||
|
||||
pub fn is_network_idle(&self) -> bool {
|
||||
matches!(self, LifecycleState::NetworkIdle)
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum WaitUntil {
|
||||
Load,
|
||||
DomContentLoaded,
|
||||
NetworkIdle0,
|
||||
NetworkIdle2,
|
||||
}
|
||||
|
||||
impl WaitUntil {
|
||||
pub fn from_str(s: &str) -> Self {
|
||||
match s {
|
||||
"domcontentloaded" => WaitUntil::DomContentLoaded,
|
||||
"networkidle0" | "networkIdle" | "networkidle" => WaitUntil::NetworkIdle0,
|
||||
"networkidle2" => WaitUntil::NetworkIdle2,
|
||||
_ => WaitUntil::Load,
|
||||
}
|
||||
}
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,103 @@
|
||||
pub struct BrowserProfile {
|
||||
pub user_agent: &'static str,
|
||||
pub platform: &'static str,
|
||||
pub ua_platform: &'static str,
|
||||
pub ua_platform_version: &'static str,
|
||||
}
|
||||
|
||||
pub static PROFILES: &[BrowserProfile] = &[
|
||||
BrowserProfile {
|
||||
user_agent: "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/143.0.0.0 Safari/537.36",
|
||||
platform: "Win32",
|
||||
ua_platform: "Windows",
|
||||
ua_platform_version: "10.0.0",
|
||||
},
|
||||
BrowserProfile {
|
||||
user_agent: "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/144.0.0.0 Safari/537.36",
|
||||
platform: "Win32",
|
||||
ua_platform: "Windows",
|
||||
ua_platform_version: "10.0.0",
|
||||
},
|
||||
BrowserProfile {
|
||||
user_agent: "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/145.0.0.0 Safari/537.36",
|
||||
platform: "Win32",
|
||||
ua_platform: "Windows",
|
||||
ua_platform_version: "15.0.0",
|
||||
},
|
||||
BrowserProfile {
|
||||
user_agent: "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/146.0.0.0 Safari/537.36",
|
||||
platform: "Win32",
|
||||
ua_platform: "Windows",
|
||||
ua_platform_version: "15.0.0",
|
||||
},
|
||||
BrowserProfile {
|
||||
user_agent: "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/143.0.0.0 Safari/537.36",
|
||||
platform: "MacIntel",
|
||||
ua_platform: "macOS",
|
||||
ua_platform_version: "13.6.7",
|
||||
},
|
||||
BrowserProfile {
|
||||
user_agent: "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/144.0.0.0 Safari/537.36",
|
||||
platform: "MacIntel",
|
||||
ua_platform: "macOS",
|
||||
ua_platform_version: "14.4.1",
|
||||
},
|
||||
BrowserProfile {
|
||||
user_agent: "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/145.0.0.0 Safari/537.36",
|
||||
platform: "MacIntel",
|
||||
ua_platform: "macOS",
|
||||
ua_platform_version: "14.5.0",
|
||||
},
|
||||
BrowserProfile {
|
||||
user_agent: "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/146.0.0.0 Safari/537.36",
|
||||
platform: "MacIntel",
|
||||
ua_platform: "macOS",
|
||||
ua_platform_version: "14.6.0",
|
||||
},
|
||||
];
|
||||
|
||||
pub fn random_profile() -> &'static BrowserProfile {
|
||||
let idx = (std::time::SystemTime::now()
|
||||
.duration_since(std::time::UNIX_EPOCH)
|
||||
.unwrap_or_default()
|
||||
.subsec_nanos() as usize)
|
||||
% PROFILES.len();
|
||||
&PROFILES[idx]
|
||||
}
|
||||
|
||||
/// Pick the profile for a new browser context.
|
||||
///
|
||||
/// The default is a single stable profile. Cycling through different browser
|
||||
/// identities from one address is itself a bot signal (a real address maps to a
|
||||
/// stable device), and the rotated profile does not yet carry a matching TLS or
|
||||
/// timezone fingerprint, so rotation is opt-in:
|
||||
/// OBSCURA_PROFILE=<index> pin a specific profile from PROFILES
|
||||
/// OBSCURA_ROTATE_PROFILE=1 pick a random profile per context
|
||||
pub fn select_profile() -> &'static BrowserProfile {
|
||||
if let Some(idx) = std::env::var("OBSCURA_PROFILE")
|
||||
.ok()
|
||||
.as_deref()
|
||||
.map(str::trim)
|
||||
.and_then(|s| s.parse::<usize>().ok())
|
||||
{
|
||||
if idx < PROFILES.len() {
|
||||
return &PROFILES[idx];
|
||||
}
|
||||
}
|
||||
if env_enabled("OBSCURA_ROTATE_PROFILE") {
|
||||
return random_profile();
|
||||
}
|
||||
&PROFILES[0]
|
||||
}
|
||||
|
||||
fn env_enabled(key: &str) -> bool {
|
||||
matches!(
|
||||
std::env::var(key)
|
||||
.ok()
|
||||
.as_deref()
|
||||
.map(str::trim)
|
||||
.map(str::to_ascii_lowercase)
|
||||
.as_deref(),
|
||||
Some("1") | Some("true") | Some("yes") | Some("on")
|
||||
)
|
||||
}
|
||||
@@ -0,0 +1,25 @@
|
||||
[package]
|
||||
name = "obscura-cdp"
|
||||
version.workspace = true
|
||||
edition.workspace = true
|
||||
|
||||
[dependencies]
|
||||
obscura-dom = { workspace = true }
|
||||
obscura-browser = { workspace = true }
|
||||
obscura-js = { workspace = true }
|
||||
tokio = { workspace = true }
|
||||
tokio-tungstenite = { workspace = true }
|
||||
serde = { workspace = true }
|
||||
serde_json = { workspace = true }
|
||||
url = { workspace = true }
|
||||
uuid = { workspace = true }
|
||||
tracing = { workspace = true }
|
||||
thiserror = { workspace = true }
|
||||
anyhow = { workspace = true }
|
||||
futures-util = "0.3"
|
||||
base64 = "0.22"
|
||||
obscura-net = { workspace = true }
|
||||
|
||||
[dev-dependencies]
|
||||
tokio-tungstenite = { workspace = true }
|
||||
futures-util = "0.3"
|
||||
@@ -0,0 +1,162 @@
|
||||
use obscura_net::CookieInfo;
|
||||
use serde_json::Value;
|
||||
|
||||
const DEFAULT_COOKIE_PATH: &str = "/";
|
||||
|
||||
pub fn parse_cdp_cookie(value: &Value) -> Option<CookieInfo> {
|
||||
let name = value.get("name").and_then(|v| v.as_str())?.to_string();
|
||||
let cookie_value = value
|
||||
.get("value")
|
||||
.and_then(|v| v.as_str())
|
||||
.unwrap_or("")
|
||||
.to_string();
|
||||
|
||||
let url_parsed = value
|
||||
.get("url")
|
||||
.and_then(|v| v.as_str())
|
||||
.and_then(|u| url::Url::parse(u).ok());
|
||||
|
||||
let domain = value
|
||||
.get("domain")
|
||||
.and_then(|v| v.as_str())
|
||||
.map(|s| s.to_string())
|
||||
.or_else(|| url_parsed.as_ref().and_then(|u| u.host_str().map(|h| h.to_string())))
|
||||
.unwrap_or_default();
|
||||
|
||||
if domain.is_empty() {
|
||||
return None;
|
||||
}
|
||||
|
||||
let path = value
|
||||
.get("path")
|
||||
.and_then(|v| v.as_str())
|
||||
.map(|s| s.to_string())
|
||||
.or_else(|| url_parsed.as_ref().map(|u| u.path().to_string()))
|
||||
.unwrap_or_else(|| DEFAULT_COOKIE_PATH.to_string());
|
||||
|
||||
let secure = value.get("secure").and_then(|v| v.as_bool()).unwrap_or(false);
|
||||
let http_only = value.get("httpOnly").and_then(|v| v.as_bool()).unwrap_or(false);
|
||||
let same_site = value
|
||||
.get("sameSite")
|
||||
.and_then(|v| v.as_str())
|
||||
.unwrap_or("")
|
||||
.to_string();
|
||||
let expires = value.get("expires").and_then(|v| v.as_f64()).map(|f| f as i64);
|
||||
|
||||
Some(CookieInfo {
|
||||
name,
|
||||
value: cookie_value,
|
||||
domain,
|
||||
path,
|
||||
secure,
|
||||
http_only,
|
||||
same_site,
|
||||
expires,
|
||||
})
|
||||
}
|
||||
|
||||
pub struct DeleteCookiesFilter {
|
||||
pub name: String,
|
||||
pub domain: String,
|
||||
pub path: Option<String>,
|
||||
}
|
||||
|
||||
pub fn parse_delete_cookies_params(params: &Value) -> Option<DeleteCookiesFilter> {
|
||||
let name = params.get("name").and_then(|v| v.as_str())?.to_string();
|
||||
if name.is_empty() {
|
||||
return None;
|
||||
}
|
||||
|
||||
let url_parsed = params
|
||||
.get("url")
|
||||
.and_then(|v| v.as_str())
|
||||
.and_then(|u| url::Url::parse(u).ok());
|
||||
|
||||
let domain = params
|
||||
.get("domain")
|
||||
.and_then(|v| v.as_str())
|
||||
.map(|s| s.to_string())
|
||||
.or_else(|| url_parsed.as_ref().and_then(|u| u.host_str().map(|h| h.to_string())))
|
||||
.unwrap_or_default();
|
||||
|
||||
let path = params
|
||||
.get("path")
|
||||
.and_then(|v| v.as_str())
|
||||
.map(|s| s.to_string())
|
||||
.or_else(|| url_parsed.as_ref().map(|u| u.path().to_string()));
|
||||
|
||||
Some(DeleteCookiesFilter { name, domain, path })
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use serde_json::json;
|
||||
|
||||
#[test]
|
||||
fn parse_with_explicit_domain_path() {
|
||||
let v = json!({
|
||||
"name": "session",
|
||||
"value": "abc",
|
||||
"domain": ".example.com",
|
||||
"path": "/app",
|
||||
"secure": true,
|
||||
"httpOnly": true,
|
||||
"sameSite": "Strict",
|
||||
"expires": 1_900_000_000.0,
|
||||
});
|
||||
let c = parse_cdp_cookie(&v).unwrap();
|
||||
assert_eq!(c.name, "session");
|
||||
assert_eq!(c.value, "abc");
|
||||
assert_eq!(c.domain, ".example.com");
|
||||
assert_eq!(c.path, "/app");
|
||||
assert!(c.secure);
|
||||
assert!(c.http_only);
|
||||
assert_eq!(c.same_site, "Strict");
|
||||
assert_eq!(c.expires, Some(1_900_000_000));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parse_with_url_fallback_for_domain_and_path() {
|
||||
let v = json!({
|
||||
"name": "tok",
|
||||
"value": "xyz",
|
||||
"url": "https://api.example.com/v1/things",
|
||||
});
|
||||
let c = parse_cdp_cookie(&v).unwrap();
|
||||
assert_eq!(c.domain, "api.example.com");
|
||||
assert_eq!(c.path, "/v1/things");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parse_rejects_when_no_domain_or_url() {
|
||||
let v = json!({ "name": "x", "value": "y" });
|
||||
assert!(parse_cdp_cookie(&v).is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parse_default_path_when_url_has_no_path() {
|
||||
let v = json!({
|
||||
"name": "tok",
|
||||
"value": "v",
|
||||
"domain": "example.com",
|
||||
});
|
||||
let c = parse_cdp_cookie(&v).unwrap();
|
||||
assert_eq!(c.path, "/");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn delete_filter_requires_name() {
|
||||
let v = json!({ "domain": "example.com" });
|
||||
assert!(parse_delete_cookies_params(&v).is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn delete_filter_uses_url_for_domain_and_path() {
|
||||
let v = json!({ "name": "session", "url": "https://example.com/admin" });
|
||||
let f = parse_delete_cookies_params(&v).unwrap();
|
||||
assert_eq!(f.name, "session");
|
||||
assert_eq!(f.domain, "example.com");
|
||||
assert_eq!(f.path.as_deref(), Some("/admin"));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,575 @@
|
||||
use std::collections::{HashMap, HashSet};
|
||||
use std::sync::Arc;
|
||||
|
||||
use obscura_browser::{BrowserContext, Page};
|
||||
use obscura_js::ops::InterceptedRequest;
|
||||
use serde_json::json;
|
||||
|
||||
use crate::domains;
|
||||
use crate::domains::fetch::FetchInterceptState;
|
||||
use crate::types::{CdpEvent, CdpRequest, CdpResponse};
|
||||
|
||||
pub struct CdpContext {
|
||||
pub pages: Vec<Page>,
|
||||
pub sessions: HashMap<String, String>, // session_id -> page_id
|
||||
pub pending_events: Vec<CdpEvent>,
|
||||
pub default_context: Arc<BrowserContext>,
|
||||
page_counter: u32,
|
||||
pub preload_scripts: Vec<(String, String)>, // (identifier, source)
|
||||
pub preload_counter: u32,
|
||||
// World names registered via Page.createIsolatedWorld. After every
|
||||
// navigation Obscura clears execution contexts (via
|
||||
// Runtime.executionContextsCleared) and must re-emit a
|
||||
// Runtime.executionContextCreated for each registered world, otherwise
|
||||
// Playwright/Puppeteer hang waiting for their utility world to come
|
||||
// back. Stored as plain Strings (not by-page) — for now we only model
|
||||
// a single page in CdpContext anyway.
|
||||
pub isolated_worlds: Vec<String>,
|
||||
// Set of executionContextIds Obscura has emitted via
|
||||
// Runtime.executionContextCreated. Pre-populated with the default-frame
|
||||
// contexts (`1`, `2`) that Runtime.enable / Page.navigate emit, then
|
||||
// extended each time Page.createIsolatedWorld assigns a fresh id.
|
||||
//
|
||||
// Runtime.evaluate / Runtime.callFunctionOn consult this set to reject
|
||||
// requests targeting an unknown context — matching real Chrome's
|
||||
// "Cannot find context with specified id" CDP error and unblocking the
|
||||
// Playwright locator path described in issue #51.
|
||||
pub valid_context_ids: HashSet<i64>,
|
||||
// Monotonic counter for isolated-world execution context ids. Issue #192:
|
||||
// both `Page.createIsolatedWorld` and `do_navigate` used to hardcode id
|
||||
// 100, so re-creating a context (e.g. Playwright opening a second page or
|
||||
// re-navigating) silently emitted the same id twice and the client's
|
||||
// bookkeeping diverged from the server's. Each fresh isolated context now
|
||||
// claims and increments from this counter so the ids real Chrome would
|
||||
// emit (incrementing, never reused) are mirrored.
|
||||
pub next_isolated_context_id: i64,
|
||||
pub fetch_intercept: FetchInterceptState,
|
||||
pub intercept_tx: Option<tokio::sync::mpsc::UnboundedSender<InterceptedRequest>>,
|
||||
// Open IO streams for Fetch.takeResponseBodyAsStream. Each holds a response
|
||||
// body taken out of the page cache so a large download is streamed
|
||||
// chunk-by-chunk via IO.read and freed on IO.close (issue #360). The store
|
||||
// caps how many bodies (and how many bytes) can be held at once, evicting
|
||||
// the oldest, so an abandoned or disconnected stream cannot leak unbounded.
|
||||
pub io_streams: crate::domains::io::IoStreamStore,
|
||||
}
|
||||
|
||||
impl CdpContext {
|
||||
pub fn new() -> Self {
|
||||
Self::new_with_options(None, false)
|
||||
}
|
||||
|
||||
pub fn new_with_proxy(proxy: Option<String>) -> Self {
|
||||
Self::new_with_options(proxy, false)
|
||||
}
|
||||
|
||||
pub fn new_with_options(proxy: Option<String>, stealth: bool) -> Self {
|
||||
Self::new_with_full_options(proxy, stealth, None)
|
||||
}
|
||||
|
||||
pub fn new_with_full_options(
|
||||
proxy: Option<String>,
|
||||
stealth: bool,
|
||||
user_agent: Option<String>,
|
||||
) -> Self {
|
||||
Self::new_with_security(proxy, stealth, user_agent, false)
|
||||
}
|
||||
|
||||
pub fn new_with_storage(
|
||||
proxy: Option<String>,
|
||||
stealth: bool,
|
||||
user_agent: Option<String>,
|
||||
storage_dir: Option<std::path::PathBuf>,
|
||||
) -> Self {
|
||||
Self::_new_inner(proxy, stealth, user_agent, storage_dir, false, false)
|
||||
}
|
||||
|
||||
pub fn new_with_security(
|
||||
proxy: Option<String>,
|
||||
stealth: bool,
|
||||
user_agent: Option<String>,
|
||||
allow_file_access: bool,
|
||||
) -> Self {
|
||||
Self::_new_inner(proxy, stealth, user_agent, None, allow_file_access, false)
|
||||
}
|
||||
|
||||
/// Kitchen-sink constructor that also threads `allow_private_network`
|
||||
/// (issue #33). Older `new_with_*` builders stay as-is.
|
||||
pub fn new_full(
|
||||
proxy: Option<String>,
|
||||
stealth: bool,
|
||||
user_agent: Option<String>,
|
||||
storage_dir: Option<std::path::PathBuf>,
|
||||
allow_file_access: bool,
|
||||
allow_private_network: bool,
|
||||
) -> Self {
|
||||
Self::_new_inner(
|
||||
proxy, stealth, user_agent, storage_dir, allow_file_access, allow_private_network,
|
||||
)
|
||||
}
|
||||
|
||||
fn _new_inner(
|
||||
proxy: Option<String>,
|
||||
stealth: bool,
|
||||
user_agent: Option<String>,
|
||||
storage_dir: Option<std::path::PathBuf>,
|
||||
allow_file_access: bool,
|
||||
allow_private_network: bool,
|
||||
) -> Self {
|
||||
let mut ctx = BrowserContext::with_storage_and_network(
|
||||
"default".to_string(),
|
||||
proxy,
|
||||
stealth,
|
||||
user_agent,
|
||||
storage_dir,
|
||||
allow_private_network,
|
||||
);
|
||||
ctx.allow_file_access = allow_file_access;
|
||||
let default_context = Arc::new(ctx);
|
||||
// Pre-seed with the default-frame execution context ids that
|
||||
// `Runtime.enable` (1) and post-navigation re-emission (2) advertise
|
||||
// via Runtime.executionContextCreated. Anything else has to be
|
||||
// registered explicitly (Page.createIsolatedWorld), otherwise
|
||||
// Runtime.{evaluate,callFunctionOn} should reject it per CDP spec.
|
||||
let mut valid_context_ids = HashSet::new();
|
||||
valid_context_ids.insert(1);
|
||||
valid_context_ids.insert(2);
|
||||
|
||||
CdpContext {
|
||||
pages: Vec::new(),
|
||||
sessions: HashMap::new(),
|
||||
pending_events: Vec::new(),
|
||||
default_context,
|
||||
page_counter: 0,
|
||||
preload_scripts: Vec::new(),
|
||||
preload_counter: 0,
|
||||
fetch_intercept: FetchInterceptState::new(),
|
||||
intercept_tx: None,
|
||||
isolated_worlds: Vec::new(),
|
||||
valid_context_ids,
|
||||
next_isolated_context_id: 100,
|
||||
io_streams: crate::domains::io::IoStreamStore::default(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Claim the next isolated-world execution context id and register it as
|
||||
/// valid for `Runtime.evaluate`/`callFunctionOn`. Issue #192.
|
||||
pub fn next_isolated_context(&mut self) -> i64 {
|
||||
let id = self.next_isolated_context_id;
|
||||
self.next_isolated_context_id += 1;
|
||||
self.valid_context_ids.insert(id);
|
||||
id
|
||||
}
|
||||
|
||||
pub fn create_page(&mut self) -> String {
|
||||
self.page_counter += 1;
|
||||
let page_id = format!("page-{}", self.page_counter);
|
||||
let mut page = Page::new(page_id.clone(), self.default_context.clone());
|
||||
page.navigate_blank();
|
||||
self.pages.push(page);
|
||||
page_id
|
||||
}
|
||||
|
||||
pub fn get_page(&self, id: &str) -> Option<&Page> {
|
||||
self.pages.iter().find(|p| p.id == id)
|
||||
}
|
||||
|
||||
pub fn get_page_mut(&mut self, id: &str) -> Option<&mut Page> {
|
||||
self.pages.iter_mut().find(|p| p.id == id)
|
||||
}
|
||||
|
||||
pub fn remove_page(&mut self, id: &str) {
|
||||
self.pages.retain(|p| p.id != id);
|
||||
self.sessions.retain(|_, v| v != id);
|
||||
}
|
||||
|
||||
pub fn get_session_page(&self, session_id: &Option<String>) -> Option<&Page> {
|
||||
let page_id = session_id
|
||||
.as_ref()
|
||||
.and_then(|sid| self.sessions.get(sid))?;
|
||||
self.get_page(page_id)
|
||||
}
|
||||
|
||||
pub fn get_session_page_mut(&mut self, session_id: &Option<String>) -> Option<&mut Page> {
|
||||
let page_id = session_id
|
||||
.as_ref()
|
||||
.and_then(|sid| self.sessions.get(sid))
|
||||
.cloned()?;
|
||||
|
||||
let target_has_js = self.pages.iter().any(|p| p.id == page_id && p.has_js());
|
||||
|
||||
if !target_has_js {
|
||||
for page in &mut self.pages {
|
||||
if page.id != page_id && page.has_js() {
|
||||
page.suspend_js();
|
||||
break;
|
||||
}
|
||||
}
|
||||
if let Some(target) = self.pages.iter_mut().find(|p| p.id == page_id) {
|
||||
target.resume_js();
|
||||
}
|
||||
}
|
||||
|
||||
self.get_page_mut(&page_id)
|
||||
}
|
||||
}
|
||||
|
||||
/// Whether a CDP method can be served WITHOUT acquiring the global V8 lock.
|
||||
///
|
||||
/// Methods listed here were audited to confirm they do not transitively
|
||||
/// call into a `JsRuntime`. They either don't touch any `Page` at all, or
|
||||
/// use only the immutable `get_session_page` accessor and Rust-side field
|
||||
/// reads. `get_session_page_mut` triggers `suspend_js`/`resume_js` and
|
||||
/// must stay behind the lock.
|
||||
fn is_v8_free_method(method: &str) -> bool {
|
||||
matches!(method,
|
||||
"Target.getTargets" | "Target.setDiscoverTargets"
|
||||
| "Target.attachToTarget" | "Target.attachToBrowserTarget"
|
||||
| "Target.setAutoAttach"
|
||||
| "Target.getBrowserContexts" | "Target.createBrowserContext"
|
||||
| "Target.disposeBrowserContext" | "Target.getTargetInfo"
|
||||
| "Target.detachFromTarget" | "Target.activateTarget"
|
||||
| "Browser.getVersion" | "Browser.close" | "Browser.getWindowForTarget"
|
||||
| "Browser.setDownloadBehavior" | "Browser.getWindowBounds" | "Browser.setWindowBounds"
|
||||
| "Page.enable" | "Page.disable" | "Page.getFrameTree"
|
||||
| "Page.setDownloadBehavior"
|
||||
| "Page.setLifecycleEventsEnabled"
|
||||
| "Page.addScriptToEvaluateOnNewDocument" | "Page.removeScriptToEvaluateOnNewDocument"
|
||||
| "Page.setInterceptFileChooserDialog" | "Page.getNavigationHistory"
|
||||
| "Page.resetNavigationHistory" | "Page.printToPDF"
|
||||
| "Page.captureScreenshot" | "Page.captureSnapshot"
|
||||
| "Page.createIsolatedWorld"
|
||||
| "Runtime.enable" | "Runtime.disable"
|
||||
| "Runtime.runIfWaitingForDebugger" | "Runtime.getExceptionDetails"
|
||||
| "Runtime.discardConsoleEntries"
|
||||
| "Network.enable" | "Network.disable" | "Network.setCacheDisabled"
|
||||
| "Network.setRequestInterception" | "Network.setBlockedURLs"
|
||||
| "Network.setExtraHTTPHeaders"
|
||||
| "Network.setUserAgentOverride"
|
||||
| "Network.getCookies" | "Network.getAllCookies"
|
||||
| "Network.setCookie" | "Network.setCookies"
|
||||
| "Network.deleteCookies" | "Network.clearBrowserCookies"
|
||||
| "Network.getResponseBody"
|
||||
| "Fetch.continueRequest" | "Fetch.fulfillRequest"
|
||||
| "Fetch.failRequest" | "Fetch.getResponseBody"
|
||||
| "Storage.getCookies" | "Storage.setCookies" | "Storage.deleteCookies"
|
||||
)
|
||||
}
|
||||
|
||||
pub async fn dispatch(req: &CdpRequest, ctx: &mut CdpContext) -> CdpResponse {
|
||||
// headless_chrome (and older Puppeteer) wrap every CDP call inside
|
||||
// Target.sendMessageToTarget. Unwrap and recurse BEFORE acquiring the
|
||||
// V8 lock — the recursive dispatch will acquire it for the inner call,
|
||||
// and tokio Mutex is not reentrant.
|
||||
if req.method == "Target.sendMessageToTarget" {
|
||||
return dispatch_send_message_to_target(req, ctx).await;
|
||||
}
|
||||
|
||||
// Issue #19: V8 fatal abort under concurrent CDP work.
|
||||
//
|
||||
// Every CDP handler below may end up calling into a per-Page `JsRuntime`
|
||||
// (each owning its own V8 Isolate). All of them run on a single OS
|
||||
// thread (current_thread tokio + LocalSet). When two pages' V8-touching
|
||||
// futures interleave across an `.await` (which `process_with_interception`
|
||||
// can trigger by handling new CDP messages while a navigation task is in
|
||||
// flight on `spawn_local`), V8 trips the
|
||||
// `heap->isolate() == Isolate::TryGetCurrent()` invariant and aborts the
|
||||
// process via `V8_Fatal` — no Rust panic, just `abort(3)`.
|
||||
//
|
||||
// Holding the process-wide V8 lock around the entire dispatch keeps each
|
||||
// handler contiguous on the thread: V8 fully exits one Isolate before
|
||||
// the next page is allowed in. This converts the abort into latency.
|
||||
// It overshoots — non-V8 handlers (Browser.*, Storage.*, Emulation.*)
|
||||
// also serialize — but those are cheap and the safety win dominates.
|
||||
//
|
||||
// The properly concurrent fix is to pin each `JsRuntime` to its own OS
|
||||
// thread and message-pass; that's tracked as the larger #19 follow-up.
|
||||
//
|
||||
// Optimization: methods that demonstrably never touch V8 bypass the
|
||||
// lock. During Puppeteer's newPage() setup ~8 such calls (Page.enable,
|
||||
// Runtime.enable, Page.getFrameTree, Page.addScriptToEvaluateOnNewDocument,
|
||||
// Page.createIsolatedWorld, etc.) used to serialize behind any
|
||||
// in-flight V8 work on a sibling page. Bypassing keeps the setup chain
|
||||
// running while a separate nav executes. Each listed method was audited
|
||||
// to confirm it never reaches `JsRuntime::execute_script` or DOM
|
||||
// mutation that re-enters V8; `get_session_page_mut` (which can
|
||||
// trigger `suspend_js`/`resume_js`) is NOT in the list.
|
||||
let _v8_guard = if is_v8_free_method(&req.method) {
|
||||
None
|
||||
} else {
|
||||
Some(obscura_js::v8_lock::global().lock().await)
|
||||
};
|
||||
|
||||
// Per-command V8 watchdog. The lock above keeps each handler contiguous on
|
||||
// the thread, but it does not bound how long a handler runs: a hung page (a
|
||||
// runaway Runtime.evaluate, a synchronous DOM op) would hold the
|
||||
// process-wide V8 lock and wedge every other session forever. The one-shot
|
||||
// CLI uses a process-level hard deadline for this; the long-running server
|
||||
// cannot force-exit, so we terminate just the offending isolate instead.
|
||||
// OBSCURA_CDP_COMMAND_TIMEOUT_MS tunes the bound (0 disables); the default
|
||||
// leaves headroom for the slowest legitimate navigation, which already
|
||||
// self-bounds via OBSCURA_NAV_TIMEOUT_MS plus the watchdog-bounded settle.
|
||||
let cmd_budget_ms: u64 = std::env::var("OBSCURA_CDP_COMMAND_TIMEOUT_MS")
|
||||
.ok()
|
||||
.and_then(|s| s.parse().ok())
|
||||
.unwrap_or(60_000);
|
||||
let cmd_watchdog = if cmd_budget_ms == 0 || is_v8_free_method(&req.method) {
|
||||
None
|
||||
} else {
|
||||
ctx.get_session_page(&req.session_id)
|
||||
.and_then(|p| p.isolate_handle())
|
||||
.map(|h| obscura_js::cdp_watchdog::arm(h, std::time::Duration::from_millis(cmd_budget_ms)))
|
||||
};
|
||||
|
||||
let (domain, method) = match req.method.split_once('.') {
|
||||
Some((d, m)) => (d, m),
|
||||
None => {
|
||||
return CdpResponse::error(
|
||||
req.id,
|
||||
-32601,
|
||||
format!("Invalid method format: {}", req.method),
|
||||
req.session_id.clone(),
|
||||
);
|
||||
}
|
||||
};
|
||||
|
||||
let result = match domain {
|
||||
"Target" => domains::target::handle(method, &req.params, ctx).await,
|
||||
"Browser" => domains::browser::handle(method, &req.params).await,
|
||||
"Page" => domains::page::handle(method, &req.params, ctx, &req.session_id).await,
|
||||
"DOM" => domains::dom::handle(method, &req.params, ctx, &req.session_id).await,
|
||||
"DOMSnapshot" => domains::domsnapshot::handle(method, &req.params, ctx, &req.session_id).await,
|
||||
"Runtime" => domains::runtime::handle(method, &req.params, ctx, &req.session_id).await,
|
||||
"Network" => domains::network::handle(method, &req.params, ctx, &req.session_id).await,
|
||||
"Fetch" => domains::fetch::handle(method, &req.params, ctx, &req.session_id).await,
|
||||
"IO" => domains::io::handle(method, &req.params, ctx).await,
|
||||
"Input" => domains::input::handle(method, &req.params, ctx, &req.session_id).await,
|
||||
"Storage" => domains::storage::handle(method, &req.params, ctx, &req.session_id).await,
|
||||
"LP" => domains::lp::handle(method, &req.params, ctx, &req.session_id).await,
|
||||
"Accessibility" => domains::accessibility::handle(method, &req.params, ctx, &req.session_id).await,
|
||||
// Accepted but no-op. Puppeteer's FrameManager.initialize calls
|
||||
// Audits.enable on connect — refusing it breaks puppeteer.connect()
|
||||
// before any user code runs.
|
||||
"Emulation" | "Log" | "Performance" | "Security" | "CSS"
|
||||
| "ServiceWorker" | "Inspector"
|
||||
| "Debugger" | "Profiler" | "HeapProfiler" | "Overlay"
|
||||
| "Audits" => {
|
||||
Ok(json!({}))
|
||||
}
|
||||
_ => Err(format!("Unknown domain: {}", domain)),
|
||||
};
|
||||
|
||||
// Stop the per-command watchdog. If it fired (the handler held V8 past the
|
||||
// budget), V8 is left in a terminating state, so clear that flag before the
|
||||
// next command runs on this page.
|
||||
if let Some(wd) = cmd_watchdog {
|
||||
if obscura_js::cdp_watchdog::disarm(wd) {
|
||||
tracing::warn!(
|
||||
"CDP command {} held V8 past {}ms; terminated the isolate to free the dispatcher",
|
||||
req.method,
|
||||
cmd_budget_ms
|
||||
);
|
||||
if let Some(page) = ctx.get_session_page_mut(&req.session_id) {
|
||||
page.cancel_v8_termination();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
drain_binding_calls(ctx);
|
||||
|
||||
match result {
|
||||
Ok(value) => CdpResponse::success(req.id, value, req.session_id.clone()),
|
||||
Err(msg) => {
|
||||
tracing::warn!("CDP error for {}: {}", req.method, msg);
|
||||
CdpResponse::error(req.id, -32601, msg, req.session_id.clone())
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Drain every page's binding-call queue (filled by op_binding_called when
|
||||
// page JS invokes a `Runtime.addBinding` shim) and turn each entry into a
|
||||
// Runtime.bindingCalled CDP event that the writer task forwards to the
|
||||
// connected client. Called after every dispatch — binding calls only land
|
||||
// in the queue while V8 is running inside a CDP handler, so there is no
|
||||
// window in which they could pile up without a draining opportunity.
|
||||
fn drain_binding_calls(ctx: &mut CdpContext) {
|
||||
// page_id -> session_id (any one session that holds this page).
|
||||
let page_to_session: HashMap<String, String> = ctx
|
||||
.sessions
|
||||
.iter()
|
||||
.map(|(sid, pid)| (pid.clone(), sid.clone()))
|
||||
.collect();
|
||||
|
||||
let mut events: Vec<CdpEvent> = Vec::new();
|
||||
for page in &mut ctx.pages {
|
||||
let calls = page.take_pending_binding_calls();
|
||||
if calls.is_empty() {
|
||||
continue;
|
||||
}
|
||||
let Some(session_id) = page_to_session.get(&page.id).cloned() else {
|
||||
// No session attached — drop the calls; there is no client to
|
||||
// deliver them to.
|
||||
continue;
|
||||
};
|
||||
for (name, payload) in calls {
|
||||
events.push(CdpEvent {
|
||||
method: "Runtime.bindingCalled".into(),
|
||||
// Use executionContextId=2: the default main-frame context
|
||||
// emitted post-navigation (see domains/page.rs phase1).
|
||||
// Puppeteer matches on session_id + binding name and
|
||||
// tolerates any registered context id.
|
||||
params: json!({
|
||||
"name": name,
|
||||
"payload": payload,
|
||||
"executionContextId": 2,
|
||||
}),
|
||||
session_id: Some(session_id.clone()),
|
||||
});
|
||||
}
|
||||
}
|
||||
ctx.pending_events.extend(events);
|
||||
}
|
||||
|
||||
async fn dispatch_send_message_to_target(req: &CdpRequest, ctx: &mut CdpContext) -> CdpResponse {
|
||||
let session_id = req
|
||||
.params
|
||||
.get("sessionId")
|
||||
.and_then(|v| v.as_str())
|
||||
.map(str::to_string);
|
||||
let message = match req.params.get("message").and_then(|v| v.as_str()) {
|
||||
Some(m) => m,
|
||||
None => {
|
||||
return CdpResponse::error(
|
||||
req.id,
|
||||
-32602,
|
||||
"sendMessageToTarget requires a message string".into(),
|
||||
req.session_id.clone(),
|
||||
);
|
||||
}
|
||||
};
|
||||
|
||||
let inner: CdpRequest = match serde_json::from_str(message) {
|
||||
Ok(r) => r,
|
||||
Err(e) => {
|
||||
return CdpResponse::error(
|
||||
req.id,
|
||||
-32700,
|
||||
format!("sendMessageToTarget message is not a valid CDP request: {e}"),
|
||||
req.session_id.clone(),
|
||||
);
|
||||
}
|
||||
};
|
||||
|
||||
// Override the inner session with the one supplied by the wrapper so
|
||||
// the inner dispatch routes against the right page. Boxing the future
|
||||
// sidesteps the async-fn recursion limit.
|
||||
let inner_with_session = CdpRequest {
|
||||
id: inner.id,
|
||||
method: inner.method.clone(),
|
||||
params: inner.params,
|
||||
session_id: session_id.clone().or(inner.session_id),
|
||||
};
|
||||
let inner_response = Box::pin(dispatch(&inner_with_session, ctx)).await;
|
||||
|
||||
// Re-emit the inner response as the legacy event headless_chrome (and
|
||||
// older Puppeteer) listen for instead of correlating responses by id.
|
||||
let inner_serialized = serde_json::to_string(&inner_response).unwrap_or_else(|_| "{}".into());
|
||||
ctx.pending_events.push(CdpEvent {
|
||||
method: "Target.receivedMessageFromTarget".to_string(),
|
||||
params: json!({
|
||||
"sessionId": session_id.clone().unwrap_or_default(),
|
||||
"message": inner_serialized,
|
||||
"targetId": session_id.clone().unwrap_or_default(),
|
||||
}),
|
||||
session_id: req.session_id.clone(),
|
||||
});
|
||||
|
||||
CdpResponse::success(req.id, json!({}), req.session_id.clone())
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::types::CdpRequest;
|
||||
|
||||
fn req(method: &str) -> CdpRequest {
|
||||
CdpRequest {
|
||||
id: 1,
|
||||
method: method.into(),
|
||||
params: json!({}),
|
||||
session_id: None,
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn audits_enable_returns_empty_success() {
|
||||
let mut ctx = CdpContext::new();
|
||||
let resp = dispatch(&req("Audits.enable"), &mut ctx).await;
|
||||
assert!(resp.error.is_none(), "Audits.enable should not error: {:?}", resp.error);
|
||||
assert_eq!(resp.result, Some(json!({})));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn unknown_domain_still_errors() {
|
||||
let mut ctx = CdpContext::new();
|
||||
let resp = dispatch(&req("DefinitelyNotADomain.enable"), &mut ctx).await;
|
||||
let err = resp.error.expect("unknown domain must surface as error");
|
||||
assert_eq!(err.code, -32601);
|
||||
assert!(err.message.contains("Unknown domain"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn send_message_to_target_unwraps_inner_call() {
|
||||
let mut ctx = CdpContext::new();
|
||||
let inner = json!({
|
||||
"id": 42,
|
||||
"method": "Browser.getVersion",
|
||||
"params": {},
|
||||
});
|
||||
let outer = CdpRequest {
|
||||
id: 99,
|
||||
method: "Target.sendMessageToTarget".into(),
|
||||
params: json!({
|
||||
"sessionId": "sess-1",
|
||||
"message": inner.to_string(),
|
||||
}),
|
||||
session_id: None,
|
||||
};
|
||||
|
||||
let resp = dispatch(&outer, &mut ctx).await;
|
||||
assert!(resp.error.is_none(), "wrapper must succeed: {:?}", resp.error);
|
||||
assert_eq!(resp.id, 99);
|
||||
assert_eq!(resp.result, Some(json!({})));
|
||||
|
||||
// headless_chrome reads the inner response from the
|
||||
// receivedMessageFromTarget event, not from the wrapper response.
|
||||
let evt = ctx
|
||||
.pending_events
|
||||
.iter()
|
||||
.find(|e| e.method == "Target.receivedMessageFromTarget")
|
||||
.expect("receivedMessageFromTarget event must be emitted");
|
||||
assert_eq!(evt.params["sessionId"], "sess-1");
|
||||
let inner_msg = evt.params["message"].as_str().expect("message is a string");
|
||||
let parsed: serde_json::Value = serde_json::from_str(inner_msg).unwrap();
|
||||
assert_eq!(parsed["id"], 42);
|
||||
// Browser.getVersion returns a populated result object, not an error.
|
||||
assert!(parsed.get("result").is_some(), "inner response carries result");
|
||||
assert!(parsed.get("error").is_none(), "inner response is not an error");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn send_message_to_target_rejects_invalid_message() {
|
||||
let mut ctx = CdpContext::new();
|
||||
let outer = CdpRequest {
|
||||
id: 5,
|
||||
method: "Target.sendMessageToTarget".into(),
|
||||
params: json!({
|
||||
"sessionId": "sess-1",
|
||||
"message": "{not valid json",
|
||||
}),
|
||||
session_id: None,
|
||||
};
|
||||
let resp = dispatch(&outer, &mut ctx).await;
|
||||
let err = resp.error.expect("malformed inner messages must error");
|
||||
assert_eq!(err.code, -32700);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,433 @@
|
||||
use obscura_dom::{DomTree, NodeData, NodeId};
|
||||
use serde_json::{json, Value};
|
||||
|
||||
use crate::dispatch::CdpContext;
|
||||
|
||||
/// Build a CDP AXValue for a role type.
|
||||
fn ax_value_role(role: &str) -> Value {
|
||||
json!({"type": "role", "value": role})
|
||||
}
|
||||
|
||||
/// Build a CDP AXValue for a string type.
|
||||
fn ax_value_string(s: &str) -> Value {
|
||||
json!({"type": "string", "value": s})
|
||||
}
|
||||
|
||||
/// Build a CDP AXValue for a boolean type.
|
||||
fn ax_value_boolean(b: bool) -> Value {
|
||||
json!({"type": "boolean", "value": b})
|
||||
}
|
||||
|
||||
/// Build a CDP AXValue for an integer type.
|
||||
fn ax_value_integer(i: u32) -> Value {
|
||||
json!({"type": "integer", "value": i})
|
||||
}
|
||||
|
||||
pub async fn handle(
|
||||
method: &str,
|
||||
_params: &Value,
|
||||
ctx: &mut CdpContext,
|
||||
session_id: &Option<String>,
|
||||
) -> Result<Value, String> {
|
||||
match method {
|
||||
"enable" => Ok(json!({})),
|
||||
"getFullAXTree" => {
|
||||
let page = ctx
|
||||
.get_session_page(session_id)
|
||||
.ok_or("No page")?;
|
||||
let nodes = page
|
||||
.with_dom(|dom| build_ax_nodes(dom))
|
||||
.unwrap_or_default();
|
||||
Ok(json!({ "nodes": nodes }))
|
||||
}
|
||||
_ => Ok(json!({})),
|
||||
}
|
||||
}
|
||||
|
||||
/// Walk the full DOM tree and produce CDP Accessibility AXNode array.
|
||||
fn build_ax_nodes(dom: &DomTree) -> Vec<Value> {
|
||||
let mut nodes: Vec<Value> = Vec::new();
|
||||
let mut id_counter: u32 = 0;
|
||||
// Map DOM NodeId → AX string id, populated only for nodes actually in the AX tree
|
||||
let mut dom_to_ax: std::collections::HashMap<u32, String> = std::collections::HashMap::new();
|
||||
|
||||
let document = dom.document();
|
||||
|
||||
// Collect all DOM nodes in tree order (root + descendants)
|
||||
let mut all_dom_ids: Vec<NodeId> = vec![document];
|
||||
all_dom_ids.extend(dom.descendants(document));
|
||||
|
||||
// First pass: assign AX IDs only to nodes that will produce an AX node
|
||||
let mut eligible: Vec<NodeId> = Vec::new();
|
||||
for dom_id in &all_dom_ids {
|
||||
// Quick check without full build_ax_node (just role check to avoid borrow issues)
|
||||
if let Some(node) = dom.get_node(*dom_id) {
|
||||
let role = map_role(&node.data);
|
||||
if !role.is_empty() {
|
||||
id_counter += 1;
|
||||
dom_to_ax.insert(dom_id.raw(), id_counter.to_string());
|
||||
eligible.push(*dom_id);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Second pass: build AXNode for eligible nodes
|
||||
for dom_id in &eligible {
|
||||
if let Some(ax) = build_ax_node(dom, *dom_id, &dom_to_ax) {
|
||||
nodes.push(ax);
|
||||
}
|
||||
}
|
||||
|
||||
nodes
|
||||
}
|
||||
|
||||
fn build_ax_node(
|
||||
dom: &DomTree,
|
||||
node_id: NodeId,
|
||||
dom_to_ax: &std::collections::HashMap<u32, String>,
|
||||
) -> Option<Value> {
|
||||
let node = dom.get_node(node_id)?;
|
||||
let ax_id = dom_to_ax.get(&node_id.raw())?.clone();
|
||||
|
||||
let role = map_role(&node.data);
|
||||
// Skip non-relevant nodes (Document, Doctype, Comment, PI)
|
||||
if role.is_empty() {
|
||||
return None;
|
||||
}
|
||||
|
||||
let name = compute_name(dom, &node);
|
||||
let value = compute_value(dom, &node);
|
||||
let properties = compute_properties(dom, &node);
|
||||
|
||||
let child_ids: Vec<String> = dom
|
||||
.children(node_id)
|
||||
.iter()
|
||||
.filter_map(|child_id| dom_to_ax.get(&child_id.raw()).cloned())
|
||||
.collect();
|
||||
|
||||
// Resolve parentId — walk DOM ancestors until we find one in the AX tree
|
||||
let parent_id: Option<String> = {
|
||||
let mut current = node_id;
|
||||
let mut result = None;
|
||||
loop {
|
||||
let next_parent = dom.with_node(current, |n| n.parent).flatten();
|
||||
match next_parent {
|
||||
Some(pid) => {
|
||||
if let Some(ax_pid) = dom_to_ax.get(&pid.raw()) {
|
||||
result = Some(ax_pid.clone());
|
||||
break;
|
||||
}
|
||||
current = pid;
|
||||
}
|
||||
None => break,
|
||||
}
|
||||
}
|
||||
result
|
||||
};
|
||||
|
||||
// Build node with only non-empty optional fields (per CDP spec, optional fields should be omitted when empty)
|
||||
let mut ax_node = json!({
|
||||
"nodeId": ax_id,
|
||||
"ignored": false,
|
||||
"role": ax_value_role(role),
|
||||
});
|
||||
|
||||
if let Some(ref pid) = parent_id {
|
||||
ax_node.as_object_mut().unwrap().insert("parentId".into(), json!(pid));
|
||||
}
|
||||
if let Some(ref n) = name {
|
||||
ax_node.as_object_mut().unwrap().insert("name".into(), json!(ax_value_string(n)));
|
||||
}
|
||||
if let Some(ref v) = value {
|
||||
ax_node.as_object_mut().unwrap().insert("value".into(), json!(ax_value_string(v)));
|
||||
}
|
||||
if !properties.is_empty() {
|
||||
ax_node.as_object_mut().unwrap().insert("properties".into(), json!(properties));
|
||||
}
|
||||
if !child_ids.is_empty() {
|
||||
ax_node.as_object_mut().unwrap().insert("childIds".into(), json!(child_ids));
|
||||
}
|
||||
ax_node.as_object_mut().unwrap().insert("backendDOMNodeId".into(), json!(node_id.raw()));
|
||||
|
||||
Some(ax_node)
|
||||
}
|
||||
|
||||
/// Map HTML element tag to ARIA role value.
|
||||
fn map_role(data: &NodeData) -> &'static str {
|
||||
match data {
|
||||
NodeData::Document => "RootWebArea",
|
||||
NodeData::Element { name, attrs, .. } => {
|
||||
let tag = name.local.as_ref();
|
||||
|
||||
// Check explicit role attribute first
|
||||
if let Some(role_attr) = attrs.iter().find(|a| a.name.local.as_ref() == "role") {
|
||||
return match role_attr.value.as_str() {
|
||||
"button" => "button",
|
||||
"link" => "link",
|
||||
"heading" => "heading",
|
||||
"textbox" | "searchbox" => "textbox",
|
||||
"checkbox" => "checkbox",
|
||||
"radio" => "radio",
|
||||
"listbox" => "listbox",
|
||||
"combobox" => "combobox",
|
||||
"list" => "list",
|
||||
"listitem" => "listitem",
|
||||
"navigation" => "navigation",
|
||||
"banner" => "banner",
|
||||
"main" => "main",
|
||||
"complementary" => "complementary",
|
||||
"contentinfo" => "contentinfo",
|
||||
"form" => "form",
|
||||
"table" => "table",
|
||||
"row" => "row",
|
||||
"cell" | "gridcell" => "cell",
|
||||
"img" => "image",
|
||||
"dialog" => "dialog",
|
||||
"alert" => "alert",
|
||||
"tab" => "tab",
|
||||
"tablist" => "tablist",
|
||||
"tabpanel" => "tabpanel",
|
||||
"menu" => "menu",
|
||||
"menuitem" => "menuitem",
|
||||
"toolbar" => "toolbar",
|
||||
"separator" => "separator",
|
||||
"presentation" | "none" => {
|
||||
// presentation/none roles get the role but content is still in tree
|
||||
"presentation"
|
||||
}
|
||||
_ => "generic",
|
||||
};
|
||||
}
|
||||
|
||||
match tag {
|
||||
"a" => {
|
||||
if attrs.iter().any(|a| a.name.local.as_ref() == "href") {
|
||||
"link"
|
||||
} else {
|
||||
"generic"
|
||||
}
|
||||
}
|
||||
"button" | "summary" => "button",
|
||||
"input" => {
|
||||
let type_attr = attrs
|
||||
.iter()
|
||||
.find(|a| a.name.local.as_ref() == "type")
|
||||
.map(|a| a.value.as_str())
|
||||
.unwrap_or("text");
|
||||
match type_attr {
|
||||
"submit" | "reset" | "button" | "image" => "button",
|
||||
"checkbox" => "checkbox",
|
||||
"radio" => "radio",
|
||||
"range" => "slider",
|
||||
"number" => "spinbutton",
|
||||
"search" => "searchbox",
|
||||
_ => "textbox",
|
||||
}
|
||||
}
|
||||
"textarea" => "textbox",
|
||||
"select" => {
|
||||
if attrs.iter().any(|a| {
|
||||
a.name.local.as_ref() == "multiple"
|
||||
|| a.name.local.as_ref() == "size"
|
||||
}) {
|
||||
"listbox"
|
||||
} else {
|
||||
"combobox"
|
||||
}
|
||||
}
|
||||
"h1" | "h2" | "h3" | "h4" | "h5" | "h6" => "heading",
|
||||
"img" | "svg" => "image",
|
||||
"ul" | "ol" | "menu" => "list",
|
||||
"li" => "listitem",
|
||||
"table" => "table",
|
||||
"tr" => "row",
|
||||
"td" | "th" => "cell",
|
||||
"nav" => "navigation",
|
||||
"header" => "banner",
|
||||
"main" => "main",
|
||||
"footer" => "contentinfo",
|
||||
"form" => "form",
|
||||
"dialog" => "dialog",
|
||||
"hr" => "separator",
|
||||
"label" => "LabelText",
|
||||
"article" => "article",
|
||||
"aside" => "complementary",
|
||||
"section" => "region",
|
||||
"figure" => "figure",
|
||||
"figcaption" => "StaticText",
|
||||
"p" | "div" | "span" | "pre" | "blockquote" | "code"
|
||||
| "em" | "strong" | "b" | "i" | "u" | "s" | "small"
|
||||
| "sub" | "sup" | "mark" | "del" | "ins" => "generic",
|
||||
"iframe" => "Iframe",
|
||||
_ => "generic",
|
||||
}
|
||||
}
|
||||
NodeData::Text { .. } => "StaticText",
|
||||
NodeData::Doctype { .. } | NodeData::Comment { .. } | NodeData::ProcessingInstruction { .. } => {
|
||||
""
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Compute the accessible name for a node.
|
||||
fn compute_name(dom: &DomTree, node: &obscura_dom::Node) -> Option<String> {
|
||||
if let NodeData::Element { attrs, .. } = &node.data {
|
||||
// aria-label takes highest priority
|
||||
if let Some(label) = attrs
|
||||
.iter()
|
||||
.find(|a| a.name.local.as_ref() == "aria-label")
|
||||
.map(|a| a.value.clone())
|
||||
{
|
||||
return Some(label);
|
||||
}
|
||||
|
||||
// aria-labelledby
|
||||
if let Some(labelledby) = attrs
|
||||
.iter()
|
||||
.find(|a| a.name.local.as_ref() == "aria-labelledby")
|
||||
{
|
||||
let ids: Vec<&str> = labelledby.value.split_whitespace().collect();
|
||||
let mut name = String::new();
|
||||
for id_str in ids {
|
||||
if let Some(ref_id) = dom.get_element_by_id(id_str) {
|
||||
name.push_str(&dom.text_content(ref_id));
|
||||
name.push(' ');
|
||||
}
|
||||
}
|
||||
let trimmed = name.trim().to_string();
|
||||
if !trimmed.is_empty() {
|
||||
return Some(trimmed);
|
||||
}
|
||||
}
|
||||
|
||||
// alt attribute for images
|
||||
if let Some(alt) = attrs
|
||||
.iter()
|
||||
.find(|a| a.name.local.as_ref() == "alt")
|
||||
.map(|a| a.value.clone())
|
||||
{
|
||||
if !alt.is_empty() {
|
||||
return Some(alt);
|
||||
}
|
||||
}
|
||||
|
||||
// title attribute
|
||||
if let Some(title) = attrs
|
||||
.iter()
|
||||
.find(|a| a.name.local.as_ref() == "title")
|
||||
.map(|a| a.value.clone())
|
||||
{
|
||||
if !title.is_empty() {
|
||||
return Some(title);
|
||||
}
|
||||
}
|
||||
|
||||
// placeholder
|
||||
if let Some(placeholder) = attrs
|
||||
.iter()
|
||||
.find(|a| a.name.local.as_ref() == "placeholder")
|
||||
.map(|a| a.value.clone())
|
||||
{
|
||||
if !placeholder.is_empty() {
|
||||
return Some(placeholder);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// For text nodes, the name is the text content
|
||||
if let NodeData::Text { contents } = &node.data {
|
||||
let trimmed = contents.trim().to_string();
|
||||
if !trimmed.is_empty() {
|
||||
return Some(trimmed);
|
||||
}
|
||||
}
|
||||
|
||||
None
|
||||
}
|
||||
|
||||
/// Compute the accessible value for a node (e.g., current input value).
|
||||
fn compute_value(_dom: &DomTree, node: &obscura_dom::Node) -> Option<String> {
|
||||
if let NodeData::Element { name, attrs, .. } = &node.data {
|
||||
let tag = name.local.as_ref();
|
||||
// For input elements, return the value attribute
|
||||
if tag == "input" || tag == "textarea" || tag == "select" {
|
||||
if let Some(val) = attrs
|
||||
.iter()
|
||||
.find(|a| a.name.local.as_ref() == "value")
|
||||
.map(|a| a.value.clone())
|
||||
{
|
||||
return Some(val);
|
||||
}
|
||||
}
|
||||
}
|
||||
None
|
||||
}
|
||||
|
||||
/// Compute accessibility properties for a node.
|
||||
fn compute_properties(_dom: &DomTree, node: &obscura_dom::Node) -> Vec<Value> {
|
||||
if let NodeData::Element { name, attrs, .. } = &node.data {
|
||||
let tag = name.local.as_ref();
|
||||
let mut props = Vec::new();
|
||||
|
||||
// focusable
|
||||
let focusable = matches!(
|
||||
tag,
|
||||
"a" | "button" | "input" | "select" | "textarea" | "details" | "summary"
|
||||
) || attrs.iter().any(|a| {
|
||||
let an = a.name.local.as_ref();
|
||||
an == "tabindex" || an == "contenteditable"
|
||||
});
|
||||
if focusable {
|
||||
props.push(json!({"name": "focusable", "value": ax_value_boolean(true)}));
|
||||
}
|
||||
|
||||
// editable
|
||||
if tag == "input" || tag == "textarea"
|
||||
|| attrs
|
||||
.iter()
|
||||
.any(|a| a.name.local.as_ref() == "contenteditable" && a.value != "false")
|
||||
{
|
||||
props.push(json!({"name": "editable", "value": ax_value_boolean(true)}));
|
||||
}
|
||||
|
||||
// checked for checkboxes/radios
|
||||
if attrs
|
||||
.iter()
|
||||
.any(|a| a.name.local.as_ref() == "checked")
|
||||
{
|
||||
props.push(json!({"name": "checked", "value": ax_value_boolean(true)}));
|
||||
}
|
||||
|
||||
// disabled
|
||||
if attrs
|
||||
.iter()
|
||||
.any(|a| a.name.local.as_ref() == "disabled")
|
||||
{
|
||||
props.push(json!({"name": "disabled", "value": ax_value_boolean(true)}));
|
||||
}
|
||||
|
||||
// level for headings
|
||||
if let Some(level) = tag.strip_prefix('h').and_then(|s| s.parse::<u32>().ok()) {
|
||||
if level >= 1 && level <= 6 {
|
||||
props.push(json!({"name": "level", "value": ax_value_integer(level)}));
|
||||
}
|
||||
}
|
||||
|
||||
// required
|
||||
if attrs
|
||||
.iter()
|
||||
.any(|a| a.name.local.as_ref() == "required" || a.name.local.as_ref() == "aria-required")
|
||||
{
|
||||
props.push(json!({"name": "required", "value": ax_value_boolean(true)}));
|
||||
}
|
||||
|
||||
// multiline for textarea
|
||||
if tag == "textarea" {
|
||||
props.push(json!({"name": "multiline", "value": ax_value_boolean(true)}));
|
||||
}
|
||||
|
||||
props
|
||||
} else {
|
||||
Vec::new()
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,36 @@
|
||||
use serde_json::{json, Value};
|
||||
|
||||
pub async fn handle(method: &str, _params: &Value) -> Result<Value, String> {
|
||||
match method {
|
||||
"getVersion" => Ok(json!({
|
||||
"protocolVersion": "1.3",
|
||||
"product": "Chrome/145.0.0.0",
|
||||
"revision": "@0000000000000000000000000000000000000000",
|
||||
"userAgent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/145.0.0.0 Safari/537.36",
|
||||
"jsVersion": "14.5.0.0",
|
||||
})),
|
||||
"close" => {
|
||||
Ok(json!({}))
|
||||
}
|
||||
"getWindowForTarget" => Ok(json!({
|
||||
"windowId": 1,
|
||||
"bounds": {
|
||||
"left": 0,
|
||||
"top": 0,
|
||||
"width": 1280,
|
||||
"height": 720,
|
||||
"windowState": "normal",
|
||||
}
|
||||
})),
|
||||
"setDownloadBehavior" => Ok(json!({})),
|
||||
"getWindowBounds" => Ok(json!({
|
||||
"bounds": { "left": 0, "top": 0, "width": 1280, "height": 720, "windowState": "normal" }
|
||||
})),
|
||||
// No-op acks for window-management methods Playwright sends during
|
||||
// page setup. We don't model real OS windows, but answering with {}
|
||||
// lets the client's setup sequence complete instead of tearing down
|
||||
// the page on an unknown-method error.
|
||||
"setWindowBounds" => Ok(json!({})),
|
||||
_ => Err(format!("Unknown Browser method: {}", method)),
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,436 @@
|
||||
use obscura_browser::Page;
|
||||
use obscura_dom::{DomTree, NodeData, NodeId};
|
||||
use serde_json::{json, Value};
|
||||
|
||||
use crate::dispatch::CdpContext;
|
||||
|
||||
/// Resolve a DOM `nodeId` from CDP params. Honors `nodeId`, `backendNodeId`,
|
||||
/// and `objectId` in that order. Playwright commonly passes only `objectId`
|
||||
/// (returned by a prior `DOM.resolveNode`); without this fallback those
|
||||
/// requests silently default to node 0 and click the wrong element.
|
||||
fn resolve_node_id(page: &mut Page, params: &Value) -> Result<u64, String> {
|
||||
if let Some(nid) = params.get("nodeId").and_then(|v| v.as_u64()) {
|
||||
return Ok(nid);
|
||||
}
|
||||
if let Some(nid) = params.get("backendNodeId").and_then(|v| v.as_u64()) {
|
||||
return Ok(nid);
|
||||
}
|
||||
if let Some(oid) = params.get("objectId").and_then(|v| v.as_str()) {
|
||||
let code = format!(
|
||||
"(function() {{ var o = globalThis.__obscura_objects && globalThis.__obscura_objects['{}']; \
|
||||
return (o && typeof o._nid === 'number') ? o._nid : -1; }})()",
|
||||
oid.replace('\'', "\\'")
|
||||
);
|
||||
let result = page.evaluate(&code);
|
||||
let nid = result.as_f64().map(|n| n as i64).unwrap_or(-1);
|
||||
if nid < 0 {
|
||||
return Err(format!("objectId {oid} could not be resolved to a node"));
|
||||
}
|
||||
return Ok(nid as u64);
|
||||
}
|
||||
Err("nodeId, backendNodeId, or objectId required".to_string())
|
||||
}
|
||||
|
||||
/// Standard base64 (with padding). Used to ferry file bytes to the JS layer for
|
||||
/// DOM.setFileInputFiles without pulling in a dependency.
|
||||
fn encode_base64(input: &[u8]) -> String {
|
||||
const T: &[u8; 64] = b"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/";
|
||||
let mut out = String::with_capacity((input.len() + 2) / 3 * 4);
|
||||
for chunk in input.chunks(3) {
|
||||
let b0 = chunk[0] as u32;
|
||||
let b1 = *chunk.get(1).unwrap_or(&0) as u32;
|
||||
let b2 = *chunk.get(2).unwrap_or(&0) as u32;
|
||||
let n = (b0 << 16) | (b1 << 8) | b2;
|
||||
out.push(T[((n >> 18) & 63) as usize] as char);
|
||||
out.push(T[((n >> 12) & 63) as usize] as char);
|
||||
out.push(if chunk.len() > 1 { T[((n >> 6) & 63) as usize] as char } else { '=' });
|
||||
out.push(if chunk.len() > 2 { T[(n & 63) as usize] as char } else { '=' });
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
/// Best-effort MIME type from a file extension, for the File objects created by
|
||||
/// DOM.setFileInputFiles. Defaults to application/octet-stream.
|
||||
fn guess_mime(path: &str) -> &'static str {
|
||||
let ext = std::path::Path::new(path)
|
||||
.extension()
|
||||
.and_then(|e| e.to_str())
|
||||
.unwrap_or("")
|
||||
.to_ascii_lowercase();
|
||||
match ext.as_str() {
|
||||
"jpg" | "jpeg" => "image/jpeg",
|
||||
"png" => "image/png",
|
||||
"gif" => "image/gif",
|
||||
"webp" => "image/webp",
|
||||
"svg" => "image/svg+xml",
|
||||
"bmp" => "image/bmp",
|
||||
"ico" => "image/x-icon",
|
||||
"pdf" => "application/pdf",
|
||||
"txt" => "text/plain",
|
||||
"html" | "htm" => "text/html",
|
||||
"css" => "text/css",
|
||||
"js" | "mjs" => "text/javascript",
|
||||
"json" => "application/json",
|
||||
"xml" => "application/xml",
|
||||
"csv" => "text/csv",
|
||||
"zip" => "application/zip",
|
||||
"gz" => "application/gzip",
|
||||
"mp4" => "video/mp4",
|
||||
"webm" => "video/webm",
|
||||
"mp3" => "audio/mpeg",
|
||||
"wav" => "audio/wav",
|
||||
"doc" => "application/msword",
|
||||
"docx" => "application/vnd.openxmlformats-officedocument.wordprocessingml.document",
|
||||
"xls" => "application/vnd.ms-excel",
|
||||
"xlsx" => "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
|
||||
_ => "application/octet-stream",
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn handle(
|
||||
method: &str,
|
||||
params: &Value,
|
||||
ctx: &mut CdpContext,
|
||||
session_id: &Option<String>,
|
||||
) -> Result<Value, String> {
|
||||
match method {
|
||||
"enable" => Ok(json!({})),
|
||||
"getDocument" => {
|
||||
let page = ctx.get_session_page(session_id).ok_or("No page")?;
|
||||
let depth = params.get("depth").and_then(|v| v.as_i64()).unwrap_or(2);
|
||||
page.with_dom(|dom| {
|
||||
let node = serialize_node(dom, dom.document(), depth as u32, 0);
|
||||
json!({ "root": node })
|
||||
}).ok_or_else(|| "No DOM loaded".to_string())
|
||||
}
|
||||
"querySelector" => {
|
||||
let page = ctx.get_session_page(session_id).ok_or("No page")?;
|
||||
let selector = params.get("selector").and_then(|v| v.as_str()).ok_or("selector required")?;
|
||||
let result = page.with_dom(|dom| {
|
||||
dom.query_selector(selector).ok().flatten().map(|id| id.index()).unwrap_or(0)
|
||||
}).unwrap_or(0);
|
||||
Ok(json!({ "nodeId": result }))
|
||||
}
|
||||
"querySelectorAll" => {
|
||||
let page = ctx.get_session_page(session_id).ok_or("No page")?;
|
||||
let selector = params.get("selector").and_then(|v| v.as_str()).ok_or("selector required")?;
|
||||
let ids = page.with_dom(|dom| {
|
||||
dom.query_selector_all(selector).ok()
|
||||
.map(|ids| ids.iter().map(|id| id.index() as u64).collect::<Vec<_>>())
|
||||
.unwrap_or_default()
|
||||
}).unwrap_or_default();
|
||||
Ok(json!({ "nodeIds": ids }))
|
||||
}
|
||||
"getOuterHTML" => {
|
||||
let page = ctx.get_session_page(session_id).ok_or("No page")?;
|
||||
let node_id = params.get("nodeId").and_then(|v| v.as_u64())
|
||||
.or_else(|| params.get("backendNodeId").and_then(|v| v.as_u64()))
|
||||
.ok_or("nodeId required")?;
|
||||
let html = page.with_dom(|dom| {
|
||||
dom.outer_html(NodeId::new(node_id as u32))
|
||||
}).unwrap_or_default();
|
||||
Ok(json!({ "outerHTML": html }))
|
||||
}
|
||||
"describeNode" => {
|
||||
let page = ctx.get_session_page_mut(session_id).ok_or("No page")?;
|
||||
let depth = params.get("depth").and_then(|v| v.as_i64()).unwrap_or(0);
|
||||
|
||||
let node_id = if let Some(nid) = params.get("nodeId").and_then(|v| v.as_u64())
|
||||
.or_else(|| params.get("backendNodeId").and_then(|v| v.as_u64()))
|
||||
{
|
||||
nid
|
||||
} else if let Some(oid) = params.get("objectId").and_then(|v| v.as_str()) {
|
||||
let escaped_oid = oid.replace('\\', "\\\\").replace('\'', "\\'");
|
||||
let code = format!(
|
||||
"(function() {{ var o = globalThis.__obscura_objects['{}']; if (!o) return -1; return (typeof o._nid === 'number') ? o._nid : -1; }})()",
|
||||
escaped_oid
|
||||
);
|
||||
let result = page.evaluate(&code);
|
||||
result.as_f64().map(|n| n as u64).unwrap_or(0)
|
||||
} else {
|
||||
return Err("nodeId or objectId required".to_string());
|
||||
};
|
||||
|
||||
let node = page.with_dom(|dom| {
|
||||
serialize_node(dom, NodeId::new(node_id as u32), depth as u32, 0)
|
||||
}).unwrap_or(json!(null));
|
||||
Ok(json!({ "node": node }))
|
||||
}
|
||||
"resolveNode" => {
|
||||
let page = ctx.get_session_page_mut(session_id).ok_or("No page")?;
|
||||
let node_id = if let Some(nid) = params.get("nodeId").and_then(|v| v.as_u64())
|
||||
.or_else(|| params.get("backendNodeId").and_then(|v| v.as_u64()))
|
||||
{
|
||||
nid
|
||||
} else if let Some(oid) = params.get("objectId").and_then(|v| v.as_str()) {
|
||||
let code = format!(
|
||||
"(function() {{ var o = globalThis.__obscura_objects['{}']; return (o && typeof o._nid === 'number') ? o._nid : -1; }})()",
|
||||
oid
|
||||
);
|
||||
let result = page.evaluate(&code);
|
||||
result.as_f64().map(|n| n as u64).unwrap_or(0)
|
||||
} else {
|
||||
return Err("nodeId or objectId required".to_string());
|
||||
};
|
||||
|
||||
let js_code = format!(
|
||||
"(function() {{\
|
||||
var nid = {};\
|
||||
var node = null;\
|
||||
if (globalThis._cache && globalThis._cache.has(nid)) {{\
|
||||
node = globalThis._cache.get(nid);\
|
||||
}} else {{\
|
||||
var t = +Deno.core.ops.op_dom('node_type', String(nid), '');\
|
||||
if (t === 1) node = new Element(nid);\
|
||||
else if (t === 9) node = globalThis.document;\
|
||||
else node = new Node(nid);\
|
||||
if (globalThis._cache) globalThis._cache.set(nid, node);\
|
||||
}}\
|
||||
return node;\
|
||||
}})()",
|
||||
node_id,
|
||||
);
|
||||
|
||||
let info = if let Some(js) = &mut page.js {
|
||||
match js.store_object_with_meta(&js_code) {
|
||||
Ok(info) => info,
|
||||
Err(_) => {
|
||||
return Ok(json!({
|
||||
"object": {
|
||||
"type": "object",
|
||||
"subtype": "node",
|
||||
"className": "HTMLElement",
|
||||
"objectId": format!("node-{}", node_id),
|
||||
}
|
||||
}));
|
||||
}
|
||||
}
|
||||
} else {
|
||||
return Err("No JS runtime".to_string());
|
||||
};
|
||||
|
||||
Ok(json!({
|
||||
"object": {
|
||||
"type": "object",
|
||||
"subtype": "node",
|
||||
"className": if info.class_name.is_empty() { "HTMLElement".to_string() } else { info.class_name },
|
||||
"description": info.description,
|
||||
"objectId": info.object_id.unwrap_or_else(|| format!("node-{}", node_id)),
|
||||
}
|
||||
}))
|
||||
}
|
||||
"setAttributeValue" => Ok(json!({})),
|
||||
"removeNode" => Ok(json!({})),
|
||||
"focus" => {
|
||||
// No layout engine, but obscura's JS focus() sets document.activeElement,
|
||||
// which Input.dispatchKeyEvent targets. CDP clients (browser-use) focus an
|
||||
// input via DOM.focus before typing; without this their keystrokes land on
|
||||
// nothing and the field stays empty.
|
||||
let page = ctx.get_session_page_mut(session_id).ok_or("No page")?;
|
||||
let node_id = resolve_node_id(page, params)?;
|
||||
let code = format!(
|
||||
"(function() {{ var el = globalThis._wrap && globalThis._wrap({0}); \
|
||||
if (el && typeof el.focus === 'function') {{ el.focus(); return true; }} return false; }})()",
|
||||
node_id
|
||||
);
|
||||
let _ = page.evaluate(&code);
|
||||
Ok(json!({}))
|
||||
}
|
||||
"setFileInputFiles" => {
|
||||
// Puppeteer's ElementHandle.uploadFile / Playwright's setInputFiles
|
||||
// drive an <input type=file> through this CDP call (issue #359). Read
|
||||
// each local file, then hand its bytes (base64) to the JS layer, which
|
||||
// builds real File objects and fires input+change like a real
|
||||
// selection so page code can read/upload them.
|
||||
let page = ctx.get_session_page_mut(session_id).ok_or("No page")?;
|
||||
let node_id = resolve_node_id(page, params)?;
|
||||
let paths: Vec<String> = params
|
||||
.get("files")
|
||||
.and_then(|v| v.as_array())
|
||||
.map(|a| a.iter().filter_map(|v| v.as_str().map(String::from)).collect())
|
||||
.unwrap_or_default();
|
||||
|
||||
let mut specs = Vec::with_capacity(paths.len());
|
||||
for p in &paths {
|
||||
let bytes = std::fs::read(p)
|
||||
.map_err(|e| format!("setFileInputFiles: cannot read '{}': {}", p, e))?;
|
||||
let name = std::path::Path::new(p)
|
||||
.file_name()
|
||||
.and_then(|n| n.to_str())
|
||||
.unwrap_or("file")
|
||||
.to_string();
|
||||
specs.push(json!({ "name": name, "type": guess_mime(p), "b64": encode_base64(&bytes) }));
|
||||
}
|
||||
|
||||
let specs_json = serde_json::to_string(&specs).unwrap_or_else(|_| "[]".to_string());
|
||||
let code = format!(
|
||||
"(function() {{ var el = globalThis._wrap && globalThis._wrap({0}); \
|
||||
if (el && globalThis.__obscura_setInputFiles) {{ globalThis.__obscura_setInputFiles(el, {1}); return true; }} return false; }})()",
|
||||
node_id, specs_json
|
||||
);
|
||||
let _ = page.evaluate(&code);
|
||||
Ok(json!({}))
|
||||
}
|
||||
"getBoxModel" => {
|
||||
let page = ctx.get_session_page_mut(session_id).ok_or("No page")?;
|
||||
let node_id = match resolve_node_id(page, params) {
|
||||
Ok(nid) => nid,
|
||||
Err(_) => return Ok(json!(null)),
|
||||
};
|
||||
let code = format!(
|
||||
"(function() {{\
|
||||
var el = globalThis._wrap && globalThis._wrap({0});\
|
||||
if (!el || typeof el.getBoundingClientRect !== 'function') return null;\
|
||||
var r = el.getBoundingClientRect();\
|
||||
return [r.left, r.top, r.right, r.top, r.right, r.bottom, r.left, r.bottom,\
|
||||
r.width, r.height];\
|
||||
}})()",
|
||||
node_id
|
||||
);
|
||||
let val = page.evaluate(&code);
|
||||
let (quad, w, h) = if let Some(arr) = val.as_array() {
|
||||
let nums: Vec<f64> = arr.iter().filter_map(|v| v.as_f64()).collect();
|
||||
if nums.len() >= 10 {
|
||||
let q: Vec<Value> = nums[..8].iter().map(|n| json!(n)).collect();
|
||||
(q, nums[8], nums[9])
|
||||
} else {
|
||||
(vec![json!(8),json!(8),json!(108),json!(8),json!(108),json!(28),json!(8),json!(28)], 100.0, 20.0)
|
||||
}
|
||||
} else {
|
||||
(vec![json!(8),json!(8),json!(108),json!(8),json!(108),json!(28),json!(8),json!(28)], 100.0, 20.0)
|
||||
};
|
||||
Ok(json!({
|
||||
"model": {
|
||||
"content": quad.clone(),
|
||||
"padding": quad.clone(),
|
||||
"border": quad.clone(),
|
||||
"margin": quad,
|
||||
"width": w, "height": h,
|
||||
}
|
||||
}))
|
||||
}
|
||||
"getContentQuads" => {
|
||||
let page = ctx.get_session_page_mut(session_id).ok_or("No page")?;
|
||||
let node_id = match resolve_node_id(page, params) {
|
||||
Ok(nid) => nid,
|
||||
Err(_) => return Ok(json!(null)),
|
||||
};
|
||||
let code = format!(
|
||||
"(function() {{\
|
||||
var el = globalThis._wrap && globalThis._wrap({0});\
|
||||
if (!el || typeof el.getBoundingClientRect !== 'function') return null;\
|
||||
var r = el.getBoundingClientRect();\
|
||||
return [r.left, r.top, r.right, r.top, r.right, r.bottom, r.left, r.bottom];\
|
||||
}})()",
|
||||
node_id
|
||||
);
|
||||
let val = page.evaluate(&code);
|
||||
let quad = if let Some(arr) = val.as_array() {
|
||||
let nums: Vec<f64> = arr.iter().filter_map(|v| v.as_f64()).collect();
|
||||
if nums.len() == 8 { nums.iter().map(|n| json!(n)).collect::<Vec<_>>() }
|
||||
else { vec![json!(8),json!(8),json!(108),json!(8),json!(108),json!(28),json!(8),json!(28)] }
|
||||
} else {
|
||||
vec![json!(8),json!(8),json!(108),json!(8),json!(108),json!(28),json!(8),json!(28)]
|
||||
};
|
||||
Ok(json!({ "quads": [quad] }))
|
||||
}
|
||||
_ => Err(format!("Unknown DOM method: {}", method)),
|
||||
}
|
||||
}
|
||||
|
||||
fn serialize_node(dom: &DomTree, node_id: NodeId, max_depth: u32, current_depth: u32) -> Value {
|
||||
let node = match dom.get_node(node_id) {
|
||||
Some(n) => n,
|
||||
None => return json!(null),
|
||||
};
|
||||
let children_ids = dom.children(node_id);
|
||||
let child_count = children_ids.len();
|
||||
let mut result = json!({ "nodeId": node_id.index(), "backendNodeId": node_id.index(), "childNodeCount": child_count });
|
||||
|
||||
match &node.data {
|
||||
NodeData::Document => {
|
||||
result["nodeType"] = json!(9); result["nodeName"] = json!("#document");
|
||||
result["localName"] = json!(""); result["nodeValue"] = json!("");
|
||||
result["documentURL"] = json!(""); result["baseURL"] = json!(""); result["xmlVersion"] = json!("");
|
||||
}
|
||||
NodeData::Doctype { name, public_id, system_id } => {
|
||||
result["nodeType"] = json!(10); result["nodeName"] = json!(name);
|
||||
result["localName"] = json!(""); result["nodeValue"] = json!("");
|
||||
result["publicId"] = json!(public_id); result["systemId"] = json!(system_id);
|
||||
}
|
||||
NodeData::Element { name, attrs, .. } => {
|
||||
result["nodeType"] = json!(1);
|
||||
result["nodeName"] = json!(name.local.as_ref().to_ascii_uppercase());
|
||||
result["localName"] = json!(name.local.as_ref());
|
||||
result["nodeValue"] = json!("");
|
||||
let cdp_attrs: Vec<String> = attrs.iter()
|
||||
.flat_map(|a| vec![a.name.local.to_string(), a.value.clone()]).collect();
|
||||
result["attributes"] = json!(cdp_attrs);
|
||||
}
|
||||
NodeData::Text { contents } => {
|
||||
result["nodeType"] = json!(3); result["nodeName"] = json!("#text");
|
||||
result["localName"] = json!(""); result["nodeValue"] = json!(contents);
|
||||
}
|
||||
NodeData::Comment { contents } => {
|
||||
result["nodeType"] = json!(8); result["nodeName"] = json!("#comment");
|
||||
result["localName"] = json!(""); result["nodeValue"] = json!(contents);
|
||||
}
|
||||
NodeData::ProcessingInstruction { target, data } => {
|
||||
result["nodeType"] = json!(7); result["nodeName"] = json!(target);
|
||||
result["localName"] = json!(""); result["nodeValue"] = json!(data);
|
||||
}
|
||||
}
|
||||
|
||||
if current_depth < max_depth && !children_ids.is_empty() {
|
||||
let children: Vec<Value> = children_ids.iter()
|
||||
.map(|&cid| serialize_node(dom, cid, max_depth, current_depth + 1)).collect();
|
||||
result["children"] = json!(children);
|
||||
}
|
||||
result
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::dispatch::CdpContext;
|
||||
|
||||
#[tokio::test]
|
||||
async fn dom_focus_sets_active_element() {
|
||||
// CDP clients (browser-use) focus an input via DOM.focus before typing;
|
||||
// dispatchKeyEvent then targets document.activeElement. DOM.focus must
|
||||
// actually move focus or keystrokes land on nothing.
|
||||
let mut ctx = CdpContext::new();
|
||||
let page_id = ctx.create_page();
|
||||
let session = Some(format!("{page_id}-session"));
|
||||
ctx.sessions.insert(session.clone().unwrap(), page_id.clone());
|
||||
|
||||
crate::domains::page::handle(
|
||||
"navigate",
|
||||
&json!({ "url": "data:text/html,<input id=q>", "waitUntil": "load" }),
|
||||
&mut ctx,
|
||||
&session,
|
||||
)
|
||||
.await
|
||||
.expect("navigate should succeed");
|
||||
|
||||
let qs = handle("querySelector", &json!({ "selector": "input" }), &mut ctx, &session)
|
||||
.await
|
||||
.expect("querySelector should succeed");
|
||||
let nid = qs["nodeId"].as_u64().expect("input nodeId");
|
||||
assert!(nid > 0, "the input element should be found");
|
||||
|
||||
handle("focus", &json!({ "nodeId": nid }), &mut ctx, &session)
|
||||
.await
|
||||
.expect("DOM.focus should succeed");
|
||||
|
||||
let active = ctx
|
||||
.get_session_page_mut(&session)
|
||||
.unwrap()
|
||||
.evaluate("(function(){return document.activeElement?document.activeElement.tagName:'NONE';})()");
|
||||
assert_eq!(
|
||||
active,
|
||||
json!("INPUT"),
|
||||
"DOM.focus must set document.activeElement to the focused input"
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,416 @@
|
||||
//! `DOMSnapshot.captureSnapshot` for layout-free engines.
|
||||
//!
|
||||
//! browser-use (and other CDP DOM-agent frameworks) build their interactive
|
||||
//! element index from `DOMSnapshot.captureSnapshot`: the per-node bounds,
|
||||
//! computed styles, and `isClickable` flag, correlated with `DOM.getDocument`
|
||||
//! by `backendNodeId`. Without this domain their DOM build aborts and the agent
|
||||
//! sees zero elements.
|
||||
//!
|
||||
//! Obscura has no layout/paint engine, so there is no real geometry to report.
|
||||
//! We synthesize it: every node gets a distinct, on-screen, non-icon-sized box
|
||||
//! (a simple vertical stack) plus plausible computed styles (visible, opaque,
|
||||
//! pointer cursor on interactive tags). That is enough for the element
|
||||
//! detection path, which keys off tag name / ARIA / accessibility role and does
|
||||
//! not need true geometry. Clicking still falls back to JS `.click()` since the
|
||||
//! coordinates are synthetic. `backendNodeId == nid`, matching `DOM.getDocument`.
|
||||
|
||||
use obscura_dom::{DomTree, NodeData, NodeId};
|
||||
use serde_json::{json, Value};
|
||||
|
||||
use crate::dispatch::CdpContext;
|
||||
|
||||
/// Computed-style names browser-use requests, in the exact order it expects to
|
||||
/// read them back out of each layout node's `styles` index array.
|
||||
const REQUIRED_STYLES: &[&str] = &[
|
||||
"display",
|
||||
"visibility",
|
||||
"opacity",
|
||||
"overflow",
|
||||
"overflow-x",
|
||||
"overflow-y",
|
||||
"cursor",
|
||||
"pointer-events",
|
||||
"position",
|
||||
"background-color",
|
||||
];
|
||||
|
||||
/// Cap the synthesized snapshot so a pathologically large DOM cannot produce a
|
||||
/// runaway payload. Matches the spirit of the descendants() length cap.
|
||||
const MAX_NODES: usize = 20_000;
|
||||
|
||||
pub async fn handle(
|
||||
method: &str,
|
||||
_params: &Value,
|
||||
ctx: &mut CdpContext,
|
||||
session_id: &Option<String>,
|
||||
) -> Result<Value, String> {
|
||||
match method {
|
||||
"enable" | "disable" => Ok(json!({})),
|
||||
"captureSnapshot" => {
|
||||
let page = ctx.get_session_page(session_id).ok_or("No page")?;
|
||||
let url = page.url_string();
|
||||
let title = page.title.clone();
|
||||
page.with_dom(|dom| build_capture_snapshot(dom, &url, &title))
|
||||
.ok_or_else(|| "No DOM loaded".to_string())
|
||||
}
|
||||
// Permissive no-op for the rest of the domain (e.g. getSnapshot) so a
|
||||
// client that probes it does not abort on an Unknown-method error.
|
||||
_ => Ok(json!({})),
|
||||
}
|
||||
}
|
||||
|
||||
/// String table with de-duplication. Every string in a DOMSnapshot response is
|
||||
/// referenced by its index into the top-level `strings` array.
|
||||
struct Interner {
|
||||
map: std::collections::HashMap<String, i64>,
|
||||
list: Vec<String>,
|
||||
}
|
||||
|
||||
impl Interner {
|
||||
fn new() -> Self {
|
||||
let mut s = Interner { map: std::collections::HashMap::new(), list: Vec::new() };
|
||||
s.intern(""); // index 0 is the empty string by convention
|
||||
s
|
||||
}
|
||||
fn intern(&mut self, s: &str) -> i64 {
|
||||
if let Some(&i) = self.map.get(s) {
|
||||
return i;
|
||||
}
|
||||
let i = self.list.len() as i64;
|
||||
self.list.push(s.to_string());
|
||||
self.map.insert(s.to_string(), i);
|
||||
i
|
||||
}
|
||||
}
|
||||
|
||||
/// Pre-order DFS from the document, recording each node and its parent's index.
|
||||
/// Iterative with an explicit stack: the MAX_NODES cap bounds the node count,
|
||||
/// but recursion depth is a separate axis. A deeply nested linear chain (script
|
||||
/// can build thousands of nested elements) would recurse that many frames deep
|
||||
/// and could overflow the stack before the count guard triggers. An explicit
|
||||
/// stack keeps the depth on the heap (issue #341).
|
||||
fn walk(
|
||||
dom: &DomTree,
|
||||
root: NodeId,
|
||||
root_parent: i64,
|
||||
order: &mut Vec<NodeId>,
|
||||
parent_idx: &mut Vec<i64>,
|
||||
) {
|
||||
// (node, parent_index). Children are pushed in reverse so they pop in
|
||||
// document order, preserving the original pre-order traversal.
|
||||
let mut stack: Vec<(NodeId, i64)> = vec![(root, root_parent)];
|
||||
while let Some((id, parent)) = stack.pop() {
|
||||
if order.len() >= MAX_NODES {
|
||||
break;
|
||||
}
|
||||
let my = order.len() as i64;
|
||||
order.push(id);
|
||||
parent_idx.push(parent);
|
||||
let children: Vec<NodeId> = dom.children(id);
|
||||
for child in children.into_iter().rev() {
|
||||
stack.push((child, my));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn build_capture_snapshot(dom: &DomTree, url: &str, title: &str) -> Value {
|
||||
let mut order: Vec<NodeId> = Vec::new();
|
||||
let mut parent_idx: Vec<i64> = Vec::new();
|
||||
walk(dom, dom.document(), -1, &mut order, &mut parent_idx);
|
||||
|
||||
let mut strings = Interner::new();
|
||||
let doc_url_idx = strings.intern(url);
|
||||
let title_idx = strings.intern(title);
|
||||
|
||||
let n = order.len();
|
||||
let mut node_type: Vec<i64> = Vec::with_capacity(n);
|
||||
let mut node_name: Vec<i64> = Vec::with_capacity(n);
|
||||
let mut node_value: Vec<i64> = Vec::with_capacity(n);
|
||||
let mut backend_ids: Vec<i64> = Vec::with_capacity(n);
|
||||
let mut attributes: Vec<Value> = Vec::with_capacity(n);
|
||||
let mut clickable: Vec<i64> = Vec::new();
|
||||
|
||||
// Layout arrays are 1:1 with nodes (nodeIndex[i] == i).
|
||||
let mut layout_node_index: Vec<i64> = Vec::with_capacity(n);
|
||||
let mut bounds: Vec<Value> = Vec::with_capacity(n);
|
||||
let mut styles: Vec<Value> = Vec::with_capacity(n);
|
||||
let mut paint_orders: Vec<i64> = Vec::with_capacity(n);
|
||||
let mut client_rects: Vec<Value> = Vec::with_capacity(n);
|
||||
let mut layout_text: Vec<i64> = Vec::with_capacity(n);
|
||||
|
||||
for (i, &nid) in order.iter().enumerate() {
|
||||
let node = match dom.get_node(nid) {
|
||||
Some(node) => node,
|
||||
None => {
|
||||
// Keep arrays aligned even for a vanished node.
|
||||
node_type.push(0);
|
||||
node_name.push(0);
|
||||
node_value.push(0);
|
||||
backend_ids.push(nid.index() as i64);
|
||||
attributes.push(json!([]));
|
||||
layout_node_index.push(i as i64);
|
||||
bounds.push(json!([0.0, 0.0, 0.0, 0.0]));
|
||||
styles.push(json!([]));
|
||||
paint_orders.push(i as i64);
|
||||
client_rects.push(json!([0.0, 0.0, 0.0, 0.0]));
|
||||
layout_text.push(-1);
|
||||
continue;
|
||||
}
|
||||
};
|
||||
|
||||
let (ntype, nname, nval, attrs, tag): (i64, String, String, Vec<(String, String)>, String) =
|
||||
match &node.data {
|
||||
NodeData::Document => (9, "#document".into(), String::new(), vec![], String::new()),
|
||||
NodeData::Doctype { name, .. } => {
|
||||
(10, name.to_string(), String::new(), vec![], String::new())
|
||||
}
|
||||
NodeData::Element { name, attrs, .. } => {
|
||||
let tag = name.local.as_ref().to_string();
|
||||
let flat = attrs
|
||||
.iter()
|
||||
.map(|a| (a.name.local.to_string(), a.value.clone()))
|
||||
.collect();
|
||||
(1, tag.to_ascii_uppercase(), String::new(), flat, tag.to_ascii_lowercase())
|
||||
}
|
||||
NodeData::Text { contents } => {
|
||||
(3, "#text".into(), contents.clone(), vec![], String::new())
|
||||
}
|
||||
NodeData::Comment { contents } => {
|
||||
(8, "#comment".into(), contents.clone(), vec![], String::new())
|
||||
}
|
||||
NodeData::ProcessingInstruction { target, data } => {
|
||||
(7, target.to_string(), data.clone(), vec![], String::new())
|
||||
}
|
||||
};
|
||||
|
||||
node_type.push(ntype);
|
||||
node_name.push(strings.intern(&nname));
|
||||
node_value.push(strings.intern(&nval));
|
||||
backend_ids.push(nid.index() as i64);
|
||||
|
||||
let mut attr_idx: Vec<Value> = Vec::with_capacity(attrs.len() * 2);
|
||||
for (k, v) in &attrs {
|
||||
attr_idx.push(json!(strings.intern(k)));
|
||||
attr_idx.push(json!(strings.intern(v)));
|
||||
}
|
||||
attributes.push(json!(attr_idx));
|
||||
|
||||
let interactive = matches!(
|
||||
tag.as_str(),
|
||||
"a" | "button" | "input" | "select" | "textarea" | "summary" | "details" | "option" | "label"
|
||||
);
|
||||
let has_onclick = attrs.iter().any(|(k, _)| k.eq_ignore_ascii_case("onclick"));
|
||||
if interactive || has_onclick {
|
||||
clickable.push(i as i64);
|
||||
}
|
||||
|
||||
// Tags that never render box content; report display:none so the agent
|
||||
// does not treat them as visible.
|
||||
let hidden = matches!(
|
||||
tag.as_str(),
|
||||
"head" | "meta" | "title" | "script" | "style" | "link" | "noscript" | "base"
|
||||
);
|
||||
let display = if ntype == 1 && hidden { "none" } else { "block" };
|
||||
let cursor = if interactive { "pointer" } else { "auto" };
|
||||
let style_vals = [
|
||||
display,
|
||||
"visible",
|
||||
"1",
|
||||
"visible",
|
||||
"visible",
|
||||
"visible",
|
||||
cursor,
|
||||
"auto",
|
||||
"static",
|
||||
"rgba(0, 0, 0, 0)",
|
||||
];
|
||||
debug_assert_eq!(style_vals.len(), REQUIRED_STYLES.len());
|
||||
let style_idx: Vec<Value> = style_vals.iter().map(|s| json!(strings.intern(s))).collect();
|
||||
styles.push(json!(style_idx));
|
||||
|
||||
// Synthetic geometry: a vertical stack, full-width, 18px tall. Distinct
|
||||
// and non-icon-sized so visibility/size heuristics include the element;
|
||||
// the coordinates are not real (no layout engine).
|
||||
let y = (i as f64) * 18.0;
|
||||
bounds.push(json!([0.0, y, 1280.0, 18.0]));
|
||||
client_rects.push(json!([0.0, y, 1280.0, 18.0]));
|
||||
paint_orders.push(i as i64);
|
||||
layout_node_index.push(i as i64);
|
||||
layout_text.push(-1);
|
||||
}
|
||||
|
||||
let content_height = (n as i64) * 18;
|
||||
json!({
|
||||
"documents": [{
|
||||
"documentURL": doc_url_idx,
|
||||
"title": title_idx,
|
||||
"baseURL": doc_url_idx,
|
||||
"contentLanguage": 0,
|
||||
"encodingName": 0,
|
||||
"publicId": 0,
|
||||
"systemId": 0,
|
||||
"frameId": 0,
|
||||
"nodes": {
|
||||
"parentIndex": parent_idx,
|
||||
"nodeType": node_type,
|
||||
"nodeName": node_name,
|
||||
"nodeValue": node_value,
|
||||
"backendNodeId": backend_ids,
|
||||
"attributes": attributes,
|
||||
"isClickable": { "index": clickable },
|
||||
},
|
||||
"layout": {
|
||||
"nodeIndex": layout_node_index,
|
||||
"styles": styles,
|
||||
"bounds": bounds,
|
||||
"text": layout_text,
|
||||
"paintOrders": paint_orders,
|
||||
"clientRects": client_rects,
|
||||
},
|
||||
"textBoxes": { "layoutIndex": [], "bounds": [], "start": [], "length": [] },
|
||||
"scrollOffsetX": 0.0,
|
||||
"scrollOffsetY": 0.0,
|
||||
"contentWidth": 1280,
|
||||
"contentHeight": content_height,
|
||||
}],
|
||||
"strings": strings.list,
|
||||
})
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::dispatch::CdpContext;
|
||||
|
||||
fn collect_backend_ids(node: &Value, out: &mut Vec<i64>) {
|
||||
if let Some(id) = node.get("backendNodeId").and_then(|v| v.as_i64()) {
|
||||
out.push(id);
|
||||
}
|
||||
if let Some(children) = node.get("children").and_then(|v| v.as_array()) {
|
||||
for c in children {
|
||||
collect_backend_ids(c, out);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn find_backend_id_by_name(node: &Value, name: &str) -> Option<i64> {
|
||||
if node.get("nodeName").and_then(|v| v.as_str()) == Some(name) {
|
||||
return node.get("backendNodeId").and_then(|v| v.as_i64());
|
||||
}
|
||||
node.get("children")
|
||||
.and_then(|v| v.as_array())
|
||||
.and_then(|children| children.iter().find_map(|c| find_backend_id_by_name(c, name)))
|
||||
}
|
||||
|
||||
async fn navigate(ctx: &mut CdpContext, body: &str) -> String {
|
||||
let page_id = ctx.create_page();
|
||||
let session_id = format!("{}-session", page_id);
|
||||
ctx.sessions.insert(session_id.clone(), page_id.clone());
|
||||
let url = format!("data:text/html,{body}");
|
||||
crate::domains::page::handle(
|
||||
"navigate",
|
||||
&json!({ "url": url, "waitUntil": "load" }),
|
||||
ctx,
|
||||
&Some(session_id.clone()),
|
||||
)
|
||||
.await
|
||||
.expect("navigate should succeed");
|
||||
session_id
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn capture_snapshot_matches_getdocument_and_flags_clickable() {
|
||||
let mut ctx = CdpContext::new();
|
||||
let session = navigate(&mut ctx, "<button id=go>Go</button><a href=/x>L</a>").await;
|
||||
|
||||
// DOM.getDocument is the structural source; the snapshot must use the
|
||||
// same backendNodeId scheme so a client can correlate the two.
|
||||
let doc = crate::domains::dom::handle(
|
||||
"getDocument",
|
||||
&json!({ "depth": -1 }),
|
||||
&mut ctx,
|
||||
&Some(session.clone()),
|
||||
)
|
||||
.await
|
||||
.expect("getDocument should succeed");
|
||||
let mut doc_ids = Vec::new();
|
||||
collect_backend_ids(&doc["root"], &mut doc_ids);
|
||||
assert!(!doc_ids.is_empty(), "getDocument returned no nodes");
|
||||
|
||||
let snap = handle("captureSnapshot", &json!({}), &mut ctx, &Some(session.clone()))
|
||||
.await
|
||||
.expect("captureSnapshot should succeed");
|
||||
|
||||
let documents = snap["documents"].as_array().expect("documents array");
|
||||
assert_eq!(documents.len(), 1);
|
||||
let nodes = &documents[0]["nodes"];
|
||||
let layout = &documents[0]["layout"];
|
||||
|
||||
let snap_ids: Vec<i64> = nodes["backendNodeId"]
|
||||
.as_array()
|
||||
.unwrap()
|
||||
.iter()
|
||||
.map(|v| v.as_i64().unwrap())
|
||||
.collect();
|
||||
|
||||
// Every node from getDocument must appear in the snapshot under the
|
||||
// identical backendNodeId.
|
||||
for id in &doc_ids {
|
||||
assert!(
|
||||
snap_ids.contains(id),
|
||||
"snapshot is missing backendNodeId {id} present in getDocument"
|
||||
);
|
||||
}
|
||||
|
||||
// String table is populated (everything is referenced by index).
|
||||
assert!(
|
||||
snap["strings"].as_array().unwrap().len() > 1,
|
||||
"string table should be populated"
|
||||
);
|
||||
|
||||
// Layout arrays are aligned 1:1 with nodes and carry bounds + the 10
|
||||
// computed styles browser-use reads back positionally.
|
||||
let n = snap_ids.len();
|
||||
assert_eq!(layout["nodeIndex"].as_array().unwrap().len(), n);
|
||||
assert_eq!(layout["bounds"].as_array().unwrap().len(), n);
|
||||
assert_eq!(layout["styles"].as_array().unwrap().len(), n);
|
||||
assert_eq!(
|
||||
layout["bounds"][0].as_array().unwrap().len(),
|
||||
4,
|
||||
"bounds entries are [x, y, w, h]"
|
||||
);
|
||||
assert_eq!(
|
||||
layout["styles"][0].as_array().unwrap().len(),
|
||||
REQUIRED_STYLES.len(),
|
||||
"each layout node carries all required computed styles"
|
||||
);
|
||||
|
||||
// Interactive elements are flagged isClickable (by node index).
|
||||
let clickable: Vec<i64> = nodes["isClickable"]["index"]
|
||||
.as_array()
|
||||
.unwrap()
|
||||
.iter()
|
||||
.map(|v| v.as_i64().unwrap())
|
||||
.collect();
|
||||
for tag in ["BUTTON", "A"] {
|
||||
let bid = find_backend_id_by_name(&doc["root"], tag)
|
||||
.unwrap_or_else(|| panic!("{tag} should be in the document"));
|
||||
let idx = snap_ids
|
||||
.iter()
|
||||
.position(|&id| id == bid)
|
||||
.unwrap_or_else(|| panic!("{tag} should be in the snapshot")) as i64;
|
||||
assert!(clickable.contains(&idx), "{tag} must be flagged isClickable");
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn unknown_domsnapshot_method_is_permissive_noop() {
|
||||
// Probing the domain (e.g. getSnapshot) must not abort with an
|
||||
// Unknown-method error the way an unhandled domain would.
|
||||
let mut ctx = CdpContext::new();
|
||||
let r = handle("getSnapshot", &json!({}), &mut ctx, &None)
|
||||
.await
|
||||
.expect("unknown DOMSnapshot methods are a permissive no-op");
|
||||
assert!(r.is_object());
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,214 @@
|
||||
use std::collections::HashMap;
|
||||
|
||||
use serde_json::{json, Value};
|
||||
|
||||
use crate::dispatch::CdpContext;
|
||||
|
||||
pub struct PausedRequest {
|
||||
pub request_id: String,
|
||||
pub url: String,
|
||||
pub method: String,
|
||||
pub headers: HashMap<String, String>,
|
||||
pub resource_type: String,
|
||||
pub resolver: tokio::sync::oneshot::Sender<FetchResolution>,
|
||||
}
|
||||
|
||||
pub enum FetchResolution {
|
||||
Continue {
|
||||
url: Option<String>,
|
||||
method: Option<String>,
|
||||
headers: Option<HashMap<String, String>>,
|
||||
post_data: Option<String>,
|
||||
},
|
||||
Fulfill {
|
||||
status: u16,
|
||||
headers: Vec<(String, String)>,
|
||||
body: String,
|
||||
},
|
||||
Fail {
|
||||
reason: String,
|
||||
},
|
||||
}
|
||||
|
||||
pub struct FetchInterceptState {
|
||||
pub enabled: bool,
|
||||
pub patterns: Vec<String>,
|
||||
pub paused: HashMap<String, PausedRequest>,
|
||||
request_counter: u64,
|
||||
}
|
||||
|
||||
impl FetchInterceptState {
|
||||
pub fn new() -> Self {
|
||||
FetchInterceptState {
|
||||
enabled: false,
|
||||
patterns: Vec::new(),
|
||||
paused: HashMap::new(),
|
||||
request_counter: 0,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn next_request_id(&mut self) -> String {
|
||||
self.request_counter += 1;
|
||||
format!("interception-{}", self.request_counter)
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn handle(
|
||||
method: &str,
|
||||
params: &Value,
|
||||
ctx: &mut CdpContext,
|
||||
session_id: &Option<String>,
|
||||
) -> Result<Value, String> {
|
||||
match method {
|
||||
"enable" => {
|
||||
let patterns = params
|
||||
.get("patterns")
|
||||
.and_then(|v| v.as_array())
|
||||
.map(|arr| {
|
||||
arr.iter()
|
||||
.filter_map(|p| {
|
||||
p.get("urlPattern")
|
||||
.and_then(|v| v.as_str())
|
||||
.map(|s| s.to_string())
|
||||
})
|
||||
.collect::<Vec<_>>()
|
||||
})
|
||||
.unwrap_or_else(|| vec!["*".to_string()]);
|
||||
|
||||
ctx.fetch_intercept.enabled = true;
|
||||
ctx.fetch_intercept.patterns = patterns.clone();
|
||||
let tx_clone = ctx.intercept_tx.clone();
|
||||
if let Some(page) = ctx.get_session_page_mut(session_id) {
|
||||
page.intercept_block_patterns = patterns.clone();
|
||||
if let Some(tx) = tx_clone {
|
||||
page.set_intercept_tx(tx);
|
||||
}
|
||||
page.enable_intercept(true);
|
||||
}
|
||||
|
||||
tracing::info!("Fetch interception enabled");
|
||||
Ok(json!({}))
|
||||
}
|
||||
"disable" => {
|
||||
ctx.fetch_intercept.enabled = false;
|
||||
ctx.fetch_intercept.patterns.clear();
|
||||
if let Some(page) = ctx.get_session_page_mut(session_id) {
|
||||
page.intercept_block_patterns.clear();
|
||||
page.enable_intercept(false);
|
||||
}
|
||||
let paused: Vec<_> = ctx.fetch_intercept.paused.drain().collect();
|
||||
for (_, req) in paused {
|
||||
let _ = req.resolver.send(FetchResolution::Continue {
|
||||
url: None,
|
||||
method: None,
|
||||
headers: None,
|
||||
post_data: None,
|
||||
});
|
||||
}
|
||||
Ok(json!({}))
|
||||
}
|
||||
"continueRequest" => {
|
||||
let request_id = params
|
||||
.get("requestId")
|
||||
.and_then(|v| v.as_str())
|
||||
.ok_or("requestId required")?;
|
||||
|
||||
if let Some(paused) = ctx.fetch_intercept.paused.remove(request_id) {
|
||||
let _ = paused.resolver.send(FetchResolution::Continue {
|
||||
url: params.get("url").and_then(|v| v.as_str()).map(|s| s.to_string()),
|
||||
method: params.get("method").and_then(|v| v.as_str()).map(|s| s.to_string()),
|
||||
headers: None,
|
||||
post_data: params.get("postData").and_then(|v| v.as_str()).map(|s| s.to_string()),
|
||||
});
|
||||
}
|
||||
Ok(json!({}))
|
||||
}
|
||||
"fulfillRequest" => {
|
||||
let request_id = params
|
||||
.get("requestId")
|
||||
.and_then(|v| v.as_str())
|
||||
.ok_or("requestId required")?;
|
||||
|
||||
let status = params
|
||||
.get("responseCode")
|
||||
.and_then(|v| v.as_u64())
|
||||
.unwrap_or(200) as u16;
|
||||
let headers: HashMap<String, String> = params
|
||||
.get("responseHeaders")
|
||||
.and_then(|v| v.as_array())
|
||||
.map(|arr| {
|
||||
arr.iter()
|
||||
.filter_map(|h| {
|
||||
let name = h.get("name")?.as_str()?.to_string();
|
||||
let value = h.get("value")?.as_str()?.to_string();
|
||||
Some((name, value))
|
||||
})
|
||||
.collect()
|
||||
})
|
||||
.unwrap_or_default();
|
||||
let body = params
|
||||
.get("body")
|
||||
.and_then(|v| v.as_str())
|
||||
.unwrap_or("")
|
||||
.to_string();
|
||||
|
||||
if let Some(paused) = ctx.fetch_intercept.paused.remove(request_id) {
|
||||
let _ = paused.resolver.send(FetchResolution::Fulfill {
|
||||
status,
|
||||
headers: headers.into_iter().collect(),
|
||||
body,
|
||||
});
|
||||
}
|
||||
Ok(json!({}))
|
||||
}
|
||||
"failRequest" => {
|
||||
let request_id = params
|
||||
.get("requestId")
|
||||
.and_then(|v| v.as_str())
|
||||
.ok_or("requestId required")?;
|
||||
|
||||
let reason = params
|
||||
.get("errorReason")
|
||||
.and_then(|v| v.as_str())
|
||||
.unwrap_or("Failed")
|
||||
.to_string();
|
||||
|
||||
if let Some(paused) = ctx.fetch_intercept.paused.remove(request_id) {
|
||||
let _ = paused.resolver.send(FetchResolution::Fail { reason });
|
||||
}
|
||||
Ok(json!({}))
|
||||
}
|
||||
"getResponseBody" => {
|
||||
Ok(json!({ "body": "", "base64Encoded": false }))
|
||||
}
|
||||
"takeResponseBodyAsStream" => {
|
||||
// Hand the client a streaming handle for a large response body so it
|
||||
// can pull it in chunks via IO.read and free it with IO.close,
|
||||
// instead of receiving one giant base64 blob (issue #360). The body
|
||||
// is moved out of the page cache into the stream, so it is held once
|
||||
// and released on close. Requires the body to have been cached
|
||||
// (raise OBSCURA_NETWORK_BODY_BUFFER_BYTES for large downloads).
|
||||
let request_id = params
|
||||
.get("requestId")
|
||||
.and_then(|v| v.as_str())
|
||||
.ok_or("Fetch.takeResponseBodyAsStream requires requestId")?;
|
||||
|
||||
let bytes = {
|
||||
let page = ctx.get_session_page_mut(session_id).ok_or("No page")?;
|
||||
page.take_response_body_raw(request_id)
|
||||
}
|
||||
.or_else(|| {
|
||||
ctx.pages
|
||||
.iter_mut()
|
||||
.find_map(|p| p.take_response_body_raw(request_id))
|
||||
})
|
||||
.ok_or_else(|| {
|
||||
format!("Fetch.takeResponseBodyAsStream: no cached body for {request_id}")
|
||||
})?;
|
||||
|
||||
let handle = ctx.io_streams.insert(bytes);
|
||||
Ok(json!({ "stream": handle }))
|
||||
}
|
||||
_ => Err(format!("Unknown Fetch method: {}", method)),
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,220 @@
|
||||
use serde_json::{json, Value};
|
||||
|
||||
use crate::dispatch::CdpContext;
|
||||
|
||||
// Insert `escaped_text` at the caret, replacing any non-collapsed selection
|
||||
// the way a real browser does when you type over selected text (for example
|
||||
// after a triple-click select-all). selectionStart is null during ordinary
|
||||
// typing, so the legacy append path is kept when no selection is tracked.
|
||||
fn insert_text_js(escaped_text: &str) -> String {
|
||||
format!(
|
||||
"(function() {{\
|
||||
var t = document.activeElement;\
|
||||
if (!t || (t.localName !== 'input' && t.localName !== 'textarea')) return;\
|
||||
var v = t.value || '';\
|
||||
var s = t.selectionStart, e = t.selectionEnd;\
|
||||
if (s == null) {{\
|
||||
globalThis.__obscura_setFieldValue(t, 'value', v + '{text}');\
|
||||
}} else {{\
|
||||
s = Math.max(0, Math.min(s, v.length));\
|
||||
e = (e == null) ? s : Math.max(0, Math.min(e, v.length));\
|
||||
var lo = Math.min(s, e), hi = Math.max(s, e);\
|
||||
globalThis.__obscura_setFieldValue(t, 'value', v.slice(0, lo) + '{text}' + v.slice(hi));\
|
||||
var caret = lo + ('{text}').length;\
|
||||
t.setSelectionRange(caret, caret);\
|
||||
}}\
|
||||
t.dispatchEvent(globalThis.__obscura_markTrusted(new Event('input', {{bubbles:true}})));\
|
||||
}})()",
|
||||
text = escaped_text,
|
||||
)
|
||||
}
|
||||
|
||||
// Backspace deletes the selected range when there is one, so the common
|
||||
// "triple-click to select-all, then Backspace to clear" pattern works. With a
|
||||
// collapsed caret it removes the character before the caret, and with no
|
||||
// selection tracked it falls back to trimming the last character (legacy).
|
||||
const BACKSPACE_JS: &str = "(function() {\
|
||||
var t = document.activeElement;\
|
||||
if (!t || (t.localName !== 'input' && t.localName !== 'textarea')) return;\
|
||||
var v = t.value || '';\
|
||||
var s = t.selectionStart, e = t.selectionEnd;\
|
||||
if (s == null) {\
|
||||
globalThis.__obscura_setFieldValue(t, 'value', v.slice(0, -1));\
|
||||
} else {\
|
||||
s = Math.max(0, Math.min(s, v.length));\
|
||||
e = (e == null) ? s : Math.max(0, Math.min(e, v.length));\
|
||||
if (s !== e) {\
|
||||
var lo = Math.min(s, e), hi = Math.max(s, e);\
|
||||
globalThis.__obscura_setFieldValue(t, 'value', v.slice(0, lo) + v.slice(hi));\
|
||||
t.setSelectionRange(lo, lo);\
|
||||
} else if (s > 0) {\
|
||||
globalThis.__obscura_setFieldValue(t, 'value', v.slice(0, s - 1) + v.slice(s));\
|
||||
t.setSelectionRange(s - 1, s - 1);\
|
||||
}\
|
||||
}\
|
||||
t.dispatchEvent(globalThis.__obscura_markTrusted(new Event('input', {bubbles:true})));\
|
||||
})()";
|
||||
|
||||
pub async fn handle(
|
||||
method: &str,
|
||||
params: &Value,
|
||||
ctx: &mut CdpContext,
|
||||
session_id: &Option<String>,
|
||||
) -> Result<Value, String> {
|
||||
match method {
|
||||
"dispatchMouseEvent" => {
|
||||
let event_type = params.get("type").and_then(|v| v.as_str()).unwrap_or("");
|
||||
let x = params.get("x").and_then(|v| v.as_f64()).unwrap_or(0.0);
|
||||
let y = params.get("y").and_then(|v| v.as_f64()).unwrap_or(0.0);
|
||||
let _button = params.get("button").and_then(|v| v.as_str()).unwrap_or("left");
|
||||
let click_count = params.get("clickCount").and_then(|v| v.as_u64()).unwrap_or(1);
|
||||
|
||||
if event_type == "mousePressed" {
|
||||
if let Some(page) = ctx.get_session_page_mut(session_id) {
|
||||
let code = format!(
|
||||
"(function() {{\
|
||||
var target = (document.elementFromPoint && document.elementFromPoint({x},{y})) || globalThis.__obscura_click_target || document.activeElement || document.body;\
|
||||
if (!target) return;\
|
||||
globalThis.__obscura_click_target = target;\
|
||||
var evt = globalThis.__obscura_markTrusted(new MouseEvent('mousedown', {{bubbles:true,cancelable:true,clientX:{x},clientY:{y},button:0,detail:{click_count}}}));\
|
||||
target.dispatchEvent(evt);\
|
||||
var click = globalThis.__obscura_markTrusted(new MouseEvent('click', {{bubbles:true,cancelable:true,clientX:{x},clientY:{y},button:0,detail:{click_count}}}));\
|
||||
var cancelled = !target.dispatchEvent(click);\
|
||||
if (!cancelled) {{\
|
||||
var link = target.closest ? target.closest('a[href]') : null;\
|
||||
if (!link && target.tagName === 'A' && target.getAttribute('href')) link = target;\
|
||||
if (link) {{\
|
||||
var href = link.getAttribute('href');\
|
||||
if (href && !href.startsWith('#') && !href.startsWith('javascript:')) {{\
|
||||
location.assign(href);\
|
||||
}}\
|
||||
}} else {{\
|
||||
var tag = target.tagName;\
|
||||
var type = (target.getAttribute && target.getAttribute('type') || '').toLowerCase();\
|
||||
if (tag === 'BUTTON' && type !== 'button' && type !== 'reset') {{\
|
||||
var form = target.closest ? target.closest('form') : null;\
|
||||
if (form && typeof form.submit === 'function') {{ try {{ form.submit(target); }} catch(e) {{}} }}\
|
||||
}} else if (tag === 'INPUT' && (type === 'submit' || type === 'image')) {{\
|
||||
var form2 = target.closest ? target.closest('form') : null;\
|
||||
if (form2 && typeof form2.submit === 'function') {{ try {{ form2.submit(target); }} catch(e) {{}} }}\
|
||||
}} else if (tag === 'INPUT' && (type === 'checkbox' || type === 'radio')) {{\
|
||||
target.checked = !target.checked;\
|
||||
try {{ target.dispatchEvent(globalThis.__obscura_markTrusted(new Event('change', {{bubbles:true}}))); }} catch(e) {{}}\
|
||||
}} else if ({click_count} >= 3 && (tag === 'INPUT' || tag === 'TEXTAREA')) {{\
|
||||
// Triple-click selects all text (browser native behavior not replicated
|
||||
// by synthetic MouseEvent, so we do it manually).
|
||||
var len = target.value ? target.value.length : 0;\
|
||||
if (target.setSelectionRange) {{\
|
||||
target.setSelectionRange(0, len);\
|
||||
}} else {{\
|
||||
target.selectionStart = 0;\
|
||||
target.selectionEnd = len;\
|
||||
}}\
|
||||
}}\
|
||||
}}\
|
||||
}}\
|
||||
}})()",
|
||||
x = x, y = y, click_count = click_count,
|
||||
);
|
||||
page.evaluate(&code);
|
||||
page.process_pending_navigation().await.map_err(|e| e.to_string())?;
|
||||
}
|
||||
} else if event_type == "mouseReleased" {
|
||||
if let Some(page) = ctx.get_session_page_mut(session_id) {
|
||||
let code = format!(
|
||||
"(function() {{\
|
||||
var target = (document.elementFromPoint && document.elementFromPoint({x},{y})) || globalThis.__obscura_click_target || document.activeElement || document.body;\
|
||||
if (!target) return;\
|
||||
var evt = globalThis.__obscura_markTrusted(new MouseEvent('mouseup', {{bubbles:true,cancelable:true,clientX:{x},clientY:{y},button:0}}));\
|
||||
target.dispatchEvent(evt);\
|
||||
}})()",
|
||||
x = x, y = y,
|
||||
);
|
||||
page.evaluate(&code);
|
||||
}
|
||||
}
|
||||
|
||||
Ok(json!({}))
|
||||
}
|
||||
"dispatchKeyEvent" => {
|
||||
let event_type = params.get("type").and_then(|v| v.as_str()).unwrap_or("");
|
||||
let key = params.get("key").and_then(|v| v.as_str()).unwrap_or("");
|
||||
let code = params.get("code").and_then(|v| v.as_str()).unwrap_or("");
|
||||
let text = params.get("text").and_then(|v| v.as_str()).unwrap_or("");
|
||||
|
||||
if let Some(page) = ctx.get_session_page_mut(session_id) {
|
||||
match event_type {
|
||||
"keyDown" | "rawKeyDown" => {
|
||||
let js = format!(
|
||||
"(function() {{\
|
||||
var target = document.activeElement || document.body;\
|
||||
var evt = globalThis.__obscura_markTrusted(new KeyboardEvent('keydown', {{bubbles:true,cancelable:true,key:'{key}',code:'{code}'}}));\
|
||||
target.dispatchEvent(evt);\
|
||||
}})()",
|
||||
key = key.replace('\'', "\\'"),
|
||||
code = code.replace('\'', "\\'"),
|
||||
);
|
||||
page.evaluate(&js);
|
||||
|
||||
if !text.is_empty() && text != "\r" && text != "\n" {
|
||||
// Need to escape backslash BEFORE single-quote so the new
|
||||
// backslashes from quote escaping don't get double-escaped.
|
||||
let escaped_text = text.replace('\\', "\\\\").replace('\'', "\\'");
|
||||
page.evaluate(&insert_text_js(&escaped_text));
|
||||
}
|
||||
|
||||
if key == "Enter" {
|
||||
// In a textarea Enter inserts a newline; in input fields
|
||||
// it submits the containing form. Real Chrome distinguishes
|
||||
// these two and we should too: previously every Enter tried
|
||||
// to submit the nearest form even from a textarea.
|
||||
let js = "(function() {\
|
||||
var target = document.activeElement;\
|
||||
if (!target) return;\
|
||||
target.dispatchEvent(globalThis.__obscura_markTrusted(new KeyboardEvent('keypress', {bubbles:true,key:'Enter',code:'Enter'})));\
|
||||
if (target.localName === 'textarea') {\
|
||||
globalThis.__obscura_setFieldValue(target, 'value', (target.value || '') + '\\n');\
|
||||
target.dispatchEvent(globalThis.__obscura_markTrusted(new Event('input', {bubbles:true})));\
|
||||
} else {\
|
||||
var form = target.form || (target.closest && target.closest('form'));\
|
||||
if (form && typeof form.submit === 'function') form.submit();\
|
||||
}\
|
||||
})()";
|
||||
page.evaluate(js);
|
||||
}
|
||||
|
||||
if key == "Backspace" {
|
||||
page.evaluate(BACKSPACE_JS);
|
||||
}
|
||||
}
|
||||
"keyUp" => {
|
||||
let js = format!(
|
||||
"(function() {{\
|
||||
var target = document.activeElement || document.body;\
|
||||
var evt = globalThis.__obscura_markTrusted(new KeyboardEvent('keyup', {{bubbles:true,key:'{key}',code:'{code}'}}));\
|
||||
target.dispatchEvent(evt);\
|
||||
}})()",
|
||||
key = key.replace('\'', "\\'"),
|
||||
code = code.replace('\'', "\\'"),
|
||||
);
|
||||
page.evaluate(&js);
|
||||
}
|
||||
"char" => {
|
||||
if !text.is_empty() {
|
||||
let escaped_text = text.replace('\\', "\\\\").replace('\'', "\\'");
|
||||
page.evaluate(&insert_text_js(&escaped_text));
|
||||
// Pump event loop so Angular change detection picks up the input
|
||||
page.settle(50).await;
|
||||
}
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
|
||||
Ok(json!({}))
|
||||
}
|
||||
"dispatchTouchEvent" => Ok(json!({})),
|
||||
"setIgnoreInputEvents" => Ok(json!({})),
|
||||
_ => Err(format!("Unknown Input method: {}", method)),
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,183 @@
|
||||
use std::collections::{HashMap, VecDeque};
|
||||
|
||||
use base64::Engine as _;
|
||||
use serde_json::{json, Value};
|
||||
|
||||
use crate::dispatch::CdpContext;
|
||||
|
||||
// Default chunk size when the client does not pass `size`. Chrome uses a similar
|
||||
// order of magnitude; keeping chunks bounded is the point of streaming (issue
|
||||
// #360), so we never return the whole body in one IO.read.
|
||||
const DEFAULT_CHUNK: usize = 1 << 20; // 1 MiB
|
||||
|
||||
fn io_stream_max_entries() -> usize {
|
||||
std::env::var("OBSCURA_IO_STREAM_MAX_ENTRIES")
|
||||
.ok()
|
||||
.and_then(|v| v.parse().ok())
|
||||
.unwrap_or(32)
|
||||
}
|
||||
|
||||
fn io_stream_max_bytes() -> usize {
|
||||
std::env::var("OBSCURA_IO_STREAM_MAX_BYTES")
|
||||
.ok()
|
||||
.and_then(|v| v.parse().ok())
|
||||
.unwrap_or(256 * 1024 * 1024)
|
||||
}
|
||||
|
||||
/// Bounded store of the response bodies handed out by
|
||||
/// Fetch.takeResponseBodyAsStream. Streaming exists to keep large downloads out
|
||||
/// of memory (issue #360), but each taken body is moved out of the page's
|
||||
/// LRU-bounded cache into this map, which lives for the whole server lifetime.
|
||||
/// A client that opens streams and never calls IO.close, or simply disconnects
|
||||
/// mid-download, would otherwise pin every taken body forever and reintroduce
|
||||
/// exactly the unbounded accumulation streaming was meant to avoid. Cap the
|
||||
/// number of open streams and their total bytes, evicting the oldest first, so
|
||||
/// memory stays bounded regardless of client behavior. Reading an evicted
|
||||
/// handle fails cleanly (the client re-takes or gives up), which is the right
|
||||
/// trade against an OOM.
|
||||
#[derive(Default)]
|
||||
pub struct IoStreamStore {
|
||||
streams: HashMap<String, (Vec<u8>, usize)>,
|
||||
order: VecDeque<String>,
|
||||
total_bytes: usize,
|
||||
counter: u64,
|
||||
}
|
||||
|
||||
impl IoStreamStore {
|
||||
/// Store a body and return its handle, evicting the oldest streams if this
|
||||
/// pushes the store past its entry or byte cap.
|
||||
pub fn insert(&mut self, bytes: Vec<u8>) -> String {
|
||||
let handle = format!("stream-{}", self.counter);
|
||||
self.counter += 1;
|
||||
self.total_bytes += bytes.len();
|
||||
self.streams.insert(handle.clone(), (bytes, 0));
|
||||
self.order.push_back(handle.clone());
|
||||
|
||||
let max_entries = io_stream_max_entries().max(1);
|
||||
let max_bytes = io_stream_max_bytes();
|
||||
// Evict oldest streams while over either cap. Never evict the stream we
|
||||
// just inserted (it sits at the back of `order`); a lone oversized body
|
||||
// is kept, since the client explicitly asked to stream it.
|
||||
while self.order.len() > 1
|
||||
&& (self.order.len() > max_entries || self.total_bytes > max_bytes)
|
||||
{
|
||||
if let Some(oldest) = self.order.pop_front() {
|
||||
if let Some((b, _)) = self.streams.remove(&oldest) {
|
||||
self.total_bytes -= b.len();
|
||||
}
|
||||
}
|
||||
}
|
||||
handle
|
||||
}
|
||||
|
||||
/// Read up to `size` bytes from the stream, advancing its cursor. Returns
|
||||
/// the base64 chunk and whether EOF was reached, or None for an unknown or
|
||||
/// already-freed handle.
|
||||
pub fn read(&mut self, handle: &str, size: usize) -> Option<(String, bool)> {
|
||||
let (bytes, cursor) = self.streams.get_mut(handle)?;
|
||||
let start = (*cursor).min(bytes.len());
|
||||
let end = start.saturating_add(size.max(1)).min(bytes.len());
|
||||
let data = base64::engine::general_purpose::STANDARD.encode(&bytes[start..end]);
|
||||
*cursor = end;
|
||||
Some((data, end >= bytes.len()))
|
||||
}
|
||||
|
||||
/// Free a stream's buffer (IO.close). A no-op for an unknown handle.
|
||||
pub fn remove(&mut self, handle: &str) {
|
||||
if let Some((b, _)) = self.streams.remove(handle) {
|
||||
self.total_bytes -= b.len();
|
||||
self.order.retain(|h| h != handle);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// CDP IO domain. Streams a response body handed out by
|
||||
/// Fetch.takeResponseBodyAsStream: IO.read returns the next base64 chunk and
|
||||
/// IO.close frees the buffer. Nothing here runs unless a client opened a stream.
|
||||
pub async fn handle(method: &str, params: &Value, ctx: &mut CdpContext) -> Result<Value, String> {
|
||||
match method {
|
||||
"read" => {
|
||||
let handle = params
|
||||
.get("handle")
|
||||
.and_then(|v| v.as_str())
|
||||
.ok_or("IO.read requires handle")?;
|
||||
let size = params
|
||||
.get("size")
|
||||
.and_then(|v| v.as_u64())
|
||||
.map(|s| s as usize)
|
||||
.unwrap_or(DEFAULT_CHUNK);
|
||||
|
||||
let (data, eof) = ctx
|
||||
.io_streams
|
||||
.read(handle, size)
|
||||
.ok_or_else(|| format!("IO.read: unknown handle {handle}"))?;
|
||||
|
||||
Ok(json!({ "data": data, "eof": eof, "base64Encoded": true }))
|
||||
}
|
||||
"close" => {
|
||||
let handle = params
|
||||
.get("handle")
|
||||
.and_then(|v| v.as_str())
|
||||
.ok_or("IO.close requires handle")?;
|
||||
ctx.io_streams.remove(handle);
|
||||
Ok(json!({}))
|
||||
}
|
||||
_ => Err(format!("Unknown IO method: {}", method)),
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
fn decode(s: &str) -> Vec<u8> {
|
||||
base64::engine::general_purpose::STANDARD.decode(s).unwrap()
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn reads_chunks_then_frees() {
|
||||
let mut store = IoStreamStore::default();
|
||||
let h = store.insert(b"hello".to_vec());
|
||||
|
||||
let (d1, eof1) = store.read(&h, 3).unwrap();
|
||||
assert_eq!(decode(&d1), b"hel");
|
||||
assert!(!eof1);
|
||||
|
||||
let (d2, eof2) = store.read(&h, 3).unwrap();
|
||||
assert_eq!(decode(&d2), b"lo");
|
||||
assert!(eof2);
|
||||
|
||||
store.remove(&h);
|
||||
assert!(store.read(&h, 3).is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn evicts_oldest_over_entry_cap() {
|
||||
std::env::set_var("OBSCURA_IO_STREAM_MAX_ENTRIES", "3");
|
||||
let mut store = IoStreamStore::default();
|
||||
let h0 = store.insert(vec![0]);
|
||||
let h1 = store.insert(vec![1]);
|
||||
let _h2 = store.insert(vec![2]);
|
||||
let h3 = store.insert(vec![3]); // 4th entry, cap 3 -> h0 evicted
|
||||
std::env::remove_var("OBSCURA_IO_STREAM_MAX_ENTRIES");
|
||||
|
||||
assert!(store.read(&h0, 10).is_none(), "oldest stream should be evicted");
|
||||
assert!(store.read(&h1, 10).is_some());
|
||||
assert!(store.read(&h3, 10).is_some());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn evicts_over_byte_cap_but_keeps_newest() {
|
||||
std::env::set_var("OBSCURA_IO_STREAM_MAX_BYTES", "10");
|
||||
let mut store = IoStreamStore::default();
|
||||
let h0 = store.insert(vec![0u8; 8]);
|
||||
let h1 = store.insert(vec![1u8; 8]); // 16 > 10 -> h0 evicted
|
||||
// A single body larger than the cap is still kept (client asked for it).
|
||||
let big = store.insert(vec![2u8; 100]);
|
||||
std::env::remove_var("OBSCURA_IO_STREAM_MAX_BYTES");
|
||||
|
||||
assert!(store.read(&h0, 100).is_none());
|
||||
assert!(store.read(&big, 200).is_some(), "the just-inserted body is never evicted");
|
||||
let _ = h1;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,21 @@
|
||||
use obscura_browser::HTML_TO_MARKDOWN_JS;
|
||||
use serde_json::{json, Value};
|
||||
|
||||
use crate::dispatch::CdpContext;
|
||||
|
||||
pub async fn handle(
|
||||
method: &str,
|
||||
_params: &Value,
|
||||
ctx: &mut CdpContext,
|
||||
session_id: &Option<String>,
|
||||
) -> Result<Value, String> {
|
||||
match method {
|
||||
"getMarkdown" => {
|
||||
let page = ctx.get_session_page_mut(session_id).ok_or("No page")?;
|
||||
let result = page.evaluate(HTML_TO_MARKDOWN_JS);
|
||||
let markdown = result.as_str().unwrap_or("").to_string();
|
||||
Ok(json!({ "markdown": markdown }))
|
||||
}
|
||||
_ => Err(format!("Unknown LP method: {}", method)),
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,13 @@
|
||||
pub mod target;
|
||||
pub mod browser;
|
||||
pub mod page;
|
||||
pub mod dom;
|
||||
pub mod domsnapshot;
|
||||
pub mod runtime;
|
||||
pub mod network;
|
||||
pub mod fetch;
|
||||
pub mod io;
|
||||
pub mod input;
|
||||
pub mod storage;
|
||||
pub mod accessibility;
|
||||
pub mod lp;
|
||||
@@ -0,0 +1,430 @@
|
||||
use std::sync::Arc;
|
||||
|
||||
use serde_json::{json, Value};
|
||||
|
||||
use obscura_net::CookieJar;
|
||||
|
||||
use crate::cookie_params::{parse_cdp_cookie, parse_delete_cookies_params};
|
||||
use crate::dispatch::CdpContext;
|
||||
|
||||
const SESSION_COOKIE_EXPIRES: i64 = -1;
|
||||
const DEFAULT_SECURE_PORT: u16 = 443;
|
||||
const DEFAULT_INSECURE_PORT: u16 = 80;
|
||||
const SOURCE_SCHEME_SECURE: &str = "Secure";
|
||||
const SOURCE_SCHEME_NONSECURE: &str = "NonSecure";
|
||||
const DEFAULT_SAME_SITE: &str = "Lax";
|
||||
|
||||
// Resolve the cookie jar for a Network request: prefer the session's page jar,
|
||||
// fall back to the default browser context. Puppeteer and Playwright both call
|
||||
// Network.setCookie/getCookies/deleteCookies BEFORE attaching to a target —
|
||||
// requiring a session would break those flows (Storage.* already mirrors this).
|
||||
fn cookie_jar_for<'a>(ctx: &'a CdpContext, session_id: &Option<String>) -> &'a Arc<CookieJar> {
|
||||
ctx.get_session_page(session_id)
|
||||
.map(|p| &p.context.cookie_jar)
|
||||
.unwrap_or(&ctx.default_context.cookie_jar)
|
||||
}
|
||||
|
||||
pub async fn handle(
|
||||
method: &str,
|
||||
params: &Value,
|
||||
ctx: &mut CdpContext,
|
||||
session_id: &Option<String>,
|
||||
) -> Result<Value, String> {
|
||||
match method {
|
||||
"enable" => Ok(json!({})),
|
||||
"disable" => {
|
||||
if let Some(page) = ctx.get_session_page_mut(session_id) {
|
||||
page.clear_response_bodies();
|
||||
} else {
|
||||
for page in &mut ctx.pages {
|
||||
page.clear_response_bodies();
|
||||
}
|
||||
}
|
||||
Ok(json!({}))
|
||||
}
|
||||
"setExtraHTTPHeaders" => {
|
||||
let headers = params.get("headers").and_then(|v| v.as_object());
|
||||
if let Some(page) = ctx.get_session_page(session_id) {
|
||||
if let Some(headers) = headers {
|
||||
let header_map: std::collections::HashMap<String, String> = headers
|
||||
.iter()
|
||||
.map(|(k, v)| (k.clone(), v.as_str().unwrap_or("").to_string()))
|
||||
.collect();
|
||||
page.http_client.set_extra_headers(header_map).await;
|
||||
}
|
||||
}
|
||||
Ok(json!({}))
|
||||
}
|
||||
"setUserAgentOverride" => {
|
||||
let ua = params.get("userAgent").and_then(|v| v.as_str()).unwrap_or("");
|
||||
if let Some(page) = ctx.get_session_page(session_id) {
|
||||
page.http_client.set_user_agent(ua).await;
|
||||
}
|
||||
Ok(json!({}))
|
||||
}
|
||||
"getCookies" | "getAllCookies" => {
|
||||
let cookies = cookie_jar_for(ctx, session_id).get_all_cookies();
|
||||
let cdp_cookies: Vec<Value> = cookies.iter().map(cookie_info_to_cdp_json).collect();
|
||||
Ok(json!({ "cookies": cdp_cookies }))
|
||||
}
|
||||
"setCookie" => {
|
||||
let cookie = parse_cdp_cookie(params)
|
||||
.ok_or("setCookie: missing required name/domain (or url)")?;
|
||||
cookie_jar_for(ctx, session_id).set_cookies_from_cdp(vec![cookie]);
|
||||
Ok(json!({ "success": true }))
|
||||
}
|
||||
"setCookies" => {
|
||||
if let Some(cookies) = params.get("cookies").and_then(|v| v.as_array()) {
|
||||
let parsed: Vec<_> = cookies.iter().filter_map(parse_cdp_cookie).collect();
|
||||
cookie_jar_for(ctx, session_id).set_cookies_from_cdp(parsed);
|
||||
}
|
||||
Ok(json!({}))
|
||||
}
|
||||
"deleteCookies" => {
|
||||
if let Some(filter) = parse_delete_cookies_params(params) {
|
||||
cookie_jar_for(ctx, session_id).delete_cookies_filtered(
|
||||
&filter.name,
|
||||
&filter.domain,
|
||||
filter.path.as_deref(),
|
||||
);
|
||||
}
|
||||
Ok(json!({}))
|
||||
}
|
||||
"clearBrowserCookies" => {
|
||||
cookie_jar_for(ctx, session_id).clear();
|
||||
Ok(json!({}))
|
||||
}
|
||||
"setCacheDisabled" => Ok(json!({})),
|
||||
"setRequestInterception" => Ok(json!({})),
|
||||
"setBlockedURLs" => {
|
||||
let patterns = params
|
||||
.get("urls")
|
||||
.and_then(|value| value.as_array())
|
||||
.map(|values| {
|
||||
values
|
||||
.iter()
|
||||
.filter_map(|value| value.as_str().map(ToString::to_string))
|
||||
.collect::<Vec<_>>()
|
||||
})
|
||||
.unwrap_or_default();
|
||||
|
||||
if let Some(page) = ctx.get_session_page_mut(session_id) {
|
||||
page.set_blocked_urls(patterns);
|
||||
} else {
|
||||
for page in &mut ctx.pages {
|
||||
page.set_blocked_urls(patterns.clone());
|
||||
}
|
||||
}
|
||||
Ok(json!({}))
|
||||
}
|
||||
"getResponseBody" => {
|
||||
let request_id = params
|
||||
.get("requestId")
|
||||
.and_then(|v| v.as_str())
|
||||
.ok_or("Network.getResponseBody requires requestId")?;
|
||||
|
||||
let body = if let Some(page) = ctx.get_session_page(session_id) {
|
||||
page.get_response_body(request_id)
|
||||
} else {
|
||||
ctx.pages.iter().find_map(|page| page.get_response_body(request_id))
|
||||
};
|
||||
|
||||
match body {
|
||||
Some(body) => Ok(json!({
|
||||
"body": body.body,
|
||||
"base64Encoded": body.base64_encoded,
|
||||
})),
|
||||
None => Err(format!("No response body found for requestId {}", request_id)),
|
||||
}
|
||||
}
|
||||
_ => Err(format!("Unknown Network method: {}", method)),
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use obscura_net::CookieInfo;
|
||||
|
||||
fn sample_cookie(name: &str) -> CookieInfo {
|
||||
CookieInfo {
|
||||
name: name.to_string(),
|
||||
value: "v".to_string(),
|
||||
domain: "example.com".to_string(),
|
||||
path: "/".to_string(),
|
||||
secure: false,
|
||||
http_only: false,
|
||||
same_site: String::new(),
|
||||
expires: None,
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn set_cookie_without_session_targets_default_context() {
|
||||
let mut ctx = CdpContext::new();
|
||||
let params = json!({
|
||||
"name": "sid",
|
||||
"value": "abc",
|
||||
"domain": "example.com",
|
||||
"path": "/"
|
||||
});
|
||||
let resp = handle("setCookie", ¶ms, &mut ctx, &None)
|
||||
.await
|
||||
.expect("setCookie must succeed without a session");
|
||||
assert_eq!(resp["success"], json!(true));
|
||||
let cookies = ctx.default_context.cookie_jar.get_all_cookies();
|
||||
assert_eq!(cookies.len(), 1, "default cookie jar must receive the cookie");
|
||||
assert_eq!(cookies[0].name, "sid");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn set_cookies_without_session_targets_default_context() {
|
||||
let mut ctx = CdpContext::new();
|
||||
let params = json!({
|
||||
"cookies": [
|
||||
{ "name": "a", "value": "1", "domain": "example.com", "path": "/" },
|
||||
{ "name": "b", "value": "2", "domain": "example.com", "path": "/" }
|
||||
]
|
||||
});
|
||||
handle("setCookies", ¶ms, &mut ctx, &None)
|
||||
.await
|
||||
.expect("setCookies must succeed without a session");
|
||||
assert_eq!(ctx.default_context.cookie_jar.get_all_cookies().len(), 2);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn delete_cookies_without_session_targets_default_context() {
|
||||
let mut ctx = CdpContext::new();
|
||||
ctx.default_context
|
||||
.cookie_jar
|
||||
.set_cookies_from_cdp(vec![sample_cookie("sid")]);
|
||||
let params = json!({ "name": "sid", "domain": "example.com" });
|
||||
handle("deleteCookies", ¶ms, &mut ctx, &None)
|
||||
.await
|
||||
.expect("deleteCookies must succeed without a session");
|
||||
assert!(ctx.default_context.cookie_jar.get_all_cookies().is_empty());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn get_all_cookies_returns_every_cookie_in_jar() {
|
||||
let mut ctx = CdpContext::new();
|
||||
ctx.default_context.cookie_jar.set_cookies_from_cdp(vec![
|
||||
sample_cookie("a"),
|
||||
sample_cookie("b"),
|
||||
]);
|
||||
let resp = handle("getAllCookies", &json!({}), &mut ctx, &None)
|
||||
.await
|
||||
.expect("getAllCookies must succeed");
|
||||
let arr = resp["cookies"].as_array().expect("cookies array");
|
||||
assert_eq!(arr.len(), 2);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn get_cookies_falls_back_to_default_context_when_no_session() {
|
||||
let mut ctx = CdpContext::new();
|
||||
ctx.default_context
|
||||
.cookie_jar
|
||||
.set_cookies_from_cdp(vec![sample_cookie("sid")]);
|
||||
let resp = handle("getCookies", &json!({}), &mut ctx, &None)
|
||||
.await
|
||||
.expect("getCookies must succeed without a session");
|
||||
let arr = resp["cookies"].as_array().expect("cookies array");
|
||||
assert_eq!(arr.len(), 1);
|
||||
assert_eq!(arr[0]["name"], "sid");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn clear_browser_cookies_without_session_clears_default_context() {
|
||||
let mut ctx = CdpContext::new();
|
||||
ctx.default_context
|
||||
.cookie_jar
|
||||
.set_cookies_from_cdp(vec![sample_cookie("sid")]);
|
||||
handle("clearBrowserCookies", &json!({}), &mut ctx, &None)
|
||||
.await
|
||||
.expect("clearBrowserCookies must succeed");
|
||||
assert!(ctx.default_context.cookie_jar.get_all_cookies().is_empty());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn set_blocked_urls_targets_session_page_without_enabling_interception() {
|
||||
let mut ctx = CdpContext::new();
|
||||
let page_id = ctx.create_page();
|
||||
let session_id = Some("session-1".to_string());
|
||||
ctx.sessions.insert(session_id.clone().unwrap(), page_id.clone());
|
||||
|
||||
handle(
|
||||
"setBlockedURLs",
|
||||
&json!({
|
||||
"urls": [
|
||||
"*://*.example.com/*.png",
|
||||
"*://cdn.example.com/*"
|
||||
]
|
||||
}),
|
||||
&mut ctx,
|
||||
&session_id,
|
||||
)
|
||||
.await
|
||||
.expect("setBlockedURLs must succeed for a session page");
|
||||
|
||||
let page = ctx.get_page(&page_id).unwrap();
|
||||
assert_eq!(
|
||||
page.blocked_url_patterns,
|
||||
vec![
|
||||
"*://*.example.com/*.png".to_string(),
|
||||
"*://cdn.example.com/*".to_string(),
|
||||
]
|
||||
);
|
||||
assert!(!page.intercept_enabled);
|
||||
assert!(page.intercept_block_patterns.is_empty());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn set_blocked_urls_without_session_updates_existing_pages() {
|
||||
let mut ctx = CdpContext::new();
|
||||
let left = ctx.create_page();
|
||||
let right = ctx.create_page();
|
||||
|
||||
handle(
|
||||
"setBlockedURLs",
|
||||
&json!({ "urls": ["*://tiles.example.test/*"] }),
|
||||
&mut ctx,
|
||||
&None,
|
||||
)
|
||||
.await
|
||||
.expect("setBlockedURLs must succeed without a session");
|
||||
|
||||
assert_eq!(
|
||||
ctx.get_page(&left).unwrap().blocked_url_patterns,
|
||||
vec!["*://tiles.example.test/*".to_string()]
|
||||
);
|
||||
assert_eq!(
|
||||
ctx.get_page(&right).unwrap().blocked_url_patterns,
|
||||
vec!["*://tiles.example.test/*".to_string()]
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn set_blocked_urls_replaces_existing_patterns() {
|
||||
let mut ctx = CdpContext::new();
|
||||
let page_id = ctx.create_page();
|
||||
let session_id = Some("session-1".to_string());
|
||||
ctx.sessions.insert(session_id.clone().unwrap(), page_id.clone());
|
||||
|
||||
handle(
|
||||
"setBlockedURLs",
|
||||
&json!({ "urls": ["*://old.example.test/*"] }),
|
||||
&mut ctx,
|
||||
&session_id,
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
handle(
|
||||
"setBlockedURLs",
|
||||
&json!({ "urls": ["*://new.example.test/*"] }),
|
||||
&mut ctx,
|
||||
&session_id,
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(
|
||||
ctx.get_page(&page_id).unwrap().blocked_url_patterns,
|
||||
vec!["*://new.example.test/*".to_string()]
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn get_response_body_returns_stored_document_body() {
|
||||
let mut ctx = CdpContext::new();
|
||||
let page_id = ctx.create_page();
|
||||
let session_id = Some("session-1".to_string());
|
||||
ctx.sessions.insert(session_id.clone().unwrap(), page_id.clone());
|
||||
|
||||
let page = ctx.get_page_mut(&page_id).unwrap();
|
||||
page.navigate("data:text/html,<html><body>hello body</body></html>")
|
||||
.await
|
||||
.unwrap();
|
||||
let request_id = page.network_events[0].request_id.clone();
|
||||
|
||||
let result = handle(
|
||||
"getResponseBody",
|
||||
&json!({ "requestId": request_id }),
|
||||
&mut ctx,
|
||||
&session_id,
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(result["body"], "<html><body>hello body</body></html>");
|
||||
assert_eq!(result["base64Encoded"], false);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn get_response_body_errors_for_unknown_request_id() {
|
||||
let mut ctx = CdpContext::new();
|
||||
let err = handle(
|
||||
"getResponseBody",
|
||||
&json!({ "requestId": "missing" }),
|
||||
&mut ctx,
|
||||
&None,
|
||||
)
|
||||
.await
|
||||
.unwrap_err();
|
||||
|
||||
assert!(err.contains("missing"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn network_disable_clears_stored_response_bodies() {
|
||||
let mut ctx = CdpContext::new();
|
||||
let page_id = ctx.create_page();
|
||||
let session_id = Some("session-1".to_string());
|
||||
ctx.sessions.insert(session_id.clone().unwrap(), page_id.clone());
|
||||
|
||||
let page = ctx.get_page_mut(&page_id).unwrap();
|
||||
page.navigate("data:text/html,<html><body>temporary body</body></html>")
|
||||
.await
|
||||
.unwrap();
|
||||
let request_id = page.network_events[0].request_id.clone();
|
||||
|
||||
handle("disable", &json!({}), &mut ctx, &session_id)
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let err = handle(
|
||||
"getResponseBody",
|
||||
&json!({ "requestId": request_id }),
|
||||
&mut ctx,
|
||||
&session_id,
|
||||
)
|
||||
.await
|
||||
.unwrap_err();
|
||||
assert!(err.contains("No response body found"));
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn cookie_info_to_cdp_json(c: &obscura_net::CookieInfo) -> Value {
|
||||
let expires = c.expires.unwrap_or(SESSION_COOKIE_EXPIRES);
|
||||
let session = c.expires.is_none();
|
||||
let same_site = if c.same_site.is_empty() {
|
||||
DEFAULT_SAME_SITE
|
||||
} else {
|
||||
c.same_site.as_str()
|
||||
};
|
||||
json!({
|
||||
"name": c.name,
|
||||
"value": c.value,
|
||||
"domain": c.domain,
|
||||
"path": c.path,
|
||||
"expires": expires,
|
||||
"size": c.name.len() + c.value.len(),
|
||||
"httpOnly": c.http_only,
|
||||
"secure": c.secure,
|
||||
"session": session,
|
||||
"sameSite": same_site,
|
||||
"sameParty": false,
|
||||
"sourceScheme": if c.secure { SOURCE_SCHEME_SECURE } else { SOURCE_SCHEME_NONSECURE },
|
||||
"sourcePort": if c.secure { DEFAULT_SECURE_PORT } else { DEFAULT_INSECURE_PORT },
|
||||
"priority": "Medium",
|
||||
})
|
||||
}
|
||||
@@ -0,0 +1,684 @@
|
||||
use obscura_browser::lifecycle::WaitUntil;
|
||||
use serde_json::{json, Value};
|
||||
|
||||
use crate::dispatch::CdpContext;
|
||||
use crate::types::CdpEvent;
|
||||
use crate::util::url_is_file_scheme;
|
||||
|
||||
/// Emit the post-navigation event stream into `ctx.pending_events`. Shared
|
||||
/// by both the in-process `do_navigate` path and the spawned path in
|
||||
/// `server::process_navigation`, so the recent goto-returns-Response /
|
||||
/// per-isolated-world fixes don't have to be duplicated.
|
||||
pub fn emit_navigation_events(
|
||||
ctx: &mut CdpContext,
|
||||
session_id: &Option<String>,
|
||||
frame_id: &str,
|
||||
loader_id: &str,
|
||||
page_url: &str,
|
||||
page_id: &str,
|
||||
network_events: &[obscura_browser::NetworkEvent],
|
||||
wait_until: WaitUntil,
|
||||
reached_network_idle: bool,
|
||||
) {
|
||||
let es = session_id.clone();
|
||||
let ts = timestamp();
|
||||
|
||||
// Real Chrome uses the navigation's loaderId as the main document's
|
||||
// request id, and Puppeteer/Playwright identify the navigation response
|
||||
// via `requestId === loaderId && type === "Document"` (issue #189).
|
||||
let nav_request_ids: Vec<String> = {
|
||||
let mut nav_seen = false;
|
||||
network_events.iter().map(|ev| {
|
||||
if !nav_seen && ev.resource_type == "Document" && ev.url == page_url {
|
||||
nav_seen = true;
|
||||
loader_id.to_string()
|
||||
} else {
|
||||
ev.request_id.clone()
|
||||
}
|
||||
}).collect()
|
||||
};
|
||||
let nav_idx: Option<usize> = network_events
|
||||
.iter()
|
||||
.position(|ev| ev.resource_type == "Document" && ev.url == page_url);
|
||||
|
||||
// The main resource's body is stored under its internal request id, but the
|
||||
// client sees it as `loader_id` (the requestId we report above). Alias it so
|
||||
// Network.getResponseBody(loaderId) resolves, which is the only way a client
|
||||
// navigating straight to an image/PDF/other resource can read the main body
|
||||
// (issue #340). Also read the real Content-Type so frameNavigated reports the
|
||||
// actual mime instead of a hardcoded text/html.
|
||||
let mut nav_mime = "text/html".to_string();
|
||||
if let Some(idx) = nav_idx {
|
||||
let internal_id = &network_events[idx].request_id;
|
||||
if let Some(ct) = network_events[idx].response_headers.get("content-type") {
|
||||
// Strip any `; charset=...` parameter; frameNavigated wants the essence.
|
||||
nav_mime = ct.split(';').next().unwrap_or(ct).trim().to_string();
|
||||
}
|
||||
if internal_id != loader_id {
|
||||
if let Some(page) = ctx.get_page_mut(page_id) {
|
||||
page.alias_response_body(internal_id, loader_id);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Playwright needs `Network.requestWillBeSent` for the main document to
|
||||
// arrive BEFORE `Page.frameNavigated` (issue #190).
|
||||
if let Some(idx) = nav_idx {
|
||||
let net_event = &network_events[idx];
|
||||
let rid = &nav_request_ids[idx];
|
||||
ctx.pending_events.push(CdpEvent {
|
||||
method: "Network.requestWillBeSent".into(),
|
||||
params: json!({"requestId": rid, "loaderId": loader_id, "documentURL": page_url, "request": {"url": net_event.url, "method": net_event.method, "headers": net_event.headers}, "timestamp": net_event.timestamp, "wallTime": net_event.timestamp, "initiator": {"type": "other"}, "type": net_event.resource_type, "frameId": frame_id}),
|
||||
session_id: es.clone(),
|
||||
});
|
||||
}
|
||||
|
||||
let mut phase1 = vec![
|
||||
CdpEvent { method: "Page.lifecycleEvent".into(), params: json!({"frameId": frame_id, "loaderId": loader_id, "name": "init", "timestamp": ts}), session_id: es.clone() },
|
||||
CdpEvent { method: "Runtime.executionContextsCleared".into(), params: json!({}), session_id: es.clone() },
|
||||
CdpEvent { method: "Page.frameNavigated".into(), params: json!({"frame": {"id": frame_id, "loaderId": loader_id, "url": page_url, "domainAndRegistry": "", "securityOrigin": page_url, "mimeType": nav_mime, "adFrameStatus": {"adFrameType": "none"}}, "type": "Navigation"}), session_id: es.clone() },
|
||||
CdpEvent { method: "Runtime.executionContextCreated".into(), params: json!({"context": {"id": 2, "origin": page_url, "name": "", "uniqueId": format!("ctx-nav-{}", page_id), "auxData": {"isDefault": true, "type": "default", "frameId": frame_id}}}), session_id: es.clone() },
|
||||
];
|
||||
let world_names: Vec<String> = if ctx.isolated_worlds.is_empty() {
|
||||
vec!["__puppeteer_utility_world__24.40.0".to_string()]
|
||||
} else {
|
||||
ctx.isolated_worlds.clone()
|
||||
};
|
||||
// Issue #192: fresh, monotonically increasing executionContextId per re-create.
|
||||
for world_name in &world_names {
|
||||
let world_ctx_id = ctx.next_isolated_context();
|
||||
phase1.push(CdpEvent {
|
||||
method: "Runtime.executionContextCreated".into(),
|
||||
params: json!({"context": {"id": world_ctx_id, "origin": page_url, "name": world_name, "uniqueId": format!("ctx-isolated-nav-{}-{}", page_id, world_ctx_id), "auxData": {"isDefault": false, "type": "isolated", "frameId": frame_id}}}),
|
||||
session_id: es.clone(),
|
||||
});
|
||||
}
|
||||
phase1.push(CdpEvent { method: "Page.lifecycleEvent".into(), params: json!({"frameId": frame_id, "loaderId": loader_id, "name": "commit", "timestamp": ts}), session_id: es.clone() });
|
||||
ctx.pending_events.extend(phase1);
|
||||
|
||||
if ctx.fetch_intercept.enabled {
|
||||
for (i, net_event) in network_events.iter().enumerate() {
|
||||
let rid = &nav_request_ids[i];
|
||||
ctx.pending_events.push(CdpEvent {
|
||||
method: "Fetch.requestPaused".into(),
|
||||
params: json!({
|
||||
"requestId": rid,
|
||||
"request": {
|
||||
"url": net_event.url,
|
||||
"method": net_event.method,
|
||||
"headers": net_event.headers,
|
||||
},
|
||||
"frameId": frame_id,
|
||||
"resourceType": net_event.resource_type,
|
||||
"networkId": rid,
|
||||
}),
|
||||
session_id: es.clone(),
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
for (i, net_event) in network_events.iter().enumerate() {
|
||||
let rid = &nav_request_ids[i];
|
||||
if Some(i) != nav_idx {
|
||||
ctx.pending_events.push(CdpEvent {
|
||||
method: "Network.requestWillBeSent".into(),
|
||||
params: json!({"requestId": rid, "loaderId": loader_id, "documentURL": page_url, "request": {"url": net_event.url, "method": net_event.method, "headers": net_event.headers}, "timestamp": net_event.timestamp, "wallTime": net_event.timestamp, "initiator": {"type": "other"}, "type": net_event.resource_type, "frameId": frame_id}),
|
||||
session_id: es.clone(),
|
||||
});
|
||||
}
|
||||
ctx.pending_events.push(CdpEvent {
|
||||
method: "Network.responseReceived".into(),
|
||||
params: json!({"requestId": rid, "loaderId": loader_id, "timestamp": net_event.timestamp, "type": net_event.resource_type, "response": {"url": net_event.url, "status": net_event.status, "statusText": "", "headers": &*net_event.response_headers, "mimeType": net_event.response_headers.get("content-type").cloned().unwrap_or_default()}, "frameId": frame_id}),
|
||||
session_id: es.clone(),
|
||||
});
|
||||
ctx.pending_events.push(CdpEvent {
|
||||
method: "Network.loadingFinished".into(),
|
||||
params: json!({"requestId": rid, "timestamp": net_event.timestamp, "encodedDataLength": net_event.body_size}),
|
||||
session_id: es.clone(),
|
||||
});
|
||||
}
|
||||
|
||||
let mut phase3 = vec![
|
||||
CdpEvent { method: "Page.lifecycleEvent".into(), params: json!({"frameId": frame_id, "loaderId": loader_id, "name": "DOMContentLoaded", "timestamp": ts}), session_id: es.clone() },
|
||||
CdpEvent { method: "Page.domContentEventFired".into(), params: json!({"timestamp": ts}), session_id: es.clone() },
|
||||
CdpEvent { method: "Page.lifecycleEvent".into(), params: json!({"frameId": frame_id, "loaderId": loader_id, "name": "load", "timestamp": ts}), session_id: es.clone() },
|
||||
CdpEvent { method: "Page.loadEventFired".into(), params: json!({"timestamp": ts}), session_id: es.clone() },
|
||||
];
|
||||
if reached_network_idle || matches!(wait_until, WaitUntil::Load | WaitUntil::DomContentLoaded) {
|
||||
let idle_ts = timestamp();
|
||||
phase3.push(CdpEvent { method: "Page.lifecycleEvent".into(), params: json!({"frameId": frame_id, "loaderId": loader_id, "name": "networkIdle", "timestamp": idle_ts}), session_id: es.clone() });
|
||||
}
|
||||
phase3.push(CdpEvent { method: "Page.frameStoppedLoading".into(), params: json!({"frameId": frame_id}), session_id: es });
|
||||
ctx.pending_events.extend(phase3);
|
||||
|
||||
// Target.targetInfoChanged: strict CDP clients (browser-use, and
|
||||
// Puppeteer/Playwright `page.url()` tracking) cache the TargetInfo from
|
||||
// attachedToTarget and only refresh it on this event. Without it they keep
|
||||
// reporting the pre-navigation url/title (about:blank) and never see the
|
||||
// loaded page. Emit it browser-level (no sessionId) with the new url/title.
|
||||
let (tic_title, tic_ctx) = ctx
|
||||
.get_page(page_id)
|
||||
.map(|p| (p.title.clone(), p.context.id.clone()))
|
||||
.unwrap_or_default();
|
||||
ctx.pending_events.push(CdpEvent::new(
|
||||
"Target.targetInfoChanged",
|
||||
json!({
|
||||
"targetInfo": {
|
||||
"targetId": page_id,
|
||||
"type": "page",
|
||||
"title": tic_title,
|
||||
"url": page_url,
|
||||
"attached": true,
|
||||
"canAccessOpener": false,
|
||||
"browserContextId": tic_ctx,
|
||||
}
|
||||
}),
|
||||
));
|
||||
}
|
||||
|
||||
/// Parse the `waitUntil` argument that Puppeteer/Playwright pass on
|
||||
/// `Page.navigate`.
|
||||
pub fn parse_wait_until(params: &Value) -> WaitUntil {
|
||||
params
|
||||
.get("waitUntil")
|
||||
.and_then(|v| {
|
||||
if let Some(s) = v.as_str() {
|
||||
Some(WaitUntil::from_str(s))
|
||||
} else if let Some(arr) = v.as_array() {
|
||||
arr.iter()
|
||||
.filter_map(|item| item.as_str())
|
||||
.map(WaitUntil::from_str)
|
||||
.max_by_key(|w| match w {
|
||||
WaitUntil::DomContentLoaded => 0,
|
||||
WaitUntil::Load => 1,
|
||||
WaitUntil::NetworkIdle2 => 2,
|
||||
WaitUntil::NetworkIdle0 => 3,
|
||||
})
|
||||
} else {
|
||||
None
|
||||
}
|
||||
})
|
||||
// Puppeteer and Playwright drive navigation via `Page.navigate`
|
||||
// without a server-side waitUntil — they wait for `Page.lifecycleEvent`
|
||||
// on the client side. Defaulting the server to `Load` means we run
|
||||
// every parser/deferred/async script on JS-heavy pages before
|
||||
// emitting `load`, which on sites like github.com / reddit.com
|
||||
// pushes nav past 25s and clients time out at 15s. Real Chrome
|
||||
// streams `DOMContentLoaded` as soon as the parser is done; we
|
||||
// batch our event emission at the end of navigation, so the
|
||||
// closest we can get is to default to `DomContentLoaded` and skip
|
||||
// the full-load wait. CLI callers that pass `--wait-until load`
|
||||
// (or `networkidle*`) are unaffected; they get the old behaviour.
|
||||
.unwrap_or(WaitUntil::DomContentLoaded)
|
||||
}
|
||||
|
||||
async fn do_navigate(
|
||||
url: &str,
|
||||
params: &Value,
|
||||
ctx: &mut CdpContext,
|
||||
session_id: &Option<String>,
|
||||
) -> Result<Value, String> {
|
||||
let wait_until = parse_wait_until(params);
|
||||
|
||||
// Block CDP-initiated file:// navigation by default.
|
||||
// Anyone who can reach the CDP port (default localhost,
|
||||
// but Docker images bind 0.0.0.0) could otherwise read
|
||||
// any file the obscura process can read. Opt in via
|
||||
// `obscura serve --allow-file-access` when local-HTML
|
||||
// testing is the intended workflow.
|
||||
if url_is_file_scheme(url) && !ctx.default_context.allow_file_access {
|
||||
return Err(
|
||||
"Page.navigate to file:// is disabled. Restart with `obscura serve --allow-file-access` to enable.".to_string()
|
||||
);
|
||||
}
|
||||
|
||||
let preload_scripts: Vec<String> = ctx.preload_scripts.iter().map(|(_, s)| s.clone()).collect();
|
||||
|
||||
let (frame_id, loader_id, network_events, page_url, page_id, reached_network_idle) = {
|
||||
let page = ctx.get_session_page_mut(session_id).ok_or("No page for session")?;
|
||||
let frame_id = page.frame_id.clone();
|
||||
let loader_id = format!("loader-{}", uuid::Uuid::new_v4());
|
||||
|
||||
// Preloads (addBinding shims, addScriptToEvaluateOnNewDocument sources)
|
||||
// must run BEFORE the page's own scripts (CDP contract). Hand them to
|
||||
// the page so navigate_single can inject them at the right point.
|
||||
page.set_preload_scripts(preload_scripts);
|
||||
|
||||
let nav_method = params.get("__method").and_then(|v| v.as_str()).unwrap_or("GET");
|
||||
let nav_body = params.get("__body").and_then(|v| v.as_str()).unwrap_or("");
|
||||
if nav_method == "POST" && !nav_body.is_empty() {
|
||||
page.navigate_with_wait_post(url, wait_until, nav_method, nav_body).await.map_err(|e| e.to_string())?;
|
||||
} else {
|
||||
page.navigate_with_wait(url, wait_until).await.map_err(|e| e.to_string())?;
|
||||
}
|
||||
|
||||
let reached_network_idle = page.lifecycle.is_network_idle();
|
||||
let network_events: Vec<_> = page.network_events.drain(..).collect();
|
||||
let page_url = page.url_string();
|
||||
let page_id = page.id.clone();
|
||||
(frame_id, loader_id, network_events, page_url, page_id, reached_network_idle)
|
||||
};
|
||||
|
||||
emit_navigation_events(
|
||||
ctx,
|
||||
session_id,
|
||||
&frame_id,
|
||||
&loader_id,
|
||||
&page_url,
|
||||
&page_id,
|
||||
&network_events,
|
||||
wait_until,
|
||||
reached_network_idle,
|
||||
);
|
||||
|
||||
Ok(json!({
|
||||
"frameId": frame_id,
|
||||
"loaderId": loader_id,
|
||||
}))
|
||||
}
|
||||
|
||||
pub async fn handle(
|
||||
method: &str,
|
||||
params: &Value,
|
||||
ctx: &mut CdpContext,
|
||||
session_id: &Option<String>,
|
||||
) -> Result<Value, String> {
|
||||
match method {
|
||||
"enable" => Ok(json!({})),
|
||||
"navigate" => {
|
||||
let url = params.get("url").and_then(|v| v.as_str())
|
||||
.ok_or("url required")?;
|
||||
do_navigate(url, params, ctx, session_id).await
|
||||
}
|
||||
"reload" => {
|
||||
let current_url = ctx.get_session_page(session_id)
|
||||
.map(|p| p.url_string())
|
||||
.unwrap_or_else(|| "about:blank".to_string());
|
||||
let reload_params = json!({
|
||||
"waitUntil": params.get("waitUntil").cloned().unwrap_or(json!("load"))
|
||||
});
|
||||
do_navigate(¤t_url, &reload_params, ctx, session_id).await
|
||||
}
|
||||
"getFrameTree" => {
|
||||
let page = ctx.get_session_page(session_id).ok_or("No page for session")?;
|
||||
Ok(json!({
|
||||
"frameTree": {
|
||||
"frame": {
|
||||
"id": page.frame_id,
|
||||
"loaderId": "initial-loader",
|
||||
"url": page.url_string(),
|
||||
"domainAndRegistry": "",
|
||||
"securityOrigin": page.url_string(),
|
||||
"mimeType": "text/html",
|
||||
"adFrameStatus": { "adFrameType": "none" },
|
||||
},
|
||||
"childFrames": [],
|
||||
}
|
||||
}))
|
||||
}
|
||||
"createIsolatedWorld" => {
|
||||
let (frame_id_param, world_name, page_url, page_id) = {
|
||||
let page = ctx.get_session_page(session_id).ok_or("No page for session")?;
|
||||
(
|
||||
params.get("frameId").and_then(|v| v.as_str())
|
||||
.unwrap_or(&page.frame_id).to_string(),
|
||||
params.get("worldName").and_then(|v| v.as_str())
|
||||
.unwrap_or("").to_string(),
|
||||
page.url_string(),
|
||||
page.id.clone(),
|
||||
)
|
||||
};
|
||||
// Track this world so Page.navigate can re-emit a context for it
|
||||
// post-navigation. Without this, Playwright (and Puppeteer)
|
||||
// hang in any operation that uses the utility world — including
|
||||
// page.title() — because their utility world is gone after
|
||||
// Runtime.executionContextsCleared and never re-created.
|
||||
if !world_name.is_empty() && !ctx.isolated_worlds.contains(&world_name) {
|
||||
ctx.isolated_worlds.push(world_name.clone());
|
||||
}
|
||||
// Issue #192: every isolated world emission gets a fresh id from
|
||||
// the monotonic counter and is registered as a valid contextId.
|
||||
// Reusing id 100 across navigations made Playwright's bookkeeping
|
||||
// diverge (it expected 101 on the second nav) and Runtime.evaluate
|
||||
// failed with "Cannot find context with specified id: 101".
|
||||
let context_id = ctx.next_isolated_context();
|
||||
|
||||
ctx.pending_events.push(CdpEvent {
|
||||
method: "Runtime.executionContextCreated".to_string(),
|
||||
params: json!({
|
||||
"context": {
|
||||
"id": context_id,
|
||||
"origin": page_url,
|
||||
"name": world_name,
|
||||
"uniqueId": format!("ctx-isolated-{}-{}", page_id, context_id),
|
||||
"auxData": {
|
||||
"isDefault": false,
|
||||
"type": "isolated",
|
||||
"frameId": frame_id_param,
|
||||
}
|
||||
}
|
||||
}),
|
||||
session_id: session_id.clone(),
|
||||
});
|
||||
|
||||
Ok(json!({ "executionContextId": context_id }))
|
||||
}
|
||||
"setLifecycleEventsEnabled" => Ok(json!({})),
|
||||
"addScriptToEvaluateOnNewDocument" => {
|
||||
let source = params.get("source").and_then(|v| v.as_str()).unwrap_or("");
|
||||
ctx.preload_counter += 1;
|
||||
let identifier = format!("{}", ctx.preload_counter);
|
||||
if !source.is_empty() {
|
||||
ctx.preload_scripts.push((identifier.clone(), source.to_string()));
|
||||
}
|
||||
Ok(json!({ "identifier": identifier }))
|
||||
}
|
||||
"removeScriptToEvaluateOnNewDocument" => {
|
||||
let identifier = params.get("identifier").and_then(|v| v.as_str()).unwrap_or("");
|
||||
ctx.preload_scripts.retain(|(id, _)| id != identifier);
|
||||
Ok(json!({}))
|
||||
}
|
||||
"setInterceptFileChooserDialog" => Ok(json!({})),
|
||||
// Obscura does not download files to disk, so there is no behavior to
|
||||
// configure; ack it so clients that set it do not warn (issue #340).
|
||||
"setDownloadBehavior" => Ok(json!({})),
|
||||
"getLayoutMetrics" => {
|
||||
// Obscura has no visual layout engine, so we return a fixed
|
||||
// 1280x720 viewport (Chrome's default) and try to derive the
|
||||
// content height from document.documentElement.scrollHeight.
|
||||
// Playwright calls this before every page.screenshot() and
|
||||
// would otherwise fail with "Unknown Page method".
|
||||
let width = 1280.0_f64;
|
||||
let height = 720.0_f64;
|
||||
let content_height = ctx
|
||||
.get_session_page_mut(session_id)
|
||||
.map(|p| p.evaluate("document.documentElement && document.documentElement.scrollHeight"))
|
||||
.and_then(|v| v.as_f64())
|
||||
.filter(|n| *n > 0.0)
|
||||
.unwrap_or(height);
|
||||
let layout_viewport = json!({
|
||||
"pageX": 0, "pageY": 0,
|
||||
"clientWidth": width, "clientHeight": height,
|
||||
});
|
||||
let visual_viewport = json!({
|
||||
"offsetX": 0.0, "offsetY": 0.0,
|
||||
"pageX": 0.0, "pageY": 0.0,
|
||||
"clientWidth": width, "clientHeight": height,
|
||||
"scale": 1.0, "zoom": 1.0,
|
||||
});
|
||||
let content_size = json!({
|
||||
"x": 0.0, "y": 0.0,
|
||||
"width": width, "height": content_height,
|
||||
});
|
||||
Ok(json!({
|
||||
"layoutViewport": layout_viewport,
|
||||
"visualViewport": visual_viewport,
|
||||
"contentSize": content_size,
|
||||
"cssLayoutViewport": layout_viewport,
|
||||
"cssVisualViewport": visual_viewport,
|
||||
"cssContentSize": content_size,
|
||||
}))
|
||||
}
|
||||
"getNavigationHistory" => {
|
||||
let page = ctx.get_session_page(session_id).ok_or("No page for session")?;
|
||||
// Synthesize an entry for the current page when history is empty
|
||||
// (initial about:blank, never-navigated targets). Puppeteer's
|
||||
// goBack reads `currentIndex` and `entries[currentIndex-1]`;
|
||||
// an empty entries[] used to make every back/forward fail.
|
||||
let entries: Vec<Value> = if page.history.is_empty() {
|
||||
vec![json!({
|
||||
"id": 0,
|
||||
"url": page.url_string(),
|
||||
"userTypedURL": page.url_string(),
|
||||
"title": page.title,
|
||||
"transitionType": "typed",
|
||||
})]
|
||||
} else {
|
||||
page.history.iter().enumerate().map(|(i, url)| json!({
|
||||
"id": i as u64,
|
||||
"url": url,
|
||||
"userTypedURL": url,
|
||||
"title": if i == page.history_index { page.title.clone() } else { String::new() },
|
||||
"transitionType": "typed",
|
||||
})).collect()
|
||||
};
|
||||
Ok(json!({
|
||||
"currentIndex": if page.history.is_empty() { 0 } else { page.history_index },
|
||||
"entries": entries,
|
||||
}))
|
||||
}
|
||||
"navigateToHistoryEntry" => {
|
||||
let entry_id = params.get("entryId").and_then(|v| v.as_u64()).unwrap_or(0) as usize;
|
||||
let target_url = {
|
||||
let page = ctx.get_session_page_mut(session_id).ok_or("No page for session")?;
|
||||
let url = page.history.get(entry_id).cloned();
|
||||
if url.is_some() {
|
||||
page.set_history_index(entry_id);
|
||||
}
|
||||
url
|
||||
};
|
||||
if let Some(url) = target_url {
|
||||
// Stash + restore history so push_history doesn't clobber
|
||||
// the cursor we just moved.
|
||||
let stash = {
|
||||
let page = ctx.get_session_page_mut(session_id).ok_or("No page for session")?;
|
||||
(page.history.clone(), page.history_index)
|
||||
};
|
||||
let (frame_id, page_id, network_events, page_url, reached_idle) = {
|
||||
let page = ctx.get_session_page_mut(session_id).ok_or("No page for session")?;
|
||||
page.navigate_with_wait(&url, WaitUntil::DomContentLoaded).await.map_err(|e| e.to_string())?;
|
||||
page.history = stash.0;
|
||||
page.history_index = stash.1;
|
||||
(
|
||||
page.frame_id.clone(),
|
||||
page.id.clone(),
|
||||
page.network_events.drain(..).collect::<Vec<_>>(),
|
||||
page.url_string(),
|
||||
page.lifecycle.is_network_idle(),
|
||||
)
|
||||
};
|
||||
let loader_id = format!("loader-{}", uuid::Uuid::new_v4());
|
||||
emit_navigation_events(
|
||||
ctx, session_id,
|
||||
&frame_id, &loader_id, &page_url, &page_id,
|
||||
&network_events, WaitUntil::DomContentLoaded, reached_idle,
|
||||
);
|
||||
}
|
||||
Ok(json!({}))
|
||||
}
|
||||
"resetNavigationHistory" => {
|
||||
if let Some(page) = ctx.get_session_page_mut(session_id) {
|
||||
page.history.clear();
|
||||
page.history_index = 0;
|
||||
}
|
||||
Ok(json!({}))
|
||||
}
|
||||
"printToPDF" => {
|
||||
// Obscura has no layout/rendering engine, so PDF generation is
|
||||
// intentionally not implemented. Returning a distinct, descriptive
|
||||
// error (rather than the generic "Unknown Page method" fallback)
|
||||
// tells Playwright/Puppeteer/headless_chrome clients exactly why
|
||||
// the call failed and what to do instead.
|
||||
Err(
|
||||
"Page.printToPDF is not supported by Obscura: no layout engine. \
|
||||
Use Runtime.evaluate (e.g. page.evaluate) to extract the rendered \
|
||||
HTML, then render to PDF in your client (wkhtmltopdf, weasyprint, \
|
||||
a separate headless Chromium pipeline, etc.)."
|
||||
.to_string(),
|
||||
)
|
||||
}
|
||||
"captureScreenshot" | "captureSnapshot" => {
|
||||
// Same story as printToPDF: rasterising a page needs a layout and
|
||||
// paint pipeline that Obscura intentionally does not have. Reply
|
||||
// with a clear error so clients can fail fast instead of waiting
|
||||
// on the generic "Unknown Page method" reply.
|
||||
Err(format!(
|
||||
"Page.{method} is not supported by Obscura: no layout or paint engine. \
|
||||
For visual snapshots, drive a real headless Chromium for the \
|
||||
screenshot leg of your pipeline and use Obscura for the scraping leg."
|
||||
))
|
||||
}
|
||||
_ => Err(format!("Unknown Page method: {}", method)),
|
||||
}
|
||||
}
|
||||
|
||||
fn timestamp() -> f64 {
|
||||
std::time::SystemTime::now()
|
||||
.duration_since(std::time::UNIX_EPOCH)
|
||||
.unwrap_or_default()
|
||||
.as_secs_f64()
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::dispatch::CdpContext;
|
||||
|
||||
#[tokio::test]
|
||||
async fn get_layout_metrics_returns_chrome_default_viewport() {
|
||||
let mut ctx = CdpContext::new();
|
||||
let result = handle("getLayoutMetrics", &json!({}), &mut ctx, &None)
|
||||
.await
|
||||
.expect("getLayoutMetrics should succeed without a session");
|
||||
|
||||
// CDP spec requires three top-level shapes; Playwright's screenshot
|
||||
// path reads contentSize.width/height to size the capture. Without
|
||||
// them the screenshot call panics with "cannot read property of
|
||||
// undefined".
|
||||
for key in [
|
||||
"layoutViewport",
|
||||
"visualViewport",
|
||||
"contentSize",
|
||||
"cssLayoutViewport",
|
||||
"cssVisualViewport",
|
||||
"cssContentSize",
|
||||
] {
|
||||
assert!(result.get(key).is_some(), "missing key: {key}");
|
||||
}
|
||||
|
||||
let layout = &result["layoutViewport"];
|
||||
assert_eq!(layout["clientWidth"].as_f64(), Some(1280.0));
|
||||
assert_eq!(layout["clientHeight"].as_f64(), Some(720.0));
|
||||
|
||||
let visual = &result["visualViewport"];
|
||||
assert_eq!(visual["scale"].as_f64(), Some(1.0));
|
||||
assert_eq!(visual["clientWidth"].as_f64(), Some(1280.0));
|
||||
|
||||
let content = &result["contentSize"];
|
||||
assert_eq!(content["width"].as_f64(), Some(1280.0));
|
||||
// Without a live page the content height falls back to the viewport.
|
||||
assert_eq!(content["height"].as_f64(), Some(720.0));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn unknown_page_method_still_errors() {
|
||||
let mut ctx = CdpContext::new();
|
||||
let err = handle("notARealMethod", &json!({}), &mut ctx, &None)
|
||||
.await
|
||||
.expect_err("unknown methods must surface as errors");
|
||||
assert!(err.contains("Unknown Page method"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn print_to_pdf_returns_descriptive_unsupported_error() {
|
||||
// Regression for #53: Page.printToPDF must be handled explicitly so
|
||||
// Playwright clients receive a descriptive error rather than the
|
||||
// generic "Unknown Page method" fallback.
|
||||
let mut ctx = CdpContext::new();
|
||||
let err = handle("printToPDF", &json!({}), &mut ctx, &None)
|
||||
.await
|
||||
.expect_err("printToPDF must error until a real renderer exists");
|
||||
assert!(
|
||||
!err.contains("Unknown Page method"),
|
||||
"printToPDF must NOT fall through to the catch-all: {err}"
|
||||
);
|
||||
assert!(
|
||||
err.contains("not supported by Obscura"),
|
||||
"error must clearly state PDF is unsupported: {err}"
|
||||
);
|
||||
// Direct user to a workaround so the message is actionable.
|
||||
assert!(
|
||||
err.to_lowercase().contains("evaluate")
|
||||
|| err.to_lowercase().contains("html"),
|
||||
"error must point to a workaround: {err}"
|
||||
);
|
||||
}
|
||||
|
||||
/// Regression for #45: same idea as printToPDF for captureScreenshot.
|
||||
/// Playwright's `page.screenshot()` calls Page.captureScreenshot via CDP;
|
||||
/// without an explicit arm, clients see "Unknown Page method" and have
|
||||
/// no idea why their screenshot request failed.
|
||||
#[tokio::test]
|
||||
async fn capture_screenshot_returns_descriptive_unsupported_error() {
|
||||
let mut ctx = CdpContext::new();
|
||||
let err = handle("captureScreenshot", &json!({}), &mut ctx, &None)
|
||||
.await
|
||||
.expect_err("captureScreenshot must error until a real paint exists");
|
||||
assert!(
|
||||
!err.contains("Unknown Page method"),
|
||||
"captureScreenshot must NOT fall through to the catch-all: {err}"
|
||||
);
|
||||
assert!(
|
||||
err.contains("not supported by Obscura"),
|
||||
"error must clearly state screenshot is unsupported: {err}"
|
||||
);
|
||||
// Same for the MHTML snapshot sibling method.
|
||||
let err2 = handle("captureSnapshot", &json!({}), &mut ctx, &None)
|
||||
.await
|
||||
.expect_err("captureSnapshot must error until a real renderer exists");
|
||||
assert!(
|
||||
!err2.contains("Unknown Page method"),
|
||||
"captureSnapshot must NOT fall through: {err2}"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn navigation_emits_target_info_changed_with_url_and_title() {
|
||||
// Strict CDP clients (browser-use, Puppeteer/Playwright `page.url()`)
|
||||
// refresh a target's url/title only on Target.targetInfoChanged. A
|
||||
// navigation must emit it with the post-nav url/title, otherwise those
|
||||
// clients stay stuck on the pre-nav about:blank.
|
||||
let mut ctx = CdpContext::new();
|
||||
let page_id = ctx.create_page();
|
||||
let session_id = format!("{}-session", page_id);
|
||||
ctx.sessions.insert(session_id.clone(), page_id.clone());
|
||||
|
||||
let params = json!({
|
||||
"url": "data:text/html,<title>Hello</title><button>B</button>",
|
||||
"waitUntil": "load",
|
||||
});
|
||||
handle("navigate", ¶ms, &mut ctx, &Some(session_id.clone()))
|
||||
.await
|
||||
.expect("navigate should succeed");
|
||||
|
||||
let evt = ctx
|
||||
.pending_events
|
||||
.iter()
|
||||
.find(|e| e.method == "Target.targetInfoChanged")
|
||||
.expect("navigation must emit Target.targetInfoChanged");
|
||||
// Browser-level event (no sessionId) so the root connection's
|
||||
// targetInfoChanged handler receives it.
|
||||
assert!(
|
||||
evt.session_id.is_none(),
|
||||
"targetInfoChanged must be browser-level (no sessionId)"
|
||||
);
|
||||
let info = evt.params["targetInfo"].clone();
|
||||
|
||||
// The payload must carry the live post-navigation url/title and the
|
||||
// canAccessOpener field strict clients require on every TargetInfo.
|
||||
let (exp_url, exp_title) = {
|
||||
let page = ctx.get_page(&page_id).expect("page exists");
|
||||
(page.url_string(), page.title.clone())
|
||||
};
|
||||
assert_eq!(info["targetId"], json!(page_id));
|
||||
assert_eq!(info["type"], "page");
|
||||
assert_eq!(info["url"], json!(exp_url));
|
||||
assert_eq!(info["title"], json!(exp_title));
|
||||
assert!(
|
||||
info["url"].as_str().unwrap_or_default().starts_with("data:"),
|
||||
"url should reflect the navigated page, got {}",
|
||||
info["url"]
|
||||
);
|
||||
assert_eq!(info["canAccessOpener"], json!(false));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,530 @@
|
||||
use obscura_browser::lifecycle::WaitUntil;
|
||||
use obscura_js::runtime::RemoteObjectInfo;
|
||||
use serde_json::{json, Value};
|
||||
|
||||
use crate::dispatch::CdpContext;
|
||||
|
||||
/// Drain pending JS-initiated navigation (form.submit, location.assign, etc),
|
||||
/// then emit the same CDP nav-event sequence Page.navigate emits so
|
||||
/// Puppeteer's waitForNavigation / Playwright's wait_for_url resolves.
|
||||
/// Without this, in-page navigations look like Runtime.evaluate finishing
|
||||
/// to clients and they hang waiting for a frameNavigated that never fires.
|
||||
async fn emit_post_eval_nav(
|
||||
ctx: &mut CdpContext,
|
||||
session_id: &Option<String>,
|
||||
) -> Result<(), String> {
|
||||
let page = ctx
|
||||
.get_session_page_mut(session_id)
|
||||
.ok_or("No page")?;
|
||||
let did_navigate = page.process_pending_navigation().await.map_err(|e| e.to_string())?;
|
||||
if !did_navigate {
|
||||
return Ok(());
|
||||
}
|
||||
let (frame_id, page_url, page_id, network_events, reached_idle) = {
|
||||
let p = ctx.get_session_page_mut(session_id).ok_or("No page")?;
|
||||
(
|
||||
p.frame_id.clone(),
|
||||
p.url_string(),
|
||||
p.id.clone(),
|
||||
p.network_events.drain(..).collect::<Vec<_>>(),
|
||||
p.lifecycle.is_network_idle(),
|
||||
)
|
||||
};
|
||||
let loader_id = format!("loader-{}", uuid::Uuid::new_v4());
|
||||
super::page::emit_navigation_events(
|
||||
ctx,
|
||||
session_id,
|
||||
&frame_id,
|
||||
&loader_id,
|
||||
&page_url,
|
||||
&page_id,
|
||||
&network_events,
|
||||
WaitUntil::Load,
|
||||
reached_idle,
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub async fn handle(
|
||||
method: &str,
|
||||
params: &Value,
|
||||
ctx: &mut CdpContext,
|
||||
session_id: &Option<String>,
|
||||
) -> Result<Value, String> {
|
||||
match method {
|
||||
"enable" => {
|
||||
// puppeteer-extra's FrameManager.initialize calls Runtime.enable on
|
||||
// the browser-level connection BEFORE any page target exists. Real
|
||||
// Chrome replies with `{}` and emits executionContextCreated when
|
||||
// a context appears. Returning "No page" here breaks the standard
|
||||
// puppeteer connect/newPage flow. If there's no session, succeed
|
||||
// silently — the next Target.attachToTarget will set things up.
|
||||
match ctx.get_session_page(session_id) {
|
||||
Some(page) => {
|
||||
let event = crate::types::CdpEvent {
|
||||
method: "Runtime.executionContextCreated".to_string(),
|
||||
params: json!({
|
||||
"context": {
|
||||
"id": 1,
|
||||
"origin": page.url_string(),
|
||||
"name": "",
|
||||
"uniqueId": format!("ctx-{}", page.id),
|
||||
"auxData": {
|
||||
"isDefault": true,
|
||||
"type": "default",
|
||||
"frameId": page.frame_id,
|
||||
}
|
||||
}
|
||||
}),
|
||||
session_id: session_id.clone(),
|
||||
};
|
||||
ctx.pending_events.push(event);
|
||||
}
|
||||
None => {
|
||||
// No session attached yet — that's fine. Just ack.
|
||||
}
|
||||
}
|
||||
Ok(json!({}))
|
||||
}
|
||||
"evaluate" => {
|
||||
let expression = params
|
||||
.get("expression")
|
||||
.and_then(|v| v.as_str())
|
||||
.ok_or("expression required")?;
|
||||
let return_by_value = params
|
||||
.get("returnByValue")
|
||||
.and_then(|v| v.as_bool())
|
||||
.unwrap_or(false);
|
||||
|
||||
validate_context_id(params, "contextId", ctx, "evaluate")?;
|
||||
|
||||
let await_promise = params
|
||||
.get("awaitPromise")
|
||||
.and_then(|v| v.as_bool())
|
||||
.unwrap_or(false);
|
||||
|
||||
// CDP `timeout` field (milliseconds). Default to Chrome's
|
||||
// protocolTimeout (30s) so long evaluations don't pin the V8 lock
|
||||
// indefinitely and starve every other CDP command on the same
|
||||
// session.
|
||||
let timeout_ms = params
|
||||
.get("timeout")
|
||||
.and_then(|v| v.as_u64())
|
||||
.unwrap_or(30_000);
|
||||
|
||||
let page = ctx
|
||||
.get_session_page_mut(session_id)
|
||||
.ok_or("No page")?;
|
||||
let info = match tokio::time::timeout(
|
||||
std::time::Duration::from_millis(timeout_ms),
|
||||
page.evaluate_for_cdp(expression, return_by_value, await_promise),
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(info) => info,
|
||||
Err(_) => {
|
||||
return Err(format!(
|
||||
"Runtime.evaluate exceeded {timeout_ms}ms timeout"
|
||||
));
|
||||
}
|
||||
};
|
||||
emit_post_eval_nav(ctx, session_id).await?;
|
||||
|
||||
Ok(json!({ "result": remote_object_from_info(&info) }))
|
||||
}
|
||||
"callFunctionOn" => {
|
||||
let function_declaration = params
|
||||
.get("functionDeclaration")
|
||||
.and_then(|v| v.as_str())
|
||||
.unwrap_or("() => undefined");
|
||||
let return_by_value = params
|
||||
.get("returnByValue")
|
||||
.and_then(|v| v.as_bool())
|
||||
.unwrap_or(false);
|
||||
let await_promise = params
|
||||
.get("awaitPromise")
|
||||
.and_then(|v| v.as_bool())
|
||||
.unwrap_or(false);
|
||||
let object_id = params.get("objectId").and_then(|v| v.as_str());
|
||||
let arguments = params
|
||||
.get("arguments")
|
||||
.and_then(|v| v.as_array())
|
||||
.map(|a| a.to_vec())
|
||||
.unwrap_or_default();
|
||||
|
||||
// #51: validate executionContextId the same way Runtime.evaluate
|
||||
// does. CDP names this field `executionContextId` on
|
||||
// callFunctionOn (not `contextId`); a request may omit it when
|
||||
// `objectId` is supplied — in that case validate_context_id is a
|
||||
// no-op and the default context is used.
|
||||
validate_context_id(params, "executionContextId", ctx, "callFunctionOn")?;
|
||||
|
||||
let page = ctx
|
||||
.get_session_page_mut(session_id)
|
||||
.ok_or("No page")?;
|
||||
let info =
|
||||
page.call_function_on_for_cdp(function_declaration, object_id, &arguments, return_by_value, await_promise).await;
|
||||
emit_post_eval_nav(ctx, session_id).await?;
|
||||
|
||||
Ok(json!({ "result": remote_object_from_info(&info) }))
|
||||
}
|
||||
"getProperties" => {
|
||||
// Puppeteer's $$() flow:
|
||||
// 1. evaluate querySelectorAll → handle for the NodeList
|
||||
// 2. getProperties on that handle → indexed items
|
||||
// 3. For each item, JSHandle.asElement() checks subtype === 'node';
|
||||
// if true, wraps as ElementHandle (with click/type/etc).
|
||||
//
|
||||
// Older impl returned the raw value via JSON, dropping the node
|
||||
// identity. Items came back as `{type:'object'}` with no objectId
|
||||
// and no subtype, so asElement returned null and the caller got
|
||||
// plain JSHandles back from page.$$ -- breaking checkboxes[0].click().
|
||||
//
|
||||
// We now:
|
||||
// 1. Walk the underlying object in JS, allocating a stable child
|
||||
// oid per (parent_oid + index) and stashing each value in
|
||||
// __obscura_objects so later callFunctionOn can resolve it.
|
||||
// 2. Annotate each item with subtype:'node' + className when the
|
||||
// value has a numeric nodeType, so Puppeteer wraps it as
|
||||
// ElementHandle.
|
||||
let object_id = params.get("objectId").and_then(|v| v.as_str());
|
||||
if let Some(oid) = object_id {
|
||||
let page = ctx
|
||||
.get_session_page_mut(session_id)
|
||||
.ok_or("No page")?;
|
||||
let escaped_oid = oid.replace('\\', "\\\\").replace('\'', "\\'");
|
||||
let code = format!(
|
||||
"(function() {{\
|
||||
var obj = globalThis.__obscura_objects['{oid}'];\
|
||||
if (!obj || typeof obj !== 'object') return [];\
|
||||
var keys = Object.keys(obj);\
|
||||
return keys.map(function(k) {{\
|
||||
var v = obj[k];\
|
||||
var t = typeof v;\
|
||||
var item = {{ name: k, type: t }};\
|
||||
if (v === null) {{ item.value = null; return item; }}\
|
||||
if (t !== 'object' && t !== 'function') {{ item.value = v; return item; }}\
|
||||
var childOid = '{oid}::' + k;\
|
||||
globalThis.__obscura_objects[childOid] = v;\
|
||||
item.childOid = childOid;\
|
||||
if (typeof v.nodeType === 'number') {{\
|
||||
item.subtype = 'node';\
|
||||
item.className = v.constructor && v.constructor.name ? v.constructor.name : (v.tagName ? 'HTML' + v.tagName.charAt(0) + v.tagName.slice(1).toLowerCase() + 'Element' : 'Node');\
|
||||
item.description = v.tagName ? v.tagName.toLowerCase() : (v.nodeName || 'node');\
|
||||
}} else if (Array.isArray(v)) {{\
|
||||
item.subtype = 'array';\
|
||||
item.className = 'Array';\
|
||||
item.description = 'Array(' + v.length + ')';\
|
||||
}} else {{\
|
||||
item.className = (v.constructor && v.constructor.name) || 'Object';\
|
||||
item.description = item.className;\
|
||||
}}\
|
||||
return item;\
|
||||
}});\
|
||||
}})()",
|
||||
oid = escaped_oid,
|
||||
);
|
||||
let result = page.evaluate(&code);
|
||||
if let serde_json::Value::Array(props) = result {
|
||||
let descriptors: Vec<Value> = props
|
||||
.iter()
|
||||
.map(|p| {
|
||||
let name = p.get("name").and_then(|v| v.as_str()).unwrap_or("");
|
||||
let prop_type =
|
||||
p.get("type").and_then(|v| v.as_str()).unwrap_or("undefined");
|
||||
let mut remote = json!({ "type": prop_type });
|
||||
if let Some(child_oid) = p.get("childOid").and_then(|v| v.as_str()) {
|
||||
remote["type"] = json!("object");
|
||||
if let Some(sub) = p.get("subtype").and_then(|v| v.as_str()) {
|
||||
remote["subtype"] = json!(sub);
|
||||
}
|
||||
if let Some(cls) = p.get("className").and_then(|v| v.as_str()) {
|
||||
remote["className"] = json!(cls);
|
||||
}
|
||||
if let Some(desc) = p.get("description").and_then(|v| v.as_str()) {
|
||||
remote["description"] = json!(desc);
|
||||
}
|
||||
remote["objectId"] = json!(child_oid);
|
||||
} else if let Some(val) = p.get("value") {
|
||||
match val {
|
||||
Value::Null => {
|
||||
remote["type"] = json!("object");
|
||||
remote["subtype"] = json!("null");
|
||||
remote["value"] = json!(null);
|
||||
}
|
||||
Value::String(s) => {
|
||||
remote["type"] = json!("string");
|
||||
remote["value"] = json!(s);
|
||||
}
|
||||
Value::Number(n) => {
|
||||
remote["type"] = json!("number");
|
||||
remote["value"] = json!(n);
|
||||
}
|
||||
Value::Bool(b) => {
|
||||
remote["type"] = json!("boolean");
|
||||
remote["value"] = json!(b);
|
||||
}
|
||||
_ => {
|
||||
remote["value"] = val.clone();
|
||||
}
|
||||
}
|
||||
}
|
||||
json!({
|
||||
"name": name,
|
||||
"value": remote,
|
||||
"configurable": true,
|
||||
"enumerable": true,
|
||||
"writable": true,
|
||||
"isOwn": true,
|
||||
})
|
||||
})
|
||||
.collect();
|
||||
Ok(json!({ "result": descriptors, "internalProperties": [] }))
|
||||
} else {
|
||||
Ok(json!({ "result": [], "internalProperties": [] }))
|
||||
}
|
||||
} else {
|
||||
Ok(json!({ "result": [], "internalProperties": [] }))
|
||||
}
|
||||
}
|
||||
"releaseObject" => {
|
||||
if let Some(oid) = params.get("objectId").and_then(|v| v.as_str()) {
|
||||
if let Some(page) = ctx.get_session_page_mut(session_id) {
|
||||
page.release_object(oid);
|
||||
}
|
||||
}
|
||||
Ok(json!({}))
|
||||
}
|
||||
"releaseObjectGroup" => {
|
||||
if let Some(page) = ctx.get_session_page_mut(session_id) {
|
||||
page.release_object_group();
|
||||
}
|
||||
Ok(json!({}))
|
||||
}
|
||||
"addBinding" => {
|
||||
let name = params.get("name").and_then(|v| v.as_str()).unwrap_or("");
|
||||
if !name.is_empty()
|
||||
&& name.chars().all(|c| c.is_alphanumeric() || c == '_' || c == '$')
|
||||
&& !name.chars().next().unwrap_or('0').is_ascii_digit()
|
||||
{
|
||||
// The shim forwards every call back to Rust through
|
||||
// op_binding_called; the CDP dispatcher then drains the
|
||||
// queue and emits Runtime.bindingCalled events the same
|
||||
// way Chromium does. Chromium's V8InspectorImpl rejects
|
||||
// calls without exactly one argument and ToString-coerces
|
||||
// that argument before emitting it as the payload — we
|
||||
// match the coercion (`String(arg)`) and silently drop
|
||||
// calls with wrong arity, which is what Chrome does.
|
||||
let shim = format!(
|
||||
"globalThis['{name}'] = function (arg) {{\
|
||||
if (arguments.length !== 1) return;\
|
||||
try {{\
|
||||
const payload = typeof arg === 'string' ? arg : String(arg);\
|
||||
Deno.core.ops.op_binding_called('{name}', payload);\
|
||||
}} catch (e) {{ /* swallow: binding must not throw into page */ }}\
|
||||
}};",
|
||||
name = name,
|
||||
);
|
||||
// Re-install on every navigation: globalThis is wiped on
|
||||
// each new document, and puppeteer registers bindings
|
||||
// once-per-page rather than once-per-document.
|
||||
let key = format!("__obscura_binding__{}", name);
|
||||
ctx.preload_scripts.retain(|(k, _)| k != &key);
|
||||
ctx.preload_scripts.push((key, shim.clone()));
|
||||
// Install on the current page so the binding is usable
|
||||
// immediately, without waiting for the next navigation.
|
||||
if let Some(page) = ctx.get_session_page_mut(session_id) {
|
||||
page.evaluate(&shim);
|
||||
}
|
||||
}
|
||||
Ok(json!({}))
|
||||
}
|
||||
"removeBinding" => {
|
||||
let name = params.get("name").and_then(|v| v.as_str()).unwrap_or("");
|
||||
if !name.is_empty() {
|
||||
let key = format!("__obscura_binding__{}", name);
|
||||
ctx.preload_scripts.retain(|(k, _)| k != &key);
|
||||
if let Some(page) = ctx.get_session_page_mut(session_id) {
|
||||
page.evaluate(&format!("delete globalThis['{}'];", name));
|
||||
}
|
||||
}
|
||||
Ok(json!({}))
|
||||
}
|
||||
"runIfWaitingForDebugger" => Ok(json!({})),
|
||||
"getExceptionDetails" => Ok(json!({ "exceptionDetails": null })),
|
||||
"discardConsoleEntries" => Ok(json!({})),
|
||||
_ => Err(format!("Unknown Runtime method: {}", method)),
|
||||
}
|
||||
}
|
||||
|
||||
/// Reject `Runtime.{evaluate,callFunctionOn}` calls that target an execution
|
||||
/// context Obscura has not advertised. Returns `Ok(())` when the parameter is
|
||||
/// absent (defaulting to the page's default context) or when the id matches
|
||||
/// one of `ctx.valid_context_ids`. Logs a debug trace on accept for #51.
|
||||
fn validate_context_id(
|
||||
params: &Value,
|
||||
field: &str,
|
||||
ctx: &crate::dispatch::CdpContext,
|
||||
method: &str,
|
||||
) -> Result<(), String> {
|
||||
let Some(id) = params.get(field).and_then(|v| v.as_i64()) else {
|
||||
return Ok(());
|
||||
};
|
||||
if !ctx.valid_context_ids.contains(&id) {
|
||||
return Err(format!(
|
||||
"Cannot find context with specified id: {}",
|
||||
id
|
||||
));
|
||||
}
|
||||
tracing::debug!(
|
||||
target: "obscura_cdp::runtime",
|
||||
"Runtime.{}: {}={} (single-isolate routing)",
|
||||
method,
|
||||
field,
|
||||
id
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn remote_object_from_info(info: &RemoteObjectInfo) -> Value {
|
||||
let mut obj = json!({ "type": info.js_type });
|
||||
|
||||
if let Some(ref subtype) = info.subtype {
|
||||
obj["subtype"] = json!(subtype);
|
||||
}
|
||||
|
||||
if !info.class_name.is_empty() {
|
||||
obj["className"] = json!(info.class_name);
|
||||
}
|
||||
|
||||
if !info.description.is_empty() {
|
||||
obj["description"] = json!(info.description);
|
||||
}
|
||||
|
||||
if let Some(ref oid) = info.object_id {
|
||||
obj["objectId"] = json!(oid);
|
||||
}
|
||||
|
||||
if let Some(ref value) = info.value {
|
||||
obj["value"] = value.clone();
|
||||
}
|
||||
|
||||
obj
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::dispatch::CdpContext;
|
||||
|
||||
// Issue #51 — Runtime.evaluate / callFunctionOn must read and validate
|
||||
// contextId. Pre-fix the parameter was silently dropped, so Playwright's
|
||||
// locator (which targets the utility world created by
|
||||
// Page.createIsolatedWorld) ran in the wrong context and timed out.
|
||||
//
|
||||
// Phase 5.5 (RED-then-GREEN) verification:
|
||||
// - Without the prod fix, `valid_context_ids` does not exist on
|
||||
// CdpContext → these tests fail to compile.
|
||||
// - With the prod fix, all four tests pass.
|
||||
|
||||
#[tokio::test]
|
||||
async fn evaluate_rejects_unknown_context_id() {
|
||||
let mut ctx = CdpContext::new();
|
||||
let err = handle(
|
||||
"evaluate",
|
||||
&json!({ "expression": "1 + 1", "contextId": 9999 }),
|
||||
&mut ctx,
|
||||
&None,
|
||||
)
|
||||
.await
|
||||
.expect_err("unknown contextId must error per CDP spec");
|
||||
assert!(
|
||||
err.contains("Cannot find context with specified id"),
|
||||
"error must match real Chrome's wording: {err}"
|
||||
);
|
||||
assert!(err.contains("9999"), "error must include the bad id: {err}");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn call_function_on_rejects_unknown_execution_context_id() {
|
||||
let mut ctx = CdpContext::new();
|
||||
let err = handle(
|
||||
"callFunctionOn",
|
||||
&json!({
|
||||
"functionDeclaration": "() => 42",
|
||||
"executionContextId": 9999,
|
||||
}),
|
||||
&mut ctx,
|
||||
&None,
|
||||
)
|
||||
.await
|
||||
.expect_err("unknown executionContextId must error per CDP spec");
|
||||
assert!(
|
||||
err.contains("Cannot find context with specified id"),
|
||||
"error must match Chrome wording: {err}"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn evaluate_accepts_default_context_id_one() {
|
||||
// Runtime.enable advertises contextId=1 — that must be accepted as
|
||||
// valid input to evaluate, regardless of whether a page is attached.
|
||||
// (Without a page we get an Err("No page") AFTER the contextId check,
|
||||
// which proves validation passed for id=1.)
|
||||
let mut ctx = CdpContext::new();
|
||||
let result = handle(
|
||||
"evaluate",
|
||||
&json!({ "expression": "1 + 1", "contextId": 1 }),
|
||||
&mut ctx,
|
||||
&None,
|
||||
)
|
||||
.await;
|
||||
match result {
|
||||
Ok(_) => {} // accepted + executed (would happen if a page is attached)
|
||||
Err(e) => assert!(
|
||||
!e.contains("Cannot find context"),
|
||||
"contextId=1 must be accepted, got: {e}"
|
||||
),
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn create_isolated_world_registers_id_for_evaluate() {
|
||||
// Round-trip: Page.createIsolatedWorld returns contextId N, and a
|
||||
// subsequent Runtime.evaluate targeting that contextId must NOT be
|
||||
// rejected.
|
||||
let mut ctx = CdpContext::new();
|
||||
// Bypass the page-attached path of createIsolatedWorld by direct
|
||||
// insert — mirrors the same effect as calling the page handler with
|
||||
// a real session.
|
||||
ctx.valid_context_ids.insert(100);
|
||||
|
||||
let result = handle(
|
||||
"evaluate",
|
||||
&json!({ "expression": "1 + 1", "contextId": 100 }),
|
||||
&mut ctx,
|
||||
&None,
|
||||
)
|
||||
.await;
|
||||
if let Err(e) = result {
|
||||
assert!(
|
||||
!e.contains("Cannot find context"),
|
||||
"registered isolated-world contextId=100 must be accepted, got: {e}"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/// Regression for #122 item 7: puppeteer-extra's FrameManager.initialize
|
||||
/// fires Runtime.enable on the browser-level WebSocket BEFORE any page
|
||||
/// target exists. Real Chrome replies with `{}`; before the fix Obscura
|
||||
/// returned `{"error":{"code":-32601,"message":"No page"}}` and the
|
||||
/// puppeteer connect flow died.
|
||||
#[tokio::test]
|
||||
async fn enable_succeeds_when_no_session_attached() {
|
||||
let mut ctx = CdpContext::new();
|
||||
let result = handle("enable", &json!({}), &mut ctx, &None)
|
||||
.await
|
||||
.expect("Runtime.enable must succeed even with no session");
|
||||
assert_eq!(result, json!({}));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,38 @@
|
||||
use serde_json::{json, Value};
|
||||
|
||||
use crate::cookie_params::{parse_cdp_cookie, parse_delete_cookies_params};
|
||||
use crate::dispatch::CdpContext;
|
||||
use crate::domains::network::cookie_info_to_cdp_json;
|
||||
|
||||
pub async fn handle(
|
||||
method: &str,
|
||||
params: &Value,
|
||||
ctx: &mut CdpContext,
|
||||
_session_id: &Option<String>,
|
||||
) -> Result<Value, String> {
|
||||
match method {
|
||||
"getCookies" => {
|
||||
let cookies = ctx.default_context.cookie_jar.get_all_cookies();
|
||||
let cdp_cookies: Vec<Value> = cookies.iter().map(cookie_info_to_cdp_json).collect();
|
||||
Ok(json!({ "cookies": cdp_cookies }))
|
||||
}
|
||||
"setCookies" => {
|
||||
if let Some(cookies) = params.get("cookies").and_then(|v| v.as_array()) {
|
||||
let parsed: Vec<_> = cookies.iter().filter_map(parse_cdp_cookie).collect();
|
||||
ctx.default_context.cookie_jar.set_cookies_from_cdp(parsed);
|
||||
}
|
||||
Ok(json!({}))
|
||||
}
|
||||
"deleteCookies" => {
|
||||
if let Some(filter) = parse_delete_cookies_params(params) {
|
||||
ctx.default_context.cookie_jar.delete_cookies_filtered(
|
||||
&filter.name,
|
||||
&filter.domain,
|
||||
filter.path.as_deref(),
|
||||
);
|
||||
}
|
||||
Ok(json!({}))
|
||||
}
|
||||
_ => Ok(json!({})),
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,309 @@
|
||||
use serde_json::{json, Value};
|
||||
|
||||
use crate::dispatch::CdpContext;
|
||||
use crate::types::CdpEvent;
|
||||
use crate::util::url_is_file_scheme;
|
||||
|
||||
pub async fn handle(method: &str, params: &Value, ctx: &mut CdpContext) -> Result<Value, String> {
|
||||
match method {
|
||||
"setDiscoverTargets" => {
|
||||
ctx.pending_events.push(CdpEvent::new(
|
||||
"Target.targetCreated",
|
||||
json!({
|
||||
"targetInfo": {
|
||||
"targetId": "browser",
|
||||
"type": "browser",
|
||||
"title": "",
|
||||
"url": "",
|
||||
"attached": true,
|
||||
"canAccessOpener": false,
|
||||
"browserContextId": "",
|
||||
}
|
||||
}),
|
||||
));
|
||||
for page in &ctx.pages {
|
||||
ctx.pending_events.push(CdpEvent::new(
|
||||
"Target.targetCreated",
|
||||
json!({
|
||||
"targetInfo": {
|
||||
"targetId": page.id,
|
||||
"type": "page",
|
||||
"title": page.title,
|
||||
"url": page.url_string(),
|
||||
"attached": false,
|
||||
"canAccessOpener": false,
|
||||
"browserContextId": page.context.id,
|
||||
}
|
||||
}),
|
||||
));
|
||||
}
|
||||
Ok(json!({}))
|
||||
}
|
||||
"getTargets" => {
|
||||
let targets: Vec<Value> = ctx
|
||||
.pages
|
||||
.iter()
|
||||
.map(|page| {
|
||||
json!({
|
||||
"targetId": page.id,
|
||||
"type": "page",
|
||||
"title": page.title,
|
||||
"url": page.url_string(),
|
||||
"attached": true,
|
||||
"canAccessOpener": false,
|
||||
"browserContextId": page.context.id,
|
||||
})
|
||||
})
|
||||
.collect();
|
||||
Ok(json!({ "targetInfos": targets }))
|
||||
}
|
||||
"createTarget" => {
|
||||
let url = params.get("url").and_then(|v| v.as_str()).unwrap_or("about:blank");
|
||||
|
||||
// Same gate as Page.navigate (GHSA-q55h-vfv9-qcr5). Without this,
|
||||
// a CDP client can call Target.createTarget {url:"file:///etc/passwd"}
|
||||
// and then Runtime.evaluate the body off the created target,
|
||||
// bypassing the page-domain check entirely.
|
||||
if url_is_file_scheme(url) && !ctx.default_context.allow_file_access {
|
||||
return Err(
|
||||
"Target.createTarget to file:// is disabled. Restart with `obscura serve --allow-file-access` to enable.".to_string()
|
||||
);
|
||||
}
|
||||
|
||||
let page_id = ctx.create_page();
|
||||
let session_id = format!("{}-session", page_id);
|
||||
|
||||
if let Some(page) = ctx.get_page_mut(&page_id) {
|
||||
if url == "about:blank" || url.is_empty() {
|
||||
page.navigate_blank();
|
||||
} else {
|
||||
let _ = page.navigate(url).await;
|
||||
}
|
||||
}
|
||||
|
||||
ctx.sessions.insert(session_id.clone(), page_id.clone());
|
||||
|
||||
if let Some(page) = ctx.get_page(&page_id) {
|
||||
ctx.pending_events.push(CdpEvent::new(
|
||||
"Target.targetCreated",
|
||||
json!({
|
||||
"targetInfo": {
|
||||
"targetId": page_id,
|
||||
"type": "page",
|
||||
"title": page.title,
|
||||
"url": page.url_string(),
|
||||
"attached": false,
|
||||
"canAccessOpener": false,
|
||||
"browserContextId": page.context.id,
|
||||
}
|
||||
}),
|
||||
));
|
||||
}
|
||||
|
||||
if let Some(page) = ctx.get_page(&page_id) {
|
||||
ctx.pending_events.push(CdpEvent::new(
|
||||
"Target.attachedToTarget",
|
||||
json!({
|
||||
"sessionId": session_id,
|
||||
"targetInfo": {
|
||||
"targetId": page_id,
|
||||
"type": "page",
|
||||
"title": page.title,
|
||||
"url": page.url_string(),
|
||||
"attached": true,
|
||||
"canAccessOpener": false,
|
||||
"browserContextId": page.context.id,
|
||||
},
|
||||
"waitingForDebugger": false,
|
||||
}),
|
||||
));
|
||||
}
|
||||
|
||||
Ok(json!({ "targetId": page_id }))
|
||||
}
|
||||
"attachToBrowserTarget" => {
|
||||
// Playwright calls this on connect to obtain a session for the
|
||||
// implicit "browser" target. Returning Unknown method aborts
|
||||
// the connect handshake before any user code runs.
|
||||
let session_id = "browser-session".to_string();
|
||||
ctx.sessions.insert(session_id.clone(), "browser".to_string());
|
||||
|
||||
ctx.pending_events.push(CdpEvent::new(
|
||||
"Target.attachedToTarget",
|
||||
json!({
|
||||
"sessionId": session_id,
|
||||
"targetInfo": {
|
||||
"targetId": "browser",
|
||||
"type": "browser",
|
||||
"title": "",
|
||||
"url": "",
|
||||
"attached": true,
|
||||
"canAccessOpener": false,
|
||||
"browserContextId": "",
|
||||
},
|
||||
"waitingForDebugger": false,
|
||||
}),
|
||||
));
|
||||
|
||||
Ok(json!({ "sessionId": session_id }))
|
||||
}
|
||||
"attachToTarget" => {
|
||||
let target_id = params.get("targetId").and_then(|v| v.as_str())
|
||||
.ok_or("targetId required")?;
|
||||
let session_id = format!("{}-session", target_id);
|
||||
ctx.sessions.insert(session_id.clone(), target_id.to_string());
|
||||
|
||||
if let Some(page) = ctx.get_page(target_id) {
|
||||
ctx.pending_events.push(CdpEvent::new(
|
||||
"Target.attachedToTarget",
|
||||
json!({
|
||||
"sessionId": session_id,
|
||||
"targetInfo": {
|
||||
"targetId": target_id,
|
||||
"type": "page",
|
||||
"title": page.title,
|
||||
"url": page.url_string(),
|
||||
"attached": true,
|
||||
"canAccessOpener": false,
|
||||
"browserContextId": page.context.id,
|
||||
},
|
||||
"waitingForDebugger": false,
|
||||
}),
|
||||
));
|
||||
}
|
||||
|
||||
Ok(json!({ "sessionId": session_id }))
|
||||
}
|
||||
"closeTarget" => {
|
||||
let target_id = params.get("targetId").and_then(|v| v.as_str())
|
||||
.ok_or("targetId required")?;
|
||||
let session_id = format!("{}-session", target_id);
|
||||
|
||||
ctx.pending_events.push(CdpEvent::new(
|
||||
"Target.detachedFromTarget",
|
||||
json!({
|
||||
"sessionId": session_id,
|
||||
"targetId": target_id,
|
||||
}),
|
||||
));
|
||||
ctx.pending_events.push(CdpEvent::new(
|
||||
"Target.targetDestroyed",
|
||||
json!({ "targetId": target_id }),
|
||||
));
|
||||
|
||||
ctx.remove_page(target_id);
|
||||
Ok(json!({ "success": true }))
|
||||
}
|
||||
"setAutoAttach" => Ok(json!({})),
|
||||
// No multi-target lifecycle to manage: obscura runs one page per session.
|
||||
// Ack these so Chrome-shaped clients that call them do not warn (issue #340).
|
||||
"detachFromTarget" => Ok(json!({})),
|
||||
"activateTarget" => Ok(json!({})),
|
||||
"getBrowserContexts" => {
|
||||
Ok(json!({ "browserContextIds": [ctx.default_context.id] }))
|
||||
}
|
||||
"createBrowserContext" => {
|
||||
ctx.default_context.cookie_jar.clear();
|
||||
Ok(json!({ "browserContextId": ctx.default_context.id }))
|
||||
}
|
||||
"disposeBrowserContext" => {
|
||||
ctx.default_context.cookie_jar.clear();
|
||||
Ok(json!({}))
|
||||
}
|
||||
"getTargetInfo" => {
|
||||
let target_id = params.get("targetId").and_then(|v| v.as_str());
|
||||
match target_id {
|
||||
Some(id) => {
|
||||
let page = ctx.get_page(id).ok_or("Target not found")?;
|
||||
Ok(json!({
|
||||
"targetInfo": {
|
||||
"targetId": id,
|
||||
"type": "page",
|
||||
"title": page.title,
|
||||
"url": page.url_string(),
|
||||
"attached": true,
|
||||
"canAccessOpener": false,
|
||||
"browserContextId": page.context.id,
|
||||
}
|
||||
}))
|
||||
}
|
||||
None => {
|
||||
// canAccessOpener is required on every TargetInfo per the
|
||||
// CDP spec. Strict clients (chromiumoxide) panic if it's
|
||||
// missing. The browser target itself has no opener.
|
||||
Ok(json!({
|
||||
"targetInfo": {
|
||||
"targetId": "browser",
|
||||
"type": "browser",
|
||||
"title": "",
|
||||
"url": "",
|
||||
"attached": true,
|
||||
"canAccessOpener": false,
|
||||
}
|
||||
}))
|
||||
}
|
||||
}
|
||||
}
|
||||
_ => Err(format!("Unknown Target method: {}", method)),
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[tokio::test]
|
||||
async fn attach_to_browser_target_returns_session_id() {
|
||||
let mut ctx = CdpContext::new();
|
||||
let result = handle("attachToBrowserTarget", &json!({}), &mut ctx)
|
||||
.await
|
||||
.expect("attachToBrowserTarget should succeed");
|
||||
|
||||
assert_eq!(result["sessionId"], "browser-session");
|
||||
assert_eq!(
|
||||
ctx.sessions.get("browser-session").map(String::as_str),
|
||||
Some("browser")
|
||||
);
|
||||
|
||||
// Playwright/Puppeteer expect a Target.attachedToTarget event before
|
||||
// they finish wiring up the session — without it the connect promise
|
||||
// hangs.
|
||||
let attached_evt = ctx
|
||||
.pending_events
|
||||
.iter()
|
||||
.find(|e| e.method == "Target.attachedToTarget")
|
||||
.expect("attachedToTarget event must be emitted");
|
||||
assert_eq!(attached_evt.params["sessionId"], "browser-session");
|
||||
assert_eq!(attached_evt.params["targetInfo"]["type"], "browser");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn unknown_target_method_still_errors() {
|
||||
let mut ctx = CdpContext::new();
|
||||
let err = handle("notARealMethod", &json!({}), &mut ctx)
|
||||
.await
|
||||
.expect_err("unknown methods must surface as errors");
|
||||
assert!(err.contains("Unknown Target method"));
|
||||
}
|
||||
|
||||
/// Regression for #122 item 5: every TargetInfo payload must carry the
|
||||
/// `canAccessOpener` field. The browser-target branch of getTargetInfo
|
||||
/// (no targetId passed → no page) used to omit it; strict CDP clients
|
||||
/// like chromiumoxide panic when the field is missing.
|
||||
#[tokio::test]
|
||||
async fn get_target_info_browser_target_includes_can_access_opener() {
|
||||
let mut ctx = CdpContext::new();
|
||||
// No targetId → falls through to the browser-target branch.
|
||||
let result = handle("getTargetInfo", &json!({}), &mut ctx)
|
||||
.await
|
||||
.expect("getTargetInfo with no targetId must return browser info");
|
||||
|
||||
let info = &result["targetInfo"];
|
||||
assert_eq!(info["type"], "browser", "must be the browser target");
|
||||
assert!(
|
||||
info.get("canAccessOpener").is_some(),
|
||||
"canAccessOpener must be present on every TargetInfo, got: {result}"
|
||||
);
|
||||
assert_eq!(info["canAccessOpener"], false);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,11 @@
|
||||
pub mod server;
|
||||
pub mod dispatch;
|
||||
pub mod types;
|
||||
pub mod domains;
|
||||
pub mod cookie_params;
|
||||
pub(crate) mod util;
|
||||
|
||||
pub use server::{
|
||||
start, start_with_full_options, start_with_full_serve_options, start_with_host,
|
||||
start_with_host_and_security, start_with_options,
|
||||
};
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,74 @@
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
pub struct CdpRequest {
|
||||
pub id: u64,
|
||||
pub method: String,
|
||||
#[serde(default)]
|
||||
pub params: serde_json::Value,
|
||||
#[serde(rename = "sessionId")]
|
||||
pub session_id: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize)]
|
||||
pub struct CdpResponse {
|
||||
pub id: u64,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub result: Option<serde_json::Value>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub error: Option<CdpError>,
|
||||
#[serde(rename = "sessionId", skip_serializing_if = "Option::is_none")]
|
||||
pub session_id: Option<String>,
|
||||
}
|
||||
|
||||
impl CdpResponse {
|
||||
pub fn success(id: u64, result: serde_json::Value, session_id: Option<String>) -> Self {
|
||||
CdpResponse {
|
||||
id,
|
||||
result: Some(result),
|
||||
error: None,
|
||||
session_id,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn error(id: u64, code: i64, message: String, session_id: Option<String>) -> Self {
|
||||
CdpResponse {
|
||||
id,
|
||||
result: None,
|
||||
error: Some(CdpError { code, message }),
|
||||
session_id,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize)]
|
||||
pub struct CdpError {
|
||||
pub code: i64,
|
||||
pub message: String,
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize)]
|
||||
pub struct CdpEvent {
|
||||
pub method: String,
|
||||
pub params: serde_json::Value,
|
||||
#[serde(rename = "sessionId", skip_serializing_if = "Option::is_none")]
|
||||
pub session_id: Option<String>,
|
||||
}
|
||||
|
||||
impl CdpEvent {
|
||||
pub fn new(method: &str, params: serde_json::Value) -> Self {
|
||||
CdpEvent {
|
||||
method: method.to_string(),
|
||||
params,
|
||||
session_id: None,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn with_session(method: &str, params: serde_json::Value, session_id: String) -> Self {
|
||||
CdpEvent {
|
||||
method: method.to_string(),
|
||||
params,
|
||||
session_id: Some(session_id),
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,62 @@
|
||||
/// Helpers shared across CDP domain handlers.
|
||||
///
|
||||
/// The file-scheme detector here is reused by every CDP entrypoint that can
|
||||
/// trigger a navigation, so we don't end up with one domain enforcing the
|
||||
/// `--allow-file-access` gate and another silently letting `file://` through
|
||||
/// (see GHSA-q55h-vfv9-qcr5 and its incomplete-fix variant in
|
||||
/// `Target.createTarget`).
|
||||
|
||||
/// Returns true when `raw` parses as a `file:`-scheme URL, or syntactically
|
||||
/// starts with `file:` after a possible leading-whitespace strip. Matching is
|
||||
/// case-insensitive on the scheme so neither `FILE://` nor `File://` slips
|
||||
/// past callers that gate on `file://`.
|
||||
pub(crate) fn url_is_file_scheme(raw: &str) -> bool {
|
||||
url::Url::parse(raw)
|
||||
.map(|u| u.scheme().eq_ignore_ascii_case("file"))
|
||||
.unwrap_or_else(|_| {
|
||||
raw.trim_start().to_ascii_lowercase().starts_with("file:")
|
||||
})
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn matches_plain_file_url() {
|
||||
assert!(url_is_file_scheme("file:///etc/passwd"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn matches_case_insensitively() {
|
||||
assert!(url_is_file_scheme("FILE:///etc/passwd"));
|
||||
assert!(url_is_file_scheme("File:///etc/passwd"));
|
||||
assert!(url_is_file_scheme("fIlE:///etc/passwd"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn matches_with_leading_whitespace_fallback() {
|
||||
// url::Url::parse rejects leading whitespace, but the syntactic
|
||||
// fallback still catches ` file:...` so callers can't be tricked
|
||||
// into letting it through.
|
||||
assert!(url_is_file_scheme(" file:///etc/passwd"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rejects_http_https_about_data() {
|
||||
assert!(!url_is_file_scheme("http://example.com"));
|
||||
assert!(!url_is_file_scheme("https://example.com"));
|
||||
assert!(!url_is_file_scheme("about:blank"));
|
||||
assert!(!url_is_file_scheme("data:text/plain,hi"));
|
||||
assert!(!url_is_file_scheme(""));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rejects_lookalikes_that_are_not_file_scheme() {
|
||||
// The URL parser rejects these (no `://`), so the syntactic fallback
|
||||
// kicks in. `file` appearing anywhere except as the leading scheme
|
||||
// must not match.
|
||||
assert!(!url_is_file_scheme("notfile:///x"));
|
||||
assert!(!url_is_file_scheme("http://file/"));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,146 @@
|
||||
use obscura_cdp::dispatch::{dispatch, CdpContext};
|
||||
use obscura_cdp::types::CdpRequest;
|
||||
use serde_json::{json, Value};
|
||||
use tokio::io::{AsyncReadExt, AsyncWriteExt};
|
||||
use tokio::net::TcpListener;
|
||||
|
||||
async fn serve_once() -> String {
|
||||
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let addr = listener.local_addr().unwrap();
|
||||
tokio::spawn(async move {
|
||||
for _ in 0..2 {
|
||||
let (mut socket, _) = listener.accept().await.unwrap();
|
||||
tokio::spawn(async move {
|
||||
let mut buf = [0u8; 2048];
|
||||
let n = socket.read(&mut buf).await.unwrap();
|
||||
let req = String::from_utf8_lossy(&buf[..n]);
|
||||
let (status, body) = if req.starts_with("GET /submitted") {
|
||||
("200 OK", "<html><body>submitted</body></html>")
|
||||
} else {
|
||||
(
|
||||
"200 OK",
|
||||
r#"<html><body>
|
||||
<form id="f" action="/submitted">
|
||||
<input type="hidden" name="vacancy_id" value="123">
|
||||
<textarea name="message">hello</textarea>
|
||||
<input type="checkbox" name="agree" value="yes" checked>
|
||||
<button id="submit" type="submit">Go</button>
|
||||
</form>
|
||||
<script>
|
||||
function submitCompat() {
|
||||
const form = document.getElementById('f');
|
||||
const params = new URLSearchParams();
|
||||
form.querySelectorAll('input, textarea').forEach(function(field) {
|
||||
if (field.type === 'checkbox' && !field.checked) return;
|
||||
params.append(field.name, field.value);
|
||||
});
|
||||
location.href = form.action + '?' + params.toString();
|
||||
}
|
||||
document.querySelector('button').addEventListener('click', function(e) {
|
||||
e.preventDefault();
|
||||
submitCompat();
|
||||
});
|
||||
</script>
|
||||
</body></html>"#,
|
||||
)
|
||||
};
|
||||
let resp = format!(
|
||||
"HTTP/1.1 {status}\r\nContent-Type: text/html\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{body}",
|
||||
body.len()
|
||||
);
|
||||
socket.write_all(resp.as_bytes()).await.unwrap();
|
||||
});
|
||||
}
|
||||
});
|
||||
format!("http://{addr}/")
|
||||
}
|
||||
|
||||
async fn cdp(
|
||||
ctx: &mut CdpContext,
|
||||
id: u64,
|
||||
method: &str,
|
||||
params: Value,
|
||||
session_id: &str,
|
||||
) -> Value {
|
||||
let resp = dispatch(
|
||||
&CdpRequest {
|
||||
id,
|
||||
method: method.to_string(),
|
||||
params,
|
||||
session_id: Some(session_id.to_string()),
|
||||
},
|
||||
ctx,
|
||||
)
|
||||
.await;
|
||||
assert!(
|
||||
resp.error.is_none(),
|
||||
"CDP {method} failed: {:?}",
|
||||
resp.error
|
||||
);
|
||||
resp.result.unwrap_or_else(|| json!({}))
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "current_thread")]
|
||||
async fn runtime_click_submit_prevent_default_navigation_updates_page() {
|
||||
std::env::set_var("OBSCURA_ALLOW_PRIVATE_NETWORK", "1");
|
||||
let url = serve_once().await;
|
||||
let mut ctx = CdpContext::new();
|
||||
let page_id = ctx.create_page();
|
||||
let session_id = "session-1";
|
||||
ctx.sessions.insert(session_id.to_string(), page_id.clone());
|
||||
|
||||
cdp(
|
||||
&mut ctx,
|
||||
1,
|
||||
"Page.navigate",
|
||||
// Explicit `waitUntil: 'load'` so the inline `<script>` that defines
|
||||
// `submitCompat` runs. `Page.navigate` now defaults to
|
||||
// `DomContentLoaded` (matches real Chrome's lifecycle streaming so
|
||||
// Puppeteer/Playwright clients don't time out on JS-heavy pages),
|
||||
// which by design returns before parser-discovered scripts execute.
|
||||
json!({"url": url, "waitUntil": "load"}),
|
||||
session_id,
|
||||
)
|
||||
.await;
|
||||
|
||||
let submit_compat_type = cdp(
|
||||
&mut ctx,
|
||||
2,
|
||||
"Runtime.evaluate",
|
||||
json!({"expression": "typeof submitCompat", "returnByValue": true}),
|
||||
session_id,
|
||||
)
|
||||
.await;
|
||||
assert_eq!(submit_compat_type["result"]["value"], "function");
|
||||
|
||||
let button = cdp(
|
||||
&mut ctx,
|
||||
3,
|
||||
"Runtime.evaluate",
|
||||
json!({"expression": "document.getElementById('submit')"}),
|
||||
session_id,
|
||||
)
|
||||
.await;
|
||||
let object_id = button["result"]["objectId"].as_str().unwrap().to_string();
|
||||
|
||||
cdp(
|
||||
&mut ctx,
|
||||
4,
|
||||
"Runtime.callFunctionOn",
|
||||
json!({"objectId": object_id, "functionDeclaration": "function() { this.click(); }"}),
|
||||
session_id,
|
||||
)
|
||||
.await;
|
||||
|
||||
let page = ctx.get_page_mut(&page_id).unwrap();
|
||||
assert_eq!(page.url.as_ref().unwrap().path(), "/submitted");
|
||||
assert_eq!(
|
||||
page.url.as_ref().unwrap().query(),
|
||||
Some("vacancy_id=123&message=hello&agree=yes")
|
||||
);
|
||||
assert!(page
|
||||
.evaluate("document.body.textContent")
|
||||
.as_str()
|
||||
.unwrap()
|
||||
.contains("submitted"));
|
||||
}
|
||||
@@ -0,0 +1,145 @@
|
||||
//! Issue #19 smoke test: 5 parallel CDP clients each performing
|
||||
//! `Target.createTarget` + `Page.navigate` must not abort the process.
|
||||
//!
|
||||
//! NOTE on coverage. The deterministic abort in #19 requires the navigations
|
||||
//! to interleave on the shared `LocalSet` thread, which only happens once
|
||||
//! `navigate_single` actually yields — the heaviest yields come from
|
||||
//! subresource fetches in `futures::future::join_all` (page.rs:285) when the
|
||||
//! page has scripts/images to pull. `data:` URLs skip every fetch, so this
|
||||
//! test exercises the chokepoint shape (5 clients hitting `dispatch`
|
||||
//! concurrently) without driving the original abort. Treat it as a smoke
|
||||
//! check: it ensures the V8-lock plumbing compiles and isn't hitting an
|
||||
//! obvious deadlock, not as a strict regression for the abort.
|
||||
//!
|
||||
//! For an end-to-end repro of the abort, the issue author used
|
||||
//! `scrapegraphai.com` driven from Node CDP at concurrency 5. Reproducing
|
||||
//! that here would require standing up a local server with JS subresources;
|
||||
//! out of scope for this PR.
|
||||
//!
|
||||
//! Run with `cargo test -p obscura-cdp --test concurrent_navigations
|
||||
//! -- --nocapture --ignored` (the `ignored` gate keeps it out of the default
|
||||
//! suite — it boots a real CDP server, which is heavier than a unit test).
|
||||
|
||||
use std::time::Duration;
|
||||
|
||||
use futures_util::{SinkExt, StreamExt};
|
||||
use serde_json::{json, Value};
|
||||
use tokio::net::TcpListener;
|
||||
use tokio_tungstenite::{connect_async, tungstenite::Message};
|
||||
|
||||
async fn pick_port() -> u16 {
|
||||
let l = TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let port = l.local_addr().unwrap().port();
|
||||
drop(l);
|
||||
port
|
||||
}
|
||||
|
||||
async fn one_client(port: u16, id_base: u64) -> Result<(), String> {
|
||||
let url = format!("ws://127.0.0.1:{}/devtools/browser", port);
|
||||
let (mut ws, _) = connect_async(&url).await.map_err(|e| e.to_string())?;
|
||||
|
||||
// Target.createTarget — get a sessionId via Target.attachToTarget.
|
||||
let create = json!({
|
||||
"id": id_base,
|
||||
"method": "Target.createTarget",
|
||||
"params": {"url": "about:blank"},
|
||||
});
|
||||
ws.send(Message::Text(create.to_string().into()))
|
||||
.await
|
||||
.map_err(|e| e.to_string())?;
|
||||
|
||||
let mut session_id: Option<String> = None;
|
||||
let mut target_id: Option<String> = None;
|
||||
while session_id.is_none() {
|
||||
let msg = tokio::time::timeout(Duration::from_secs(5), ws.next())
|
||||
.await
|
||||
.map_err(|_| "timeout waiting for createTarget".to_string())?
|
||||
.ok_or("ws closed")?
|
||||
.map_err(|e| e.to_string())?;
|
||||
let text = match msg {
|
||||
Message::Text(t) => t.to_string(),
|
||||
_ => continue,
|
||||
};
|
||||
let v: Value = serde_json::from_str(&text).map_err(|e| e.to_string())?;
|
||||
if let Some(t) = v.get("result").and_then(|r| r.get("targetId")).and_then(|s| s.as_str()) {
|
||||
target_id = Some(t.to_string());
|
||||
}
|
||||
if let Some(s) = v
|
||||
.get("params")
|
||||
.and_then(|p| p.get("sessionId"))
|
||||
.and_then(|s| s.as_str())
|
||||
{
|
||||
session_id = Some(s.to_string());
|
||||
}
|
||||
}
|
||||
|
||||
let sid = session_id.unwrap();
|
||||
let _ = target_id; // kept for debugging — unused by the assertion path
|
||||
|
||||
// Page.navigate to a data URL — exercises init_js + execute_scripts
|
||||
// without touching the network, which is what the V8 race needs.
|
||||
let nav = json!({
|
||||
"id": id_base + 1,
|
||||
"method": "Page.navigate",
|
||||
"sessionId": sid,
|
||||
"params": {"url": "data:text/html,<html><body><h1>x</h1></body></html>"},
|
||||
});
|
||||
ws.send(Message::Text(nav.to_string().into()))
|
||||
.await
|
||||
.map_err(|e| e.to_string())?;
|
||||
|
||||
let deadline = tokio::time::Instant::now() + Duration::from_secs(10);
|
||||
loop {
|
||||
if tokio::time::Instant::now() >= deadline {
|
||||
return Err("timeout waiting for navigate response".to_string());
|
||||
}
|
||||
let remaining = deadline - tokio::time::Instant::now();
|
||||
let msg = tokio::time::timeout(remaining, ws.next())
|
||||
.await
|
||||
.map_err(|_| "timeout".to_string())?
|
||||
.ok_or("ws closed mid-navigate")?
|
||||
.map_err(|e| e.to_string())?;
|
||||
let text = match msg {
|
||||
Message::Text(t) => t.to_string(),
|
||||
_ => continue,
|
||||
};
|
||||
let v: Value = serde_json::from_str(&text).map_err(|e| e.to_string())?;
|
||||
if v.get("id").and_then(|x| x.as_u64()) == Some(id_base + 1) {
|
||||
return Ok(());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "current_thread")]
|
||||
|
||||
async fn concurrency_5_does_not_abort_v8() {
|
||||
let port = pick_port().await;
|
||||
let local = tokio::task::LocalSet::new();
|
||||
local
|
||||
.run_until(async {
|
||||
tokio::task::spawn_local(async move {
|
||||
let _ = obscura_cdp::server::start(port).await;
|
||||
});
|
||||
// Give the listener a beat.
|
||||
tokio::time::sleep(Duration::from_millis(150)).await;
|
||||
|
||||
let mut handles = Vec::new();
|
||||
for i in 0..5u64 {
|
||||
let id_base = (i + 1) * 1000;
|
||||
handles.push(tokio::task::spawn_local(async move {
|
||||
one_client(port, id_base).await
|
||||
}));
|
||||
}
|
||||
|
||||
let mut ok = 0usize;
|
||||
for (i, h) in handles.into_iter().enumerate() {
|
||||
match h.await {
|
||||
Ok(Ok(())) => ok += 1,
|
||||
Ok(Err(e)) => panic!("client {} failed: {}", i, e),
|
||||
Err(e) => panic!("client {} join error: {}", i, e),
|
||||
}
|
||||
}
|
||||
assert_eq!(ok, 5, "all 5 concurrent clients must complete navigate");
|
||||
})
|
||||
.await;
|
||||
}
|
||||
@@ -0,0 +1,215 @@
|
||||
//! Issue #19 hard repro: 5 parallel clients with Fetch interception enabled
|
||||
//! must not abort V8.
|
||||
//!
|
||||
//! Per maintainer comment on PR #36 (https://github.com/h4ckf0r0day/obscura/pull/36#issuecomment-4328238087):
|
||||
//!
|
||||
//! "Tested locally and the V8 fatal still reproduces when Fetch.enable
|
||||
//! interception is on with concurrent navigations:
|
||||
//! - 5 clients enable Fetch with patterns: ["*"]
|
||||
//! - All 5 issue Page.navigate concurrently
|
||||
//! - Server aborts with: Check failed: heap->isolate() == Isolate::TryGetCurrent()"
|
||||
//!
|
||||
//! The smoke test (concurrent_navigations.rs) used data: URLs which skip
|
||||
//! the subresource fetch path that drives the abort. This test:
|
||||
//! 1. Calls Fetch.enable patterns:["*"] on each client
|
||||
//! 2. Navigates to a URL with subresources (so op_fetch_url is reached
|
||||
//! and parks on resolve_rx.await INSIDE the v8_lock guard)
|
||||
//! 3. Auto-replies to Fetch.requestPaused with Fetch.continueRequest
|
||||
//! (so the parked op resumes — that's where two Isolates can collide)
|
||||
//!
|
||||
//! Run with `cargo test -p obscura-cdp --test concurrent_navigations_with_fetch
|
||||
//! -- --nocapture --ignored`.
|
||||
|
||||
use std::time::Duration;
|
||||
|
||||
use futures_util::{SinkExt, StreamExt};
|
||||
use serde_json::{json, Value};
|
||||
use tokio::net::TcpListener;
|
||||
use tokio_tungstenite::{connect_async, tungstenite::Message};
|
||||
|
||||
async fn pick_port() -> u16 {
|
||||
let l = TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let port = l.local_addr().unwrap().port();
|
||||
drop(l);
|
||||
port
|
||||
}
|
||||
|
||||
/// One client:
|
||||
/// 1. Target.createTarget → sessionId
|
||||
/// 2. Fetch.enable patterns:["*"]
|
||||
/// 3. Page.navigate to URL
|
||||
/// 4. Loop: respond to Fetch.requestPaused with Fetch.continueRequest
|
||||
/// until Page.navigate's response arrives.
|
||||
async fn one_client_with_fetch(port: u16, id_base: u64, target_url: &str) -> Result<(), String> {
|
||||
let url = format!("ws://127.0.0.1:{}/devtools/browser", port);
|
||||
let (mut ws, _) = connect_async(&url).await.map_err(|e| e.to_string())?;
|
||||
|
||||
// 1. Target.createTarget
|
||||
let create = json!({
|
||||
"id": id_base,
|
||||
"method": "Target.createTarget",
|
||||
"params": {"url": "about:blank"},
|
||||
});
|
||||
ws.send(Message::Text(create.to_string().into()))
|
||||
.await
|
||||
.map_err(|e| e.to_string())?;
|
||||
|
||||
let mut session_id: Option<String> = None;
|
||||
while session_id.is_none() {
|
||||
let msg = tokio::time::timeout(Duration::from_secs(5), ws.next())
|
||||
.await
|
||||
.map_err(|_| "timeout waiting for createTarget".to_string())?
|
||||
.ok_or("ws closed")?
|
||||
.map_err(|e| e.to_string())?;
|
||||
let text = match msg {
|
||||
Message::Text(t) => t.to_string(),
|
||||
_ => continue,
|
||||
};
|
||||
let v: Value = serde_json::from_str(&text).map_err(|e| e.to_string())?;
|
||||
if let Some(s) = v
|
||||
.get("params")
|
||||
.and_then(|p| p.get("sessionId"))
|
||||
.and_then(|s| s.as_str())
|
||||
{
|
||||
session_id = Some(s.to_string());
|
||||
}
|
||||
}
|
||||
let sid = session_id.unwrap();
|
||||
|
||||
// 2. Fetch.enable patterns:["*"]
|
||||
let fetch_enable = json!({
|
||||
"id": id_base + 1,
|
||||
"method": "Fetch.enable",
|
||||
"sessionId": sid,
|
||||
"params": {
|
||||
"patterns": [{"urlPattern": "*"}],
|
||||
},
|
||||
});
|
||||
ws.send(Message::Text(fetch_enable.to_string().into()))
|
||||
.await
|
||||
.map_err(|e| e.to_string())?;
|
||||
|
||||
// Wait for Fetch.enable response.
|
||||
let deadline = tokio::time::Instant::now() + Duration::from_secs(5);
|
||||
loop {
|
||||
if tokio::time::Instant::now() >= deadline {
|
||||
return Err("timeout waiting for Fetch.enable response".to_string());
|
||||
}
|
||||
let remaining = deadline - tokio::time::Instant::now();
|
||||
let msg = tokio::time::timeout(remaining, ws.next())
|
||||
.await
|
||||
.map_err(|_| "timeout".to_string())?
|
||||
.ok_or("ws closed")?
|
||||
.map_err(|e| e.to_string())?;
|
||||
let text = match msg {
|
||||
Message::Text(t) => t.to_string(),
|
||||
_ => continue,
|
||||
};
|
||||
let v: Value = serde_json::from_str(&text).map_err(|e| e.to_string())?;
|
||||
if v.get("id").and_then(|x| x.as_u64()) == Some(id_base + 1) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
// 3. Page.navigate
|
||||
let nav = json!({
|
||||
"id": id_base + 2,
|
||||
"method": "Page.navigate",
|
||||
"sessionId": sid,
|
||||
"params": {"url": target_url},
|
||||
});
|
||||
ws.send(Message::Text(nav.to_string().into()))
|
||||
.await
|
||||
.map_err(|e| e.to_string())?;
|
||||
|
||||
// 4. Drain events. Auto-respond to Fetch.requestPaused with continueRequest.
|
||||
let deadline = tokio::time::Instant::now() + Duration::from_secs(60);
|
||||
let mut auto_id = id_base + 1000;
|
||||
loop {
|
||||
if tokio::time::Instant::now() >= deadline {
|
||||
return Err(format!("client {} timeout waiting for navigate", id_base));
|
||||
}
|
||||
let remaining = deadline - tokio::time::Instant::now();
|
||||
let msg = tokio::time::timeout(remaining, ws.next())
|
||||
.await
|
||||
.map_err(|_| "timeout".to_string())?
|
||||
.ok_or("ws closed mid-navigate")?
|
||||
.map_err(|e| e.to_string())?;
|
||||
let text = match msg {
|
||||
Message::Text(t) => t.to_string(),
|
||||
_ => continue,
|
||||
};
|
||||
let v: Value = match serde_json::from_str::<Value>(&text) {
|
||||
Ok(v) => v,
|
||||
Err(_) => continue,
|
||||
};
|
||||
|
||||
// navigate response?
|
||||
if v.get("id").and_then(|x| x.as_u64()) == Some(id_base + 2) {
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
// Fetch.requestPaused event?
|
||||
if v.get("method").and_then(|m| m.as_str()) == Some("Fetch.requestPaused") {
|
||||
let req_id = v
|
||||
.get("params")
|
||||
.and_then(|p| p.get("requestId"))
|
||||
.and_then(|s| s.as_str())
|
||||
.unwrap_or("")
|
||||
.to_string();
|
||||
auto_id += 1;
|
||||
let cont = json!({
|
||||
"id": auto_id,
|
||||
"method": "Fetch.continueRequest",
|
||||
"sessionId": sid,
|
||||
"params": {"requestId": req_id},
|
||||
});
|
||||
ws.send(Message::Text(cont.to_string().into()))
|
||||
.await
|
||||
.map_err(|e| e.to_string())?;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// PR #36 maintainer's exact repro.
|
||||
#[tokio::test(flavor = "current_thread")]
|
||||
|
||||
async fn fetch_intercept_concurrency_5_does_not_abort_v8() {
|
||||
// Use example.com first — minimal subresources but real network. If this
|
||||
// doesn't reproduce, escalate to a JS-heavy page in a follow-up.
|
||||
let target_url = "https://example.com/";
|
||||
|
||||
let port = pick_port().await;
|
||||
let local = tokio::task::LocalSet::new();
|
||||
local
|
||||
.run_until(async {
|
||||
tokio::task::spawn_local(async move {
|
||||
let _ = obscura_cdp::server::start(port).await;
|
||||
});
|
||||
tokio::time::sleep(Duration::from_millis(250)).await;
|
||||
|
||||
let mut handles = Vec::new();
|
||||
for i in 0..5u64 {
|
||||
let id_base = (i + 1) * 1000;
|
||||
let url = target_url.to_string();
|
||||
handles.push(tokio::task::spawn_local(async move {
|
||||
one_client_with_fetch(port, id_base, &url).await
|
||||
}));
|
||||
}
|
||||
|
||||
let mut ok = 0usize;
|
||||
let mut errors = Vec::new();
|
||||
for (i, h) in handles.into_iter().enumerate() {
|
||||
match h.await {
|
||||
Ok(Ok(())) => ok += 1,
|
||||
Ok(Err(e)) => errors.push(format!("client {}: {}", i, e)),
|
||||
Err(e) => errors.push(format!("client {} join: {}", i, e)),
|
||||
}
|
||||
}
|
||||
if !errors.is_empty() {
|
||||
panic!("errors: {:#?}", errors);
|
||||
}
|
||||
assert_eq!(ok, 5, "all 5 concurrent clients must complete navigate");
|
||||
})
|
||||
.await;
|
||||
}
|
||||
@@ -0,0 +1,141 @@
|
||||
//! Issue #62 regression test: `/json/version` (HTTP control plane) must
|
||||
//! respond promptly even while V8 JS evaluation blocks the LocalSet thread.
|
||||
//!
|
||||
//! Before the fix, the HTTP accept loop competed with the CDP processor on
|
||||
//! the same `LocalSet`, so a synchronous JS `while` loop starved every other
|
||||
//! task — including the async `TcpListener::accept()`. The dedicated accept
|
||||
//! thread (std::net::TcpListener on a separate OS thread) fixes this.
|
||||
//!
|
||||
//! Run with `cargo test -p obscura-cdp --test control_plane_unblocked
|
||||
//! -- --nocapture --ignored`.
|
||||
|
||||
use std::io::{Read, Write};
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
use futures_util::{SinkExt, StreamExt};
|
||||
use serde_json::{json, Value};
|
||||
use tokio::net::TcpListener;
|
||||
use tokio_tungstenite::{connect_async, tungstenite::Message};
|
||||
|
||||
const HTTP_TIMEOUT: Duration = Duration::from_secs(3);
|
||||
const JS_DURATION_MS: u64 = 5000;
|
||||
|
||||
// Pick a free port for the test server. There is a small TOCTOU window
|
||||
// between dropping the listener here and the server's own bind a few lines
|
||||
// later — under heavy CI parallelism another process could steal the port
|
||||
// in between. The test is `#[ignore]` and opt-in via `--ignored`, so it
|
||||
// runs serially in practice. If we ever flip it to a default-on integration
|
||||
// test, replace this with a SO_REUSEPORT-aware port lease or pass the
|
||||
// already-bound listener into the server.
|
||||
async fn pick_port() -> u16 {
|
||||
let l = TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let port = l.local_addr().unwrap().port();
|
||||
drop(l);
|
||||
port
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "current_thread")]
|
||||
|
||||
async fn http_control_plane_unblocked_during_long_js() {
|
||||
let port = pick_port().await;
|
||||
let port_clone = port;
|
||||
|
||||
let local = tokio::task::LocalSet::new();
|
||||
local
|
||||
.run_until(async {
|
||||
tokio::task::spawn_local(async move {
|
||||
let _ = obscura_cdp::server::start(port).await;
|
||||
});
|
||||
// Give the listener + accept thread time to bind.
|
||||
tokio::time::sleep(Duration::from_millis(200)).await;
|
||||
|
||||
// Connect via WebSocket, create a target, then send a
|
||||
// long-running JS evaluation on that target's session.
|
||||
let url = format!("ws://127.0.0.1:{}/devtools/browser", port);
|
||||
let (mut ws, _) = connect_async(&url).await.unwrap();
|
||||
|
||||
let create = json!({
|
||||
"id": 1,
|
||||
"method": "Target.createTarget",
|
||||
"params": {"url": "about:blank"},
|
||||
});
|
||||
ws.send(Message::Text(create.to_string().into()))
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let mut session_id: Option<String> = None;
|
||||
while session_id.is_none() {
|
||||
let msg = ws.next().await.unwrap().unwrap();
|
||||
if let Message::Text(t) = msg {
|
||||
let v: Value = serde_json::from_str(&t).unwrap();
|
||||
session_id = v
|
||||
.get("params")
|
||||
.and_then(|p| p.get("sessionId"))
|
||||
.and_then(|s| s.as_str())
|
||||
.map(|s| s.to_string());
|
||||
}
|
||||
}
|
||||
let sid = session_id.unwrap();
|
||||
|
||||
// Fire a synchronous JS loop that holds V8 for JS_DURATION_MS.
|
||||
let code = format!(
|
||||
"var s=Date.now();while(Date.now()-s<{}){{}}'done'",
|
||||
JS_DURATION_MS
|
||||
);
|
||||
let eval = json!({
|
||||
"id": 2,
|
||||
"method": "Runtime.evaluate",
|
||||
"sessionId": sid,
|
||||
"params": {
|
||||
"expression": code,
|
||||
"awaitPromise": false,
|
||||
"returnByValue": true
|
||||
}
|
||||
});
|
||||
ws.send(Message::Text(eval.to_string().into()))
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
// Give the JS evaluation a moment to enter V8.
|
||||
tokio::time::sleep(Duration::from_millis(300)).await;
|
||||
|
||||
// Issue the HTTP request from a separate OS thread so the
|
||||
// LocalSet isn't blocked by the synchronous TCP connect/read.
|
||||
let handle = std::thread::spawn(move || {
|
||||
let start = Instant::now();
|
||||
let mut stream = std::net::TcpStream::connect_timeout(
|
||||
&format!("127.0.0.1:{}", port_clone)
|
||||
.parse()
|
||||
.unwrap(),
|
||||
HTTP_TIMEOUT,
|
||||
)?;
|
||||
stream.set_read_timeout(Some(HTTP_TIMEOUT))?;
|
||||
|
||||
let request = format!(
|
||||
"GET /json/version HTTP/1.1\r\nHost: 127.0.0.1:{}\r\nConnection: close\r\n\r\n",
|
||||
port_clone
|
||||
);
|
||||
stream.write_all(request.as_bytes())?;
|
||||
|
||||
let mut response = String::new();
|
||||
stream.read_to_string(&mut response)?;
|
||||
let elapsed = start.elapsed();
|
||||
Ok::<(String, Duration), std::io::Error>((response, elapsed))
|
||||
});
|
||||
|
||||
let result = handle.join().unwrap();
|
||||
let (response, elapsed) = result.unwrap();
|
||||
|
||||
assert!(
|
||||
response.contains("webSocketDebuggerUrl"),
|
||||
"/json/version must return valid JSON with webSocketDebuggerUrl.\nResponse: {}",
|
||||
&response[..response.len().min(500)]
|
||||
);
|
||||
assert!(
|
||||
elapsed < Duration::from_secs(2),
|
||||
"/json/version response too slow during JS eval: {:?}",
|
||||
elapsed
|
||||
);
|
||||
})
|
||||
.await;
|
||||
}
|
||||
@@ -0,0 +1,32 @@
|
||||
[package]
|
||||
name = "obscura-cli"
|
||||
version.workspace = true
|
||||
edition.workspace = true
|
||||
|
||||
[[bin]]
|
||||
name = "obscura"
|
||||
path = "src/main.rs"
|
||||
|
||||
[[bin]]
|
||||
name = "obscura-worker"
|
||||
path = "src/worker.rs"
|
||||
|
||||
[features]
|
||||
default = []
|
||||
stealth = ["obscura-browser/stealth", "obscura-net/stealth", "obscura-mcp/stealth"]
|
||||
|
||||
[dependencies]
|
||||
obscura-browser = { workspace = true }
|
||||
obscura-mcp = { workspace = true }
|
||||
obscura-cdp = { workspace = true }
|
||||
obscura-dom = { workspace = true }
|
||||
obscura-net = { workspace = true }
|
||||
obscura-js = { workspace = true }
|
||||
tokio = { workspace = true }
|
||||
clap = { workspace = true }
|
||||
tracing = { workspace = true }
|
||||
tracing-subscriber = { workspace = true }
|
||||
anyhow = { workspace = true }
|
||||
url = { workspace = true }
|
||||
serde = { workspace = true }
|
||||
serde_json = { workspace = true }
|
||||
@@ -0,0 +1,36 @@
|
||||
use std::env;
|
||||
|
||||
fn env_value(name: &str) -> Option<String> {
|
||||
env::var(name)
|
||||
.ok()
|
||||
.map(|value| value.trim().to_string())
|
||||
.filter(|value| !value.is_empty())
|
||||
}
|
||||
|
||||
fn normalize_version(version: String) -> String {
|
||||
version.strip_prefix('v').unwrap_or(&version).to_string()
|
||||
}
|
||||
|
||||
fn github_tag_version() -> Option<String> {
|
||||
if env_value("GITHUB_REF_TYPE").as_deref() == Some("tag") {
|
||||
return env_value("GITHUB_REF_NAME").map(normalize_version);
|
||||
}
|
||||
|
||||
env_value("GITHUB_REF")
|
||||
.and_then(|value| value.strip_prefix("refs/tags/").map(str::to_owned))
|
||||
.map(normalize_version)
|
||||
}
|
||||
|
||||
fn main() {
|
||||
println!("cargo:rerun-if-env-changed=OBSCURA_VERSION");
|
||||
println!("cargo:rerun-if-env-changed=GITHUB_REF_TYPE");
|
||||
println!("cargo:rerun-if-env-changed=GITHUB_REF_NAME");
|
||||
println!("cargo:rerun-if-env-changed=GITHUB_REF");
|
||||
|
||||
let version = env_value("OBSCURA_VERSION")
|
||||
.map(normalize_version)
|
||||
.or_else(github_tag_version)
|
||||
.unwrap_or_else(|| env::var("CARGO_PKG_VERSION").expect("Cargo sets CARGO_PKG_VERSION"));
|
||||
|
||||
println!("cargo:rustc-env=OBSCURA_BUILD_VERSION={version}");
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,145 @@
|
||||
|
||||
use std::sync::Arc;
|
||||
|
||||
use obscura_browser::{BrowserContext, Page};
|
||||
use serde::{Deserialize, Serialize};
|
||||
use tokio::io::{AsyncBufReadExt, AsyncWriteExt, BufReader};
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
#[serde(tag = "cmd")]
|
||||
enum WorkerCommand {
|
||||
#[serde(rename = "navigate")]
|
||||
Navigate { url: String },
|
||||
#[serde(rename = "evaluate")]
|
||||
Evaluate { expression: String },
|
||||
#[serde(rename = "title")]
|
||||
Title,
|
||||
#[serde(rename = "dump_html")]
|
||||
DumpHtml,
|
||||
#[serde(rename = "dump_text")]
|
||||
DumpText,
|
||||
#[serde(rename = "shutdown")]
|
||||
Shutdown,
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize)]
|
||||
struct WorkerResponse {
|
||||
ok: bool,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
result: Option<serde_json::Value>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
error: Option<String>,
|
||||
}
|
||||
|
||||
impl WorkerResponse {
|
||||
fn success(result: serde_json::Value) -> Self {
|
||||
WorkerResponse { ok: true, result: Some(result), error: None }
|
||||
}
|
||||
fn error(msg: String) -> Self {
|
||||
WorkerResponse { ok: false, result: None, error: Some(msg) }
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::main(flavor = "current_thread")]
|
||||
async fn main() {
|
||||
tracing_subscriber::fmt()
|
||||
.with_env_filter("warn")
|
||||
.with_writer(std::io::stderr)
|
||||
.init();
|
||||
|
||||
let proxy = std::env::var("OBSCURA_PROXY")
|
||||
.ok()
|
||||
.map(|value| value.trim().to_string())
|
||||
.filter(|value| !value.is_empty());
|
||||
let stealth = std::env::var("OBSCURA_STEALTH")
|
||||
.map(|v| matches!(v.trim(), "1" | "true" | "yes" | "on"))
|
||||
.unwrap_or(false);
|
||||
let context = Arc::new(BrowserContext::with_options("worker".to_string(), proxy, stealth));
|
||||
let mut page = Page::new("page-1".to_string(), context);
|
||||
|
||||
let stdin = tokio::io::stdin();
|
||||
let mut stdout = tokio::io::stdout();
|
||||
let mut reader = BufReader::new(stdin);
|
||||
let mut line = String::new();
|
||||
|
||||
loop {
|
||||
line.clear();
|
||||
match reader.read_line(&mut line).await {
|
||||
Ok(0) => break,
|
||||
Ok(_) => {}
|
||||
Err(e) => {
|
||||
eprintln!("Worker stdin error: {}", e);
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
let trimmed = line.trim();
|
||||
if trimmed.is_empty() {
|
||||
continue;
|
||||
}
|
||||
|
||||
let cmd: WorkerCommand = match serde_json::from_str(trimmed) {
|
||||
Ok(c) => c,
|
||||
Err(e) => {
|
||||
let resp = WorkerResponse::error(format!("Invalid command: {}", e));
|
||||
let mut out = serde_json::to_string(&resp).unwrap();
|
||||
out.push('\n');
|
||||
let _ = stdout.write_all(out.as_bytes()).await;
|
||||
let _ = stdout.flush().await;
|
||||
continue;
|
||||
}
|
||||
};
|
||||
|
||||
let resp = match cmd {
|
||||
WorkerCommand::Navigate { url } => {
|
||||
match page.navigate(&url).await {
|
||||
Ok(()) => WorkerResponse::success(serde_json::json!({
|
||||
"title": page.title,
|
||||
"url": page.url_string(),
|
||||
})),
|
||||
Err(e) => WorkerResponse::error(e.to_string()),
|
||||
}
|
||||
}
|
||||
WorkerCommand::Evaluate { expression } => {
|
||||
let result = page.evaluate(&expression);
|
||||
WorkerResponse::success(result)
|
||||
}
|
||||
WorkerCommand::Title => {
|
||||
WorkerResponse::success(serde_json::json!(page.title))
|
||||
}
|
||||
WorkerCommand::DumpHtml => {
|
||||
let html = page.with_dom(|dom| {
|
||||
if let Ok(Some(html_node)) = dom.query_selector("html") {
|
||||
dom.outer_html(html_node)
|
||||
} else {
|
||||
dom.inner_html(dom.document())
|
||||
}
|
||||
}).unwrap_or_default();
|
||||
WorkerResponse::success(serde_json::json!(html))
|
||||
}
|
||||
WorkerCommand::DumpText => {
|
||||
let text = page.with_dom(|dom| {
|
||||
if let Ok(Some(body)) = dom.query_selector("body") {
|
||||
dom.text_content(body)
|
||||
} else {
|
||||
String::new()
|
||||
}
|
||||
}).unwrap_or_default();
|
||||
WorkerResponse::success(serde_json::json!(text))
|
||||
}
|
||||
WorkerCommand::Shutdown => {
|
||||
let resp = WorkerResponse::success(serde_json::json!("bye"));
|
||||
let mut out = serde_json::to_string(&resp).unwrap();
|
||||
out.push('\n');
|
||||
let _ = stdout.write_all(out.as_bytes()).await;
|
||||
let _ = stdout.flush().await;
|
||||
break;
|
||||
}
|
||||
};
|
||||
|
||||
let mut out = serde_json::to_string(&resp).unwrap();
|
||||
out.push('\n');
|
||||
let _ = stdout.write_all(out.as_bytes()).await;
|
||||
let _ = stdout.flush().await;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,301 @@
|
||||
use std::io::{BufRead, BufReader, Write};
|
||||
use std::process::{Child, ChildStdin, Command, Stdio};
|
||||
|
||||
const OBSCURA: &str = env!("CARGO_BIN_EXE_obscura");
|
||||
|
||||
// ── minimal MCP client ────────────────────────────────────────────────────────
|
||||
|
||||
struct McpClient {
|
||||
child: Child,
|
||||
stdin: ChildStdin,
|
||||
reader: BufReader<std::process::ChildStdout>,
|
||||
next_id: u64,
|
||||
}
|
||||
|
||||
impl McpClient {
|
||||
fn spawn() -> Self {
|
||||
let mut child = Command::new(OBSCURA)
|
||||
.args(["mcp"])
|
||||
.stdin(Stdio::piped())
|
||||
.stdout(Stdio::piped())
|
||||
.stderr(Stdio::null())
|
||||
.spawn()
|
||||
.expect("failed to spawn obscura mcp");
|
||||
|
||||
let stdin = child.stdin.take().unwrap();
|
||||
let stdout = child.stdout.take().unwrap();
|
||||
|
||||
let mut client = McpClient {
|
||||
child,
|
||||
stdin,
|
||||
reader: BufReader::new(stdout),
|
||||
next_id: 1,
|
||||
};
|
||||
|
||||
// Perform the initialize handshake automatically
|
||||
let id = client.next_id;
|
||||
client.next_id += 1;
|
||||
let _resp = client.call_raw(serde_json::json!({
|
||||
"jsonrpc": "2.0",
|
||||
"id": id,
|
||||
"method": "initialize",
|
||||
"params": {
|
||||
"protocolVersion": "2024-11-05",
|
||||
"capabilities": {},
|
||||
"clientInfo": { "name": "test-client", "version": "0.0.0" }
|
||||
}
|
||||
}));
|
||||
|
||||
// Send initialized notification (no id, no response expected)
|
||||
client.notify("notifications/initialized", serde_json::json!({}));
|
||||
|
||||
client
|
||||
}
|
||||
|
||||
fn notify(&mut self, method: &str, params: serde_json::Value) {
|
||||
let msg = serde_json::json!({ "jsonrpc": "2.0", "method": method, "params": params });
|
||||
self.write_msg(&msg);
|
||||
}
|
||||
|
||||
fn call(&mut self, method: &str, params: serde_json::Value) -> serde_json::Value {
|
||||
let id = self.next_id;
|
||||
self.next_id += 1;
|
||||
let msg = serde_json::json!({
|
||||
"jsonrpc": "2.0",
|
||||
"id": id,
|
||||
"method": method,
|
||||
"params": params
|
||||
});
|
||||
self.call_raw(msg)
|
||||
}
|
||||
|
||||
fn call_raw(&mut self, msg: serde_json::Value) -> serde_json::Value {
|
||||
self.write_msg(&msg);
|
||||
self.read_msg()
|
||||
}
|
||||
|
||||
fn tool(&mut self, name: &str, args: serde_json::Value) -> serde_json::Value {
|
||||
self.call("tools/call", serde_json::json!({ "name": name, "arguments": args }))
|
||||
}
|
||||
|
||||
fn write_msg(&mut self, msg: &serde_json::Value) {
|
||||
// MCP stdio transport: newline-delimited JSON
|
||||
let mut body = serde_json::to_string(msg).unwrap();
|
||||
body.push('\n');
|
||||
self.stdin.write_all(body.as_bytes()).unwrap();
|
||||
self.stdin.flush().unwrap();
|
||||
}
|
||||
|
||||
fn read_msg(&mut self) -> serde_json::Value {
|
||||
let mut line = String::new();
|
||||
self.reader.read_line(&mut line).expect("read line");
|
||||
serde_json::from_str(line.trim()).expect("parse JSON")
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for McpClient {
|
||||
fn drop(&mut self) {
|
||||
let _ = self.child.kill();
|
||||
let _ = self.child.wait();
|
||||
}
|
||||
}
|
||||
|
||||
// helper: extract first content text from a tools/call response
|
||||
fn content_text(resp: &serde_json::Value) -> &str {
|
||||
resp["result"]["content"][0]["text"]
|
||||
.as_str()
|
||||
.unwrap_or("")
|
||||
}
|
||||
|
||||
// ── tests ─────────────────────────────────────────────────────────────────────
|
||||
|
||||
#[test]
|
||||
fn test_initialize() {
|
||||
let mut c = McpClient::spawn();
|
||||
// A second initialize should still return a valid response
|
||||
let resp = c.call(
|
||||
"initialize",
|
||||
serde_json::json!({
|
||||
"protocolVersion": "2024-11-05",
|
||||
"capabilities": {}
|
||||
}),
|
||||
);
|
||||
assert_eq!(resp["result"]["protocolVersion"], "2024-11-05");
|
||||
assert_eq!(resp["result"]["serverInfo"]["name"], "obscura-mcp");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_ping() {
|
||||
let mut c = McpClient::spawn();
|
||||
let resp = c.call("ping", serde_json::json!({}));
|
||||
assert!(resp.get("error").is_none(), "ping should not error");
|
||||
assert_eq!(resp["result"], serde_json::json!({}));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_tools_list() {
|
||||
let mut c = McpClient::spawn();
|
||||
let resp = c.call("tools/list", serde_json::json!({}));
|
||||
let tools = resp["result"]["tools"].as_array().expect("tools array");
|
||||
assert!(!tools.is_empty());
|
||||
let names: Vec<&str> = tools.iter().map(|t| t["name"].as_str().unwrap()).collect();
|
||||
for expected in &[
|
||||
"browser_navigate",
|
||||
"browser_snapshot",
|
||||
"browser_click",
|
||||
"browser_fill",
|
||||
"browser_type",
|
||||
"browser_press_key",
|
||||
"browser_select_option",
|
||||
"browser_evaluate",
|
||||
"browser_wait_for",
|
||||
"browser_network_requests",
|
||||
"browser_console_messages",
|
||||
"browser_close",
|
||||
] {
|
||||
assert!(names.contains(expected), "missing tool: {expected}");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_resources_list() {
|
||||
let mut c = McpClient::spawn();
|
||||
let resp = c.call("resources/list", serde_json::json!({}));
|
||||
assert!(resp.get("error").is_none());
|
||||
assert_eq!(resp["result"]["resources"], serde_json::json!([]));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_prompts_list() {
|
||||
let mut c = McpClient::spawn();
|
||||
let resp = c.call("prompts/list", serde_json::json!({}));
|
||||
assert!(resp.get("error").is_none());
|
||||
assert_eq!(resp["result"]["prompts"], serde_json::json!([]));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_unknown_method_returns_error() {
|
||||
let mut c = McpClient::spawn();
|
||||
let resp = c.call("nonexistent/method", serde_json::json!({}));
|
||||
assert_eq!(resp["error"]["code"], -32601);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_notifications_are_silent() {
|
||||
// Notifications must not produce a response; the next real call should succeed
|
||||
let mut c = McpClient::spawn();
|
||||
c.notify("notifications/initialized", serde_json::json!({}));
|
||||
c.notify("some/other_notification", serde_json::json!({"x": 1}));
|
||||
// If the server accidentally replied to a notification, the next read would
|
||||
// consume that stray response and the ping result would be mismatched.
|
||||
let resp = c.call("ping", serde_json::json!({}));
|
||||
assert!(resp.get("error").is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_navigate_and_snapshot() {
|
||||
let mut c = McpClient::spawn();
|
||||
|
||||
let nav = c.tool("browser_navigate", serde_json::json!({"url": "https://example.com"}));
|
||||
assert!(nav["result"]["isError"].is_null(), "navigate failed: {nav}");
|
||||
let text = content_text(&nav);
|
||||
assert!(text.contains("example.com"), "unexpected nav text: {text}");
|
||||
|
||||
let snap = c.tool("browser_snapshot", serde_json::json!({}));
|
||||
assert!(snap["result"]["isError"].is_null(), "snapshot failed: {snap}");
|
||||
let text = content_text(&snap);
|
||||
assert!(text.contains("Example Domain"), "unexpected snapshot: {text}");
|
||||
assert!(text.contains("URL:"), "snapshot missing URL line");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_evaluate() {
|
||||
let mut c = McpClient::spawn();
|
||||
c.tool("browser_navigate", serde_json::json!({"url": "https://example.com"}));
|
||||
|
||||
let resp = c.tool(
|
||||
"browser_evaluate",
|
||||
serde_json::json!({"expression": "document.title"}),
|
||||
);
|
||||
let text = content_text(&resp);
|
||||
assert_eq!(text, "Example Domain");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_evaluate_math() {
|
||||
let mut c = McpClient::spawn();
|
||||
c.tool("browser_navigate", serde_json::json!({"url": "https://example.com"}));
|
||||
|
||||
let resp = c.tool(
|
||||
"browser_evaluate",
|
||||
serde_json::json!({"expression": "1 + 2"}),
|
||||
);
|
||||
let text = content_text(&resp);
|
||||
// V8 serialises integer results as floats ("3" or "3.0" depending on context)
|
||||
assert!(text == "3" || text == "3.0", "unexpected result: {text}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_wait_for_selector() {
|
||||
let mut c = McpClient::spawn();
|
||||
c.tool("browser_navigate", serde_json::json!({"url": "https://example.com"}));
|
||||
|
||||
let resp = c.tool(
|
||||
"browser_wait_for",
|
||||
serde_json::json!({"selector": "h1", "timeout": 5}),
|
||||
);
|
||||
assert!(resp["result"]["isError"].is_null(), "wait_for failed: {resp}");
|
||||
assert!(content_text(&resp).contains("Found"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_wait_for_timeout() {
|
||||
let mut c = McpClient::spawn();
|
||||
c.tool("browser_navigate", serde_json::json!({"url": "https://example.com"}));
|
||||
|
||||
let resp = c.tool(
|
||||
"browser_wait_for",
|
||||
serde_json::json!({"selector": "#does-not-exist", "timeout": 1}),
|
||||
);
|
||||
assert_eq!(resp["result"]["isError"], true);
|
||||
assert!(content_text(&resp).contains("Timeout"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_navigate_missing_url_returns_error() {
|
||||
let mut c = McpClient::spawn();
|
||||
let resp = c.tool("browser_navigate", serde_json::json!({}));
|
||||
assert_eq!(resp["result"]["isError"], true);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_unknown_tool_returns_error() {
|
||||
let mut c = McpClient::spawn();
|
||||
let resp = c.tool("browser_does_not_exist", serde_json::json!({}));
|
||||
assert_eq!(resp["result"]["isError"], true);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_network_requests() {
|
||||
let mut c = McpClient::spawn();
|
||||
c.tool("browser_navigate", serde_json::json!({"url": "https://example.com"}));
|
||||
|
||||
let resp = c.tool("browser_network_requests", serde_json::json!({}));
|
||||
let text = content_text(&resp);
|
||||
assert!(
|
||||
text.contains("example.com") || text.contains("No network"),
|
||||
"unexpected: {text}"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_close_resets_state() {
|
||||
let mut c = McpClient::spawn();
|
||||
c.tool("browser_navigate", serde_json::json!({"url": "https://example.com"}));
|
||||
let close = c.tool("browser_close", serde_json::json!({}));
|
||||
assert!(close["result"]["isError"].is_null());
|
||||
|
||||
// After close, snapshot should return empty/default page (no panic)
|
||||
let snap = c.tool("browser_snapshot", serde_json::json!({}));
|
||||
assert!(snap["result"]["isError"].is_null());
|
||||
}
|
||||
@@ -0,0 +1,16 @@
|
||||
[package]
|
||||
name = "obscura-dom"
|
||||
version.workspace = true
|
||||
edition.workspace = true
|
||||
|
||||
[dependencies]
|
||||
html5ever = { workspace = true }
|
||||
markup5ever = { workspace = true }
|
||||
selectors = { workspace = true }
|
||||
servo_arc = { workspace = true }
|
||||
cssparser = { workspace = true }
|
||||
thiserror = { workspace = true }
|
||||
tracing = { workspace = true }
|
||||
precomputed-hash = "0.1"
|
||||
|
||||
[dev-dependencies]
|
||||
@@ -0,0 +1,10 @@
|
||||
#[macro_use]
|
||||
extern crate html5ever;
|
||||
|
||||
pub mod tree;
|
||||
pub mod tree_sink;
|
||||
pub mod selector;
|
||||
pub mod serialize;
|
||||
|
||||
pub use tree::{Attribute, DomTree, Node, NodeData, NodeId};
|
||||
pub use tree_sink::{parse_html, parse_fragment};
|
||||
@@ -0,0 +1,789 @@
|
||||
use cssparser::{CowRcStr, ToCss};
|
||||
use html5ever::{LocalName, Namespace};
|
||||
use precomputed_hash::PrecomputedHash;
|
||||
use selectors::attr::{AttrSelectorOperation, CaseSensitivity, NamespaceConstraint};
|
||||
use selectors::bloom::BloomFilter;
|
||||
use selectors::context::QuirksMode;
|
||||
use selectors::matching::{
|
||||
ElementSelectorFlags, MatchingContext, MatchingForInvalidation, MatchingMode,
|
||||
NeedsSelectorFlags,
|
||||
};
|
||||
use selectors::parser::{self, ParseRelative, SelectorParseErrorKind};
|
||||
use selectors::{Element, OpaqueElement, SelectorList};
|
||||
use selectors::visitor::SelectorVisitor;
|
||||
|
||||
use crate::tree::{DomTree, NodeData, NodeId};
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct ObscuraSelector;
|
||||
|
||||
impl parser::SelectorImpl for ObscuraSelector {
|
||||
type ExtraMatchingData<'a> = ();
|
||||
type AttrValue = CssString;
|
||||
type Identifier = CssString;
|
||||
type LocalName = CssLocalName;
|
||||
type NamespaceUrl = CssNamespace;
|
||||
type NamespacePrefix = CssString;
|
||||
type BorrowedLocalName = CssLocalName;
|
||||
type BorrowedNamespaceUrl = CssNamespace;
|
||||
type NonTSPseudoClass = PseudoClass;
|
||||
type PseudoElement = PseudoElement;
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Hash)]
|
||||
pub struct CssString(pub String);
|
||||
|
||||
impl<'a> From<&'a str> for CssString {
|
||||
fn from(s: &'a str) -> Self {
|
||||
CssString(s.to_string())
|
||||
}
|
||||
}
|
||||
|
||||
impl AsRef<str> for CssString {
|
||||
fn as_ref(&self) -> &str {
|
||||
&self.0
|
||||
}
|
||||
}
|
||||
|
||||
impl ToCss for CssString {
|
||||
fn to_css<W: std::fmt::Write>(&self, dest: &mut W) -> std::fmt::Result {
|
||||
cssparser::serialize_string(&self.0, dest)
|
||||
}
|
||||
}
|
||||
|
||||
impl PrecomputedHash for CssString {
|
||||
fn precomputed_hash(&self) -> u32 {
|
||||
let mut h: u32 = 5381;
|
||||
for b in self.0.as_bytes() {
|
||||
h = h.wrapping_mul(33).wrapping_add(*b as u32);
|
||||
}
|
||||
h
|
||||
}
|
||||
}
|
||||
|
||||
impl Default for CssString {
|
||||
fn default() -> Self {
|
||||
CssString(String::new())
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Hash)]
|
||||
pub struct CssLocalName(pub LocalName);
|
||||
|
||||
impl<'a> From<&'a str> for CssLocalName {
|
||||
fn from(s: &'a str) -> Self {
|
||||
CssLocalName(LocalName::from(s))
|
||||
}
|
||||
}
|
||||
|
||||
impl ToCss for CssLocalName {
|
||||
fn to_css<W: std::fmt::Write>(&self, dest: &mut W) -> std::fmt::Result {
|
||||
dest.write_str(&self.0)
|
||||
}
|
||||
}
|
||||
|
||||
impl PrecomputedHash for CssLocalName {
|
||||
fn precomputed_hash(&self) -> u32 {
|
||||
self.0.precomputed_hash()
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Hash, Default)]
|
||||
pub struct CssNamespace(pub Namespace);
|
||||
|
||||
impl PrecomputedHash for CssNamespace {
|
||||
fn precomputed_hash(&self) -> u32 {
|
||||
self.0.precomputed_hash()
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub enum PseudoClass {
|
||||
Hover,
|
||||
Active,
|
||||
Focus,
|
||||
Enabled,
|
||||
Disabled,
|
||||
Checked,
|
||||
}
|
||||
|
||||
impl parser::NonTSPseudoClass for PseudoClass {
|
||||
type Impl = ObscuraSelector;
|
||||
|
||||
fn is_active_or_hover(&self) -> bool {
|
||||
matches!(self, PseudoClass::Hover | PseudoClass::Active)
|
||||
}
|
||||
|
||||
fn is_user_action_state(&self) -> bool {
|
||||
matches!(
|
||||
self,
|
||||
PseudoClass::Hover | PseudoClass::Active | PseudoClass::Focus
|
||||
)
|
||||
}
|
||||
|
||||
fn visit<V>(&self, _visitor: &mut V) -> bool
|
||||
where
|
||||
V: SelectorVisitor<Impl = Self::Impl>,
|
||||
{
|
||||
true
|
||||
}
|
||||
}
|
||||
|
||||
impl ToCss for PseudoClass {
|
||||
fn to_css<W: std::fmt::Write>(&self, dest: &mut W) -> std::fmt::Result {
|
||||
match self {
|
||||
PseudoClass::Hover => dest.write_str(":hover"),
|
||||
PseudoClass::Active => dest.write_str(":active"),
|
||||
PseudoClass::Focus => dest.write_str(":focus"),
|
||||
PseudoClass::Enabled => dest.write_str(":enabled"),
|
||||
PseudoClass::Disabled => dest.write_str(":disabled"),
|
||||
PseudoClass::Checked => dest.write_str(":checked"),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub enum PseudoElement {
|
||||
Before,
|
||||
After,
|
||||
}
|
||||
|
||||
impl parser::PseudoElement for PseudoElement {
|
||||
type Impl = ObscuraSelector;
|
||||
}
|
||||
|
||||
impl ToCss for PseudoElement {
|
||||
fn to_css<W: std::fmt::Write>(&self, dest: &mut W) -> std::fmt::Result {
|
||||
match self {
|
||||
PseudoElement::Before => dest.write_str("::before"),
|
||||
PseudoElement::After => dest.write_str("::after"),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub struct ObscuraSelectorParser;
|
||||
|
||||
impl<'i> parser::Parser<'i> for ObscuraSelectorParser {
|
||||
type Impl = ObscuraSelector;
|
||||
type Error = SelectorParseErrorKind<'i>;
|
||||
|
||||
// Allow `:has()`. The selectors crate gates relative-selector parsing on
|
||||
// this (default false), so without it `a:has(p)` failed to parse and the
|
||||
// error was swallowed into an empty match set. Matching already passes a
|
||||
// SelectorCaches (which holds the relative-selector cache), so enabling
|
||||
// parsing is sufficient.
|
||||
fn parse_has(&self) -> bool {
|
||||
true
|
||||
}
|
||||
|
||||
fn parse_non_ts_pseudo_class(
|
||||
&self,
|
||||
_location: cssparser::SourceLocation,
|
||||
name: CowRcStr<'i>,
|
||||
) -> Result<PseudoClass, cssparser::ParseError<'i, Self::Error>> {
|
||||
match name.as_ref() {
|
||||
"hover" => Ok(PseudoClass::Hover),
|
||||
"active" => Ok(PseudoClass::Active),
|
||||
"focus" => Ok(PseudoClass::Focus),
|
||||
"enabled" => Ok(PseudoClass::Enabled),
|
||||
"disabled" => Ok(PseudoClass::Disabled),
|
||||
"checked" => Ok(PseudoClass::Checked),
|
||||
_ => Err(cssparser::ParseError {
|
||||
kind: cssparser::ParseErrorKind::Custom(
|
||||
SelectorParseErrorKind::UnsupportedPseudoClassOrElement(name),
|
||||
),
|
||||
location: _location,
|
||||
}),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy)]
|
||||
pub struct DomElement<'a> {
|
||||
pub tree: &'a DomTree,
|
||||
pub node_id: NodeId,
|
||||
}
|
||||
|
||||
impl<'a> DomElement<'a> {
|
||||
pub fn new(tree: &'a DomTree, node_id: NodeId) -> Self {
|
||||
DomElement { tree, node_id }
|
||||
}
|
||||
}
|
||||
|
||||
impl<'a> std::fmt::Debug for DomElement<'a> {
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
write!(f, "DomElement({:?})", self.node_id)
|
||||
}
|
||||
}
|
||||
|
||||
impl<'a> PartialEq for DomElement<'a> {
|
||||
fn eq(&self, other: &Self) -> bool {
|
||||
self.node_id == other.node_id
|
||||
}
|
||||
}
|
||||
|
||||
impl<'a> Eq for DomElement<'a> {}
|
||||
|
||||
impl<'a> Element for DomElement<'a> {
|
||||
type Impl = ObscuraSelector;
|
||||
|
||||
fn opaque(&self) -> OpaqueElement {
|
||||
// Must be stable per node. DomElement is Copy and gets a fresh stack
|
||||
// address on every traversal step, so OpaqueElement::new(self) returns a
|
||||
// different id for the same node each call. That breaks `:has` anchor
|
||||
// matching, which compares the anchor's opaque against the element
|
||||
// reached by walking up the tree. Key off the node's stable slot in the
|
||||
// tree's Vec instead (OpaqueElement only compares the address, never
|
||||
// dereferences it, and the Vec is not mutated during a query).
|
||||
let inner = self.tree.borrow_inner();
|
||||
match inner.nodes.get(self.node_id.index()) {
|
||||
Some(slot) => OpaqueElement::new(slot),
|
||||
None => OpaqueElement::new(self),
|
||||
}
|
||||
}
|
||||
|
||||
fn parent_element(&self) -> Option<Self> {
|
||||
let node = self.tree.get_node(self.node_id)?;
|
||||
let parent_id = node.parent?;
|
||||
let parent = self.tree.get_node(parent_id)?;
|
||||
if parent.is_element() {
|
||||
Some(DomElement::new(self.tree, parent_id))
|
||||
} else {
|
||||
None
|
||||
}
|
||||
}
|
||||
|
||||
fn parent_node_is_shadow_root(&self) -> bool {
|
||||
false
|
||||
}
|
||||
|
||||
fn containing_shadow_host(&self) -> Option<Self> {
|
||||
None
|
||||
}
|
||||
|
||||
fn pseudo_element_originating_element(&self) -> Option<Self> {
|
||||
None
|
||||
}
|
||||
|
||||
fn is_pseudo_element(&self) -> bool {
|
||||
false
|
||||
}
|
||||
|
||||
fn prev_sibling_element(&self) -> Option<Self> {
|
||||
let node = self.tree.get_node(self.node_id)?;
|
||||
let mut current = node.prev_sibling;
|
||||
while let Some(sibling_id) = current {
|
||||
let sibling = self.tree.get_node(sibling_id)?;
|
||||
if sibling.is_element() {
|
||||
return Some(DomElement::new(self.tree, sibling_id));
|
||||
}
|
||||
current = sibling.prev_sibling;
|
||||
}
|
||||
None
|
||||
}
|
||||
|
||||
fn next_sibling_element(&self) -> Option<Self> {
|
||||
let node = self.tree.get_node(self.node_id)?;
|
||||
let mut current = node.next_sibling;
|
||||
while let Some(sibling_id) = current {
|
||||
let sibling = self.tree.get_node(sibling_id)?;
|
||||
if sibling.is_element() {
|
||||
return Some(DomElement::new(self.tree, sibling_id));
|
||||
}
|
||||
current = sibling.next_sibling;
|
||||
}
|
||||
None
|
||||
}
|
||||
|
||||
fn first_element_child(&self) -> Option<Self> {
|
||||
let node = self.tree.get_node(self.node_id)?;
|
||||
let mut current = node.first_child;
|
||||
while let Some(child_id) = current {
|
||||
let child = self.tree.get_node(child_id)?;
|
||||
if child.is_element() {
|
||||
return Some(DomElement::new(self.tree, child_id));
|
||||
}
|
||||
current = child.next_sibling;
|
||||
}
|
||||
None
|
||||
}
|
||||
|
||||
fn is_html_element_in_html_document(&self) -> bool {
|
||||
self.tree
|
||||
.with_node(self.node_id, |n| {
|
||||
n.as_element()
|
||||
.map(|name| name.ns == ns!(html))
|
||||
.unwrap_or(false)
|
||||
})
|
||||
.unwrap_or(false)
|
||||
}
|
||||
|
||||
fn has_local_name(&self, local_name: &CssLocalName) -> bool {
|
||||
self.tree
|
||||
.with_node(self.node_id, |n| {
|
||||
n.as_element()
|
||||
.map(|name| name.local == local_name.0)
|
||||
.unwrap_or(false)
|
||||
})
|
||||
.unwrap_or(false)
|
||||
}
|
||||
|
||||
fn has_namespace(&self, ns: &CssNamespace) -> bool {
|
||||
self.tree
|
||||
.with_node(self.node_id, |n| {
|
||||
n.as_element()
|
||||
.map(|name| name.ns == ns.0)
|
||||
.unwrap_or(false)
|
||||
})
|
||||
.unwrap_or(false)
|
||||
}
|
||||
|
||||
fn is_same_type(&self, other: &Self) -> bool {
|
||||
let self_name = self.tree.with_node(self.node_id, |n| {
|
||||
n.as_element().map(|name| (name.local.clone(), name.ns.clone()))
|
||||
}).flatten();
|
||||
let other_name = self.tree.with_node(other.node_id, |n| {
|
||||
n.as_element().map(|name| (name.local.clone(), name.ns.clone()))
|
||||
}).flatten();
|
||||
match (self_name, other_name) {
|
||||
(Some((al, ans)), Some((bl, bns))) => al == bl && ans == bns,
|
||||
_ => false,
|
||||
}
|
||||
}
|
||||
|
||||
fn attr_matches(
|
||||
&self,
|
||||
ns: &NamespaceConstraint<&CssNamespace>,
|
||||
local_name: &CssLocalName,
|
||||
operation: &AttrSelectorOperation<&CssString>,
|
||||
) -> bool {
|
||||
self.tree
|
||||
.with_node(self.node_id, |node| {
|
||||
let attrs = match node.attrs() {
|
||||
Some(a) => a,
|
||||
None => return false,
|
||||
};
|
||||
attrs.iter().any(|attr| {
|
||||
let ns_match = match ns {
|
||||
NamespaceConstraint::Any => true,
|
||||
NamespaceConstraint::Specific(expected_ns) => attr.name.ns == expected_ns.0,
|
||||
};
|
||||
if !ns_match || attr.name.local != local_name.0 {
|
||||
return false;
|
||||
}
|
||||
operation.eval_str(&attr.value)
|
||||
})
|
||||
})
|
||||
.unwrap_or(false)
|
||||
}
|
||||
|
||||
fn has_attr_in_no_namespace(&self, local_name: &CssLocalName) -> bool {
|
||||
self.tree
|
||||
.with_node(self.node_id, |node| {
|
||||
node.attrs()
|
||||
.map(|attrs| {
|
||||
attrs.iter().any(|a| {
|
||||
a.name.ns == html5ever::ns!()
|
||||
&& a.name.local == local_name.0
|
||||
})
|
||||
})
|
||||
.unwrap_or(false)
|
||||
})
|
||||
.unwrap_or(false)
|
||||
}
|
||||
|
||||
fn match_non_ts_pseudo_class(
|
||||
&self,
|
||||
_pc: &PseudoClass,
|
||||
_context: &mut MatchingContext<'_, Self::Impl>,
|
||||
) -> bool {
|
||||
false
|
||||
}
|
||||
|
||||
fn match_pseudo_element(
|
||||
&self,
|
||||
_pe: &PseudoElement,
|
||||
_context: &mut MatchingContext<'_, Self::Impl>,
|
||||
) -> bool {
|
||||
false
|
||||
}
|
||||
|
||||
fn apply_selector_flags(&self, _flags: ElementSelectorFlags) {}
|
||||
|
||||
fn is_link(&self) -> bool {
|
||||
self.tree
|
||||
.with_node(self.node_id, |n| {
|
||||
n.as_element()
|
||||
.map(|name| {
|
||||
matches!(name.local.as_ref(), "a" | "area" | "link")
|
||||
&& n.get_attribute("href").is_some()
|
||||
})
|
||||
.unwrap_or(false)
|
||||
})
|
||||
.unwrap_or(false)
|
||||
}
|
||||
|
||||
fn is_html_slot_element(&self) -> bool {
|
||||
false
|
||||
}
|
||||
|
||||
fn assigned_slot(&self) -> Option<Self> {
|
||||
None
|
||||
}
|
||||
|
||||
fn has_id(&self, id: &CssString, case_sensitivity: CaseSensitivity) -> bool {
|
||||
self.tree
|
||||
.with_node(self.node_id, |n| {
|
||||
n.get_attribute("id")
|
||||
.map(|value| case_sensitivity.eq(value.as_bytes(), id.0.as_bytes()))
|
||||
.unwrap_or(false)
|
||||
})
|
||||
.unwrap_or(false)
|
||||
}
|
||||
|
||||
fn has_class(&self, name: &CssString, case_sensitivity: CaseSensitivity) -> bool {
|
||||
self.tree
|
||||
.with_node(self.node_id, |n| {
|
||||
n.get_attribute("class")
|
||||
.map(|class_attr| {
|
||||
class_attr
|
||||
.split_whitespace()
|
||||
.any(|c| case_sensitivity.eq(c.as_bytes(), name.0.as_bytes()))
|
||||
})
|
||||
.unwrap_or(false)
|
||||
})
|
||||
.unwrap_or(false)
|
||||
}
|
||||
|
||||
fn has_custom_state(&self, _name: &CssString) -> bool {
|
||||
false
|
||||
}
|
||||
|
||||
fn imported_part(&self, _name: &CssString) -> Option<CssString> {
|
||||
None
|
||||
}
|
||||
|
||||
fn is_part(&self, _name: &CssString) -> bool {
|
||||
false
|
||||
}
|
||||
|
||||
fn is_empty(&self) -> bool {
|
||||
self.tree
|
||||
.with_node(self.node_id, |node| {
|
||||
let mut child = node.first_child;
|
||||
while let Some(child_id) = child {
|
||||
if let Some(child_node) = self.tree.get_node(child_id) {
|
||||
match &child_node.data {
|
||||
NodeData::Element { .. } => return false,
|
||||
NodeData::Text { contents } if !contents.is_empty() => return false,
|
||||
_ => {}
|
||||
}
|
||||
child = child_node.next_sibling;
|
||||
} else {
|
||||
break;
|
||||
}
|
||||
}
|
||||
true
|
||||
})
|
||||
.unwrap_or(true)
|
||||
}
|
||||
|
||||
fn is_root(&self) -> bool {
|
||||
self.tree
|
||||
.with_node(self.node_id, |n| {
|
||||
n.parent
|
||||
.map(|parent_id| {
|
||||
self.tree
|
||||
.with_node(parent_id, |p| p.is_document())
|
||||
.unwrap_or(false)
|
||||
})
|
||||
.unwrap_or(false)
|
||||
})
|
||||
.unwrap_or(false)
|
||||
}
|
||||
|
||||
fn ignores_nth_child_selectors(&self) -> bool {
|
||||
false
|
||||
}
|
||||
|
||||
fn add_element_unique_hashes(&self, _filter: &mut BloomFilter) -> bool {
|
||||
false
|
||||
}
|
||||
}
|
||||
|
||||
// Thread-local LRU cache of parsed selectors. Without this every
|
||||
// querySelector / querySelectorAll re-parses the selector string;
|
||||
// for batch-heavy DOM access (agent scraping a table, framework
|
||||
// repeatedly polling for elements) the parse cost adds up to tens
|
||||
// of ms per page. Cap of 256 entries fits a typical page's distinct
|
||||
// selectors without unbounded memory growth.
|
||||
thread_local! {
|
||||
static SELECTOR_CACHE: std::cell::RefCell<
|
||||
std::collections::HashMap<String, std::sync::Arc<SelectorList<ObscuraSelector>>>,
|
||||
> = std::cell::RefCell::new(std::collections::HashMap::with_capacity(64));
|
||||
}
|
||||
const SELECTOR_CACHE_CAP: usize = 256;
|
||||
|
||||
fn parse_selector_uncached(selector: &str) -> Result<SelectorList<ObscuraSelector>, String> {
|
||||
let mut parser_input = cssparser::ParserInput::new(selector);
|
||||
let mut parser = cssparser::Parser::new(&mut parser_input);
|
||||
SelectorList::parse(&ObscuraSelectorParser, &mut parser, ParseRelative::No)
|
||||
.map_err(|e| format!("Failed to parse selector '{}': {:?}", selector, e))
|
||||
}
|
||||
|
||||
pub fn parse_selector(selector: &str) -> Result<SelectorList<ObscuraSelector>, String> {
|
||||
// Hot path: cached. Cold path: parse + insert.
|
||||
if let Some(cached) = SELECTOR_CACHE.with(|c| c.borrow().get(selector).cloned()) {
|
||||
return Ok((*cached).clone());
|
||||
}
|
||||
let parsed = parse_selector_uncached(selector)?;
|
||||
let cached = std::sync::Arc::new(parsed.clone());
|
||||
SELECTOR_CACHE.with(|c| {
|
||||
let mut cache = c.borrow_mut();
|
||||
// Crude eviction: if at cap, dump the whole table. A real LRU
|
||||
// would be more memory-friendly but selectors are small and 256
|
||||
// is comfortably above a single page's distinct-selector count.
|
||||
if cache.len() >= SELECTOR_CACHE_CAP {
|
||||
cache.clear();
|
||||
}
|
||||
cache.insert(selector.to_string(), cached);
|
||||
});
|
||||
Ok(parsed)
|
||||
}
|
||||
|
||||
/// If `selector` is a bare ASCII id selector like `#main`, return the id (without
|
||||
/// the `#`). Conservative: escapes, combinators, commas, whitespace, non-ASCII, or
|
||||
/// a non-letter/underscore first character fall through to the full selector engine.
|
||||
fn simple_id_selector(selector: &str) -> Option<&str> {
|
||||
let id = selector.trim().strip_prefix('#')?;
|
||||
let first = id.as_bytes().first().copied()?;
|
||||
if !(first.is_ascii_alphabetic() || first == b'_') {
|
||||
return None;
|
||||
}
|
||||
if id.bytes().all(|b| b.is_ascii_alphanumeric() || b == b'_' || b == b'-') {
|
||||
Some(id)
|
||||
} else {
|
||||
None
|
||||
}
|
||||
}
|
||||
|
||||
impl DomTree {
|
||||
pub fn query_selector(&self, selector: &str) -> Result<Option<NodeId>, String> {
|
||||
self.query_selector_from(self.document(), selector)
|
||||
}
|
||||
|
||||
pub fn query_selector_all(&self, selector: &str) -> Result<Vec<NodeId>, String> {
|
||||
self.query_selector_all_from(self.document(), selector)
|
||||
}
|
||||
|
||||
pub fn query_selector_from(&self, root: NodeId, selector: &str) -> Result<Option<NodeId>, String> {
|
||||
// Fast path: a bare "#id" selector resolves through the id index in O(1)
|
||||
// instead of scanning every descendant. The index holds the first element
|
||||
// in tree order per id, which is exactly what the full scan would return.
|
||||
if let Some(id) = simple_id_selector(selector) {
|
||||
match self.get_element_by_id(id) {
|
||||
// querySelector matches strict descendants of root only, so the
|
||||
// indexed element must have root among its ancestors.
|
||||
Some(nid) if self.ancestors(nid).contains(&root) => return Ok(Some(nid)),
|
||||
// Index miss or stale entry (detached node) — fall through to full scan.
|
||||
// The id_index is best-effort: it only registers nodes at creation time
|
||||
// and does not update on reparent, so it can point to a detached clone
|
||||
// while the live node (with the same id) is elsewhere in the tree.
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
let selector_list = parse_selector(selector)?;
|
||||
let mut caches = selectors::context::SelectorCaches::default();
|
||||
let mut context = MatchingContext::new(
|
||||
MatchingMode::Normal,
|
||||
None,
|
||||
&mut caches,
|
||||
QuirksMode::NoQuirks,
|
||||
NeedsSelectorFlags::No,
|
||||
MatchingForInvalidation::No,
|
||||
);
|
||||
|
||||
for desc_id in self.descendants(root) {
|
||||
let is_element = self.with_node(desc_id, |n| n.is_element()).unwrap_or(false);
|
||||
if is_element {
|
||||
let element = DomElement::new(self, desc_id);
|
||||
if selectors::matching::matches_selector_list(
|
||||
&selector_list,
|
||||
&element,
|
||||
&mut context,
|
||||
) {
|
||||
return Ok(Some(desc_id));
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(None)
|
||||
}
|
||||
|
||||
pub fn query_selector_all_from(&self, root: NodeId, selector: &str) -> Result<Vec<NodeId>, String> {
|
||||
let selector_list = parse_selector(selector)?;
|
||||
let mut caches = selectors::context::SelectorCaches::default();
|
||||
let mut context = MatchingContext::new(
|
||||
MatchingMode::Normal,
|
||||
None,
|
||||
&mut caches,
|
||||
QuirksMode::NoQuirks,
|
||||
NeedsSelectorFlags::No,
|
||||
MatchingForInvalidation::No,
|
||||
);
|
||||
let mut results = Vec::new();
|
||||
|
||||
for desc_id in self.descendants(root) {
|
||||
let is_element = self.with_node(desc_id, |n| n.is_element()).unwrap_or(false);
|
||||
if is_element {
|
||||
let element = DomElement::new(self, desc_id);
|
||||
if selectors::matching::matches_selector_list(
|
||||
&selector_list,
|
||||
&element,
|
||||
&mut context,
|
||||
) {
|
||||
results.push(desc_id);
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(results)
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use crate::tree_sink::parse_html;
|
||||
|
||||
#[test]
|
||||
fn test_query_selector_tag() {
|
||||
let tree = parse_html("<html><body><h1>Title</h1><p>Text</p></body></html>");
|
||||
let result = tree.query_selector("h1").unwrap();
|
||||
assert!(result.is_some());
|
||||
let node = tree.get_node(result.unwrap()).unwrap();
|
||||
assert_eq!(node.as_element().unwrap().local.as_ref(), "h1");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_query_selector_class() {
|
||||
let tree =
|
||||
parse_html(r#"<div class="foo bar">Content</div><div class="baz">Other</div>"#);
|
||||
let result = tree.query_selector(".foo").unwrap();
|
||||
assert!(result.is_some());
|
||||
let node = tree.get_node(result.unwrap()).unwrap();
|
||||
assert_eq!(node.get_attribute("class"), Some("foo bar"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_query_selector_id() {
|
||||
let tree = parse_html(r#"<div id="main">Content</div>"#);
|
||||
let result = tree.query_selector("#main").unwrap();
|
||||
assert!(result.is_some());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_query_selector_all() {
|
||||
let tree = parse_html("<ul><li>1</li><li>2</li><li>3</li></ul>");
|
||||
let results = tree.query_selector_all("li").unwrap();
|
||||
assert_eq!(results.len(), 3);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_has_parses() {
|
||||
// Isolate parse from match: :has must parse, not error.
|
||||
assert!(super::parse_selector("a:has(p.bt)").is_ok(), ":has failed to parse");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_query_selector_has() {
|
||||
let tree = parse_html(r#"<a><p class="bt">x</p></a>"#);
|
||||
let all = tree.query_selector_all("a:has(p.bt)").unwrap();
|
||||
assert_eq!(all.len(), 1, "a:has(p.bt) should match the <a>");
|
||||
let none = tree.query_selector_all("a:has(span.bt)").unwrap();
|
||||
assert_eq!(none.len(), 0, "a:has(span.bt) should match nothing");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_query_selector_descendant() {
|
||||
let tree =
|
||||
parse_html(r#"<div id="outer"><div id="inner"><span>Target</span></div></div>"#);
|
||||
let result = tree.query_selector("#outer span").unwrap();
|
||||
assert!(result.is_some());
|
||||
let node = tree.get_node(result.unwrap()).unwrap();
|
||||
assert_eq!(node.as_element().unwrap().local.as_ref(), "span");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_query_selector_attribute() {
|
||||
let tree = parse_html(
|
||||
r#"<input type="text" name="user"><input type="password" name="pass">"#,
|
||||
);
|
||||
let result = tree.query_selector(r#"input[type="password"]"#).unwrap();
|
||||
assert!(result.is_some());
|
||||
let node = tree.get_node(result.unwrap()).unwrap();
|
||||
assert_eq!(node.get_attribute("name"), Some("pass"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_query_selector_no_match() {
|
||||
let tree = parse_html("<div>Hello</div>");
|
||||
let result = tree.query_selector("span").unwrap();
|
||||
assert!(result.is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_query_selector_complex() {
|
||||
let tree = parse_html(
|
||||
r#"<div class="container">
|
||||
<ul class="list">
|
||||
<li class="item active">First</li>
|
||||
<li class="item">Second</li>
|
||||
<li class="item active">Third</li>
|
||||
</ul>
|
||||
</div>"#,
|
||||
);
|
||||
let results = tree.query_selector_all(".list .item.active").unwrap();
|
||||
assert_eq!(results.len(), 2);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_query_selector_all_from_scopes_to_subtree() {
|
||||
let tree = parse_html(
|
||||
r#"<div id="a"><span class="x">in a</span></div><div id="b"><span class="x">in b</span></div>"#,
|
||||
);
|
||||
let a = tree.get_element_by_id("a").expect("div#a");
|
||||
let b = tree.get_element_by_id("b").expect("div#b");
|
||||
|
||||
// Document-rooted: sees both spans.
|
||||
assert_eq!(tree.query_selector_all(".x").unwrap().len(), 2);
|
||||
// Scoped to #a: only its descendant span.
|
||||
let in_a = tree.query_selector_all_from(a, ".x").unwrap();
|
||||
assert_eq!(in_a.len(), 1);
|
||||
let in_b = tree.query_selector_all_from(b, ".x").unwrap();
|
||||
assert_eq!(in_b.len(), 1);
|
||||
assert_ne!(in_a[0], in_b[0]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_query_selector_from_returns_first_in_subtree_only() {
|
||||
let tree = parse_html(
|
||||
r#"<section id="s"><p>first</p><p>second</p></section><p>outside</p>"#,
|
||||
);
|
||||
let s = tree.get_element_by_id("s").expect("section#s");
|
||||
|
||||
// Scoped to #s: skip the outside paragraph; return the first inside.
|
||||
let first_in_s = tree.query_selector_from(s, "p").unwrap().expect("a p inside");
|
||||
assert_eq!(tree.text_content(first_in_s), "first");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_query_selector_from_excludes_self() {
|
||||
// The root element itself must not match its own scoped query, per
|
||||
// the spec: querySelector matches descendants only.
|
||||
let tree = parse_html(r#"<div id="root" class="x"><span>child</span></div>"#);
|
||||
let root = tree.get_element_by_id("root").expect("div#root");
|
||||
|
||||
// Only descendants are candidates: `.x` on root finds nothing.
|
||||
assert!(tree.query_selector_from(root, ".x").unwrap().is_none());
|
||||
// `span` finds the child.
|
||||
assert!(tree.query_selector_from(root, "span").unwrap().is_some());
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,236 @@
|
||||
use crate::tree::{DomTree, NodeData, NodeId};
|
||||
|
||||
// A unit of pending serialization work. Held on an explicit heap stack instead
|
||||
// of the call stack so a deeply nested tree cannot overflow the thread stack and
|
||||
// abort the process (a stack overflow is a hard abort that `op_dom`'s
|
||||
// catch_unwind cannot recover). `descendants()` is iterative + capped for the
|
||||
// same reason; the serializer must be too.
|
||||
enum SerializeWork {
|
||||
// Serialize this node; `include_self` mirrors the old recursive param.
|
||||
Node(NodeId, bool),
|
||||
// Emit a previously-opened element's closing tag, after its children.
|
||||
CloseTag(String),
|
||||
}
|
||||
|
||||
impl DomTree {
|
||||
pub fn outer_html(&self, node_id: NodeId) -> String {
|
||||
let mut buf = String::new();
|
||||
self.serialize_worklist(vec![SerializeWork::Node(node_id, true)], &mut buf);
|
||||
buf
|
||||
}
|
||||
|
||||
pub fn inner_html(&self, node_id: NodeId) -> String {
|
||||
let mut buf = String::new();
|
||||
self.serialize_worklist(self.child_work(node_id), &mut buf);
|
||||
buf
|
||||
}
|
||||
|
||||
// Children as work items in document order (top of the LIFO stack first).
|
||||
fn child_work(&self, node_id: NodeId) -> Vec<SerializeWork> {
|
||||
self.children(node_id)
|
||||
.into_iter()
|
||||
.rev()
|
||||
.map(|c| SerializeWork::Node(c, true))
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn serialize_worklist(&self, mut stack: Vec<SerializeWork>, buf: &mut String) {
|
||||
// Defense in depth, mirroring descendants(): a well-formed subtree emits
|
||||
// at most one Node plus one CloseTag per node, so 2*nodes.len() work
|
||||
// items bound a valid walk. Exceeding it means the graph is cyclic (the
|
||||
// append_child / insert_before guards prevent that); stop rather than
|
||||
// spin forever. On a valid tree this bound is never reached.
|
||||
let max_steps = self
|
||||
.node_slot_count()
|
||||
.saturating_mul(2)
|
||||
.saturating_add(16);
|
||||
let mut steps = 0usize;
|
||||
|
||||
while let Some(work) = stack.pop() {
|
||||
steps += 1;
|
||||
if steps > max_steps {
|
||||
eprintln!("obscura: serialize worklist cap hit - tree has a cycle");
|
||||
break;
|
||||
}
|
||||
|
||||
let (node_id, include_self) = match work {
|
||||
SerializeWork::CloseTag(tag) => {
|
||||
buf.push_str("</");
|
||||
buf.push_str(&tag);
|
||||
buf.push('>');
|
||||
continue;
|
||||
}
|
||||
SerializeWork::Node(node_id, include_self) => (node_id, include_self),
|
||||
};
|
||||
|
||||
let node = match self.get_node(node_id) {
|
||||
Some(n) => n,
|
||||
None => continue,
|
||||
};
|
||||
|
||||
match &node.data {
|
||||
NodeData::Document => {
|
||||
for w in self.child_work(node_id) {
|
||||
stack.push(w);
|
||||
}
|
||||
}
|
||||
NodeData::Doctype { name, .. } => {
|
||||
buf.push_str("<!DOCTYPE ");
|
||||
buf.push_str(name);
|
||||
buf.push('>');
|
||||
}
|
||||
NodeData::Element { name, attrs, .. } => {
|
||||
let tag = name.local.as_ref();
|
||||
if include_self {
|
||||
buf.push('<');
|
||||
buf.push_str(tag);
|
||||
for attr in attrs {
|
||||
buf.push(' ');
|
||||
let attr_name = attr.name.local.as_ref();
|
||||
buf.push_str(attr_name);
|
||||
buf.push_str("=\"");
|
||||
escape_attr(&attr.value, buf);
|
||||
buf.push('"');
|
||||
}
|
||||
buf.push('>');
|
||||
}
|
||||
|
||||
if !is_void_element(tag) {
|
||||
// Push the closing tag first so it pops after all the
|
||||
// children we push next.
|
||||
if include_self {
|
||||
stack.push(SerializeWork::CloseTag(tag.to_string()));
|
||||
}
|
||||
for w in self.child_work(node_id) {
|
||||
stack.push(w);
|
||||
}
|
||||
}
|
||||
}
|
||||
NodeData::Text { contents } => {
|
||||
let parent_is_raw = node.parent
|
||||
.and_then(|pid| {
|
||||
self.with_node(pid, |p| {
|
||||
p.as_element()
|
||||
.map(|name| is_raw_text_element(name.local.as_ref()))
|
||||
.unwrap_or(false)
|
||||
})
|
||||
})
|
||||
.unwrap_or(false);
|
||||
|
||||
if parent_is_raw {
|
||||
buf.push_str(contents);
|
||||
} else {
|
||||
escape_text(contents, buf);
|
||||
}
|
||||
}
|
||||
NodeData::Comment { contents } => {
|
||||
buf.push_str("<!--");
|
||||
// The HTML parser can never produce a comment whose data contains
|
||||
// "-->" (that ends the comment), but script can via
|
||||
// document.createComment("a-->b"). Emitting it verbatim would close
|
||||
// the comment early and let the trailing text escape into markup.
|
||||
// Neutralize the terminator so the serialized output round-trips as
|
||||
// a single comment.
|
||||
if contents.contains("-->") {
|
||||
buf.push_str(&contents.replace("-->", "-->"));
|
||||
} else {
|
||||
buf.push_str(contents);
|
||||
}
|
||||
buf.push_str("-->");
|
||||
}
|
||||
NodeData::ProcessingInstruction { target, data } => {
|
||||
buf.push_str("<?");
|
||||
buf.push_str(target);
|
||||
buf.push(' ');
|
||||
buf.push_str(data);
|
||||
buf.push('>');
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn escape_text(s: &str, buf: &mut String) {
|
||||
for c in s.chars() {
|
||||
match c {
|
||||
'&' => buf.push_str("&"),
|
||||
'<' => buf.push_str("<"),
|
||||
'>' => buf.push_str(">"),
|
||||
_ => buf.push(c),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn escape_attr(s: &str, buf: &mut String) {
|
||||
for c in s.chars() {
|
||||
match c {
|
||||
'&' => buf.push_str("&"),
|
||||
'"' => buf.push_str("""),
|
||||
_ => buf.push(c),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn is_void_element(tag: &str) -> bool {
|
||||
matches!(
|
||||
tag,
|
||||
"area" | "base" | "br" | "col" | "embed" | "hr" | "img" | "input" | "link" | "meta"
|
||||
| "param" | "source" | "track" | "wbr"
|
||||
)
|
||||
}
|
||||
|
||||
fn is_raw_text_element(tag: &str) -> bool {
|
||||
matches!(tag, "script" | "style" | "textarea" | "title")
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use crate::tree_sink::parse_html;
|
||||
|
||||
#[test]
|
||||
fn test_outer_html() {
|
||||
let tree = parse_html(r#"<div id="test"><p>Hello</p></div>"#);
|
||||
let div = tree.get_element_by_id("test").unwrap();
|
||||
let html = tree.outer_html(div);
|
||||
assert!(html.contains(r#"<div id="test">"#));
|
||||
assert!(html.contains("<p>Hello</p>"));
|
||||
assert!(html.contains("</div>"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_inner_html() {
|
||||
let tree = parse_html(r#"<div id="test"><p>Hello</p><p>World</p></div>"#);
|
||||
let div = tree.get_element_by_id("test").unwrap();
|
||||
let html = tree.inner_html(div);
|
||||
assert!(html.contains("<p>Hello</p>"));
|
||||
assert!(html.contains("<p>World</p>"));
|
||||
assert!(!html.contains("<div"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_serialize_attributes() {
|
||||
let tree = parse_html(r#"<a href="https://example.com" class="link">Click</a>"#);
|
||||
let a = tree.query_selector("a").unwrap().unwrap();
|
||||
let html = tree.outer_html(a);
|
||||
assert!(html.contains("href=\"https://example.com\""));
|
||||
assert!(html.contains("class=\"link\""));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_serialize_special_chars() {
|
||||
let tree = parse_html("<p>Hello & World <3</p>");
|
||||
let p = tree.query_selector("p").unwrap().unwrap();
|
||||
let html = tree.outer_html(p);
|
||||
assert!(html.contains("&"));
|
||||
assert!(html.contains("<"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_void_elements() {
|
||||
let tree = parse_html(r#"<img src="test.png"><br>"#);
|
||||
let img = tree.query_selector("img").unwrap().unwrap();
|
||||
let html = tree.outer_html(img);
|
||||
assert!(html.contains("<img"));
|
||||
assert!(!html.contains("</img>"));
|
||||
}
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,323 @@
|
||||
use std::borrow::Cow;
|
||||
use std::cell::Ref;
|
||||
use std::fmt;
|
||||
|
||||
use html5ever::tendril::StrTendril;
|
||||
use html5ever::tree_builder::{ElemName, ElementFlags, NodeOrText, QuirksMode, TreeSink};
|
||||
use html5ever::{Attribute as HtmlAttribute, LocalName, Namespace, QualName};
|
||||
|
||||
use crate::tree::{Attribute, DomTree, NodeData, NodeId};
|
||||
|
||||
pub struct ObscuraElemName<'a> {
|
||||
_ref: Ref<'a, ()>,
|
||||
name: *const QualName,
|
||||
}
|
||||
|
||||
impl<'a> fmt::Debug for ObscuraElemName<'a> {
|
||||
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
let name = unsafe { &*self.name };
|
||||
write!(f, "{:?}", name)
|
||||
}
|
||||
}
|
||||
|
||||
impl<'a> ElemName for ObscuraElemName<'a> {
|
||||
fn ns(&self) -> &Namespace {
|
||||
unsafe { &(*self.name).ns }
|
||||
}
|
||||
|
||||
fn local_name(&self) -> &LocalName {
|
||||
unsafe { &(*self.name).local }
|
||||
}
|
||||
}
|
||||
|
||||
impl TreeSink for DomTree {
|
||||
type Handle = NodeId;
|
||||
type Output = Self;
|
||||
type ElemName<'a> = ObscuraElemName<'a>;
|
||||
|
||||
fn finish(self) -> Self::Output {
|
||||
self
|
||||
}
|
||||
|
||||
fn parse_error(&self, _msg: Cow<'static, str>) {}
|
||||
|
||||
fn get_document(&self) -> NodeId {
|
||||
self.document()
|
||||
}
|
||||
|
||||
fn elem_name<'a>(&'a self, target: &'a NodeId) -> ObscuraElemName<'a> {
|
||||
let borrow = self.borrow_inner();
|
||||
let node = borrow.nodes.get(target.index())
|
||||
.and_then(|n| n.as_ref())
|
||||
.expect("elem_name called on invalid node");
|
||||
let name_ptr: *const QualName = match &node.data {
|
||||
NodeData::Element { name, .. } => name as *const QualName,
|
||||
_ => panic!("elem_name called on non-element"),
|
||||
};
|
||||
let ref_guard = Ref::map(borrow, |_| &());
|
||||
ObscuraElemName {
|
||||
_ref: ref_guard,
|
||||
name: name_ptr,
|
||||
}
|
||||
}
|
||||
|
||||
fn create_element(
|
||||
&self,
|
||||
name: QualName,
|
||||
attrs: Vec<HtmlAttribute>,
|
||||
flags: ElementFlags,
|
||||
) -> NodeId {
|
||||
let converted_attrs: Vec<Attribute> = attrs
|
||||
.into_iter()
|
||||
.map(|a| Attribute {
|
||||
name: a.name,
|
||||
value: a.value.to_string(),
|
||||
})
|
||||
.collect();
|
||||
|
||||
let id = self.new_node(NodeData::Element {
|
||||
name: name.clone(),
|
||||
attrs: converted_attrs,
|
||||
template_contents: None,
|
||||
mathml_annotation_xml_integration_point: flags.mathml_annotation_xml_integration_point,
|
||||
});
|
||||
|
||||
if flags.template {
|
||||
let template_doc = self.new_node(NodeData::Document);
|
||||
self.with_node_mut(id, |node| {
|
||||
if let NodeData::Element { template_contents, .. } = &mut node.data {
|
||||
*template_contents = Some(template_doc);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
id
|
||||
}
|
||||
|
||||
fn create_comment(&self, text: StrTendril) -> NodeId {
|
||||
self.new_node(NodeData::Comment {
|
||||
contents: text.to_string(),
|
||||
})
|
||||
}
|
||||
|
||||
fn create_pi(&self, target: StrTendril, data: StrTendril) -> NodeId {
|
||||
self.new_node(NodeData::ProcessingInstruction {
|
||||
target: target.to_string(),
|
||||
data: data.to_string(),
|
||||
})
|
||||
}
|
||||
|
||||
fn append(&self, parent: &NodeId, child: NodeOrText<NodeId>) {
|
||||
match child {
|
||||
NodeOrText::AppendNode(node_id) => {
|
||||
self.append_child(*parent, node_id);
|
||||
}
|
||||
NodeOrText::AppendText(text) => {
|
||||
self.append_text(*parent, &text);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn append_based_on_parent_node(
|
||||
&self,
|
||||
element: &NodeId,
|
||||
prev_element: &NodeId,
|
||||
child: NodeOrText<NodeId>,
|
||||
) {
|
||||
let has_parent = self.with_node(*element, |n| n.parent.is_some()).unwrap_or(false);
|
||||
if has_parent {
|
||||
self.append_before_sibling(element, child);
|
||||
} else {
|
||||
self.append(prev_element, child);
|
||||
}
|
||||
}
|
||||
|
||||
fn append_doctype_to_document(
|
||||
&self,
|
||||
name: StrTendril,
|
||||
public_id: StrTendril,
|
||||
system_id: StrTendril,
|
||||
) {
|
||||
let doctype = self.new_node(NodeData::Doctype {
|
||||
name: name.to_string(),
|
||||
public_id: public_id.to_string(),
|
||||
system_id: system_id.to_string(),
|
||||
});
|
||||
let doc = self.document();
|
||||
self.append_child(doc, doctype);
|
||||
}
|
||||
|
||||
fn add_attrs_if_missing(&self, target: &NodeId, attrs: Vec<HtmlAttribute>) {
|
||||
self.with_node_mut(*target, |node| {
|
||||
if let NodeData::Element { attrs: existing, .. } = &mut node.data {
|
||||
for attr in attrs {
|
||||
let dominated = existing.iter().any(|a| a.name == attr.name);
|
||||
if !dominated {
|
||||
existing.push(Attribute {
|
||||
name: attr.name,
|
||||
value: attr.value.to_string(),
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
fn remove_from_parent(&self, target: &NodeId) {
|
||||
self.detach(*target);
|
||||
}
|
||||
|
||||
fn reparent_children(&self, node: &NodeId, new_parent: &NodeId) {
|
||||
let children = self.children(*node);
|
||||
for child_id in children {
|
||||
self.append_child(*new_parent, child_id);
|
||||
}
|
||||
}
|
||||
|
||||
fn append_before_sibling(&self, sibling: &NodeId, child: NodeOrText<NodeId>) {
|
||||
match child {
|
||||
NodeOrText::AppendNode(node_id) => {
|
||||
self.insert_before(*sibling, node_id);
|
||||
}
|
||||
NodeOrText::AppendText(text) => {
|
||||
let prev_text_id = {
|
||||
let node = self.get_node(*sibling);
|
||||
node.and_then(|n| n.prev_sibling).and_then(|prev_id| {
|
||||
let prev = self.get_node(prev_id);
|
||||
prev.and_then(|p| if p.is_text() { Some(prev_id) } else { None })
|
||||
})
|
||||
};
|
||||
|
||||
if let Some(prev_text_id) = prev_text_id {
|
||||
self.with_node_mut(prev_text_id, |node| {
|
||||
if let NodeData::Text { contents } = &mut node.data {
|
||||
contents.push_str(&text);
|
||||
}
|
||||
});
|
||||
return;
|
||||
}
|
||||
|
||||
let text_id = self.new_node(NodeData::Text {
|
||||
contents: text.to_string(),
|
||||
});
|
||||
self.insert_before(*sibling, text_id);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn get_template_contents(&self, target: &NodeId) -> NodeId {
|
||||
self.with_node(*target, |n| match &n.data {
|
||||
NodeData::Element { template_contents, .. } => *template_contents,
|
||||
_ => None,
|
||||
})
|
||||
.flatten()
|
||||
.expect("get_template_contents called on non-template element")
|
||||
}
|
||||
|
||||
fn same_node(&self, x: &NodeId, y: &NodeId) -> bool {
|
||||
x == y
|
||||
}
|
||||
|
||||
fn set_quirks_mode(&self, _mode: QuirksMode) {
|
||||
}
|
||||
|
||||
fn is_mathml_annotation_xml_integration_point(&self, target: &NodeId) -> bool {
|
||||
self.with_node(*target, |n| match &n.data {
|
||||
NodeData::Element { mathml_annotation_xml_integration_point, .. } => {
|
||||
*mathml_annotation_xml_integration_point
|
||||
}
|
||||
_ => false,
|
||||
})
|
||||
.unwrap_or(false)
|
||||
}
|
||||
}
|
||||
|
||||
pub fn parse_html(html: &str) -> DomTree {
|
||||
use html5ever::tendril::TendrilSink;
|
||||
use html5ever::{parse_document, ParseOpts};
|
||||
|
||||
let tree = DomTree::new();
|
||||
parse_document(tree, ParseOpts::default())
|
||||
.from_utf8()
|
||||
.one(html.as_bytes())
|
||||
}
|
||||
|
||||
pub fn parse_fragment(html: &str) -> DomTree {
|
||||
use html5ever::tendril::TendrilSink;
|
||||
use html5ever::{parse_fragment, ParseOpts, QualName};
|
||||
|
||||
let context_name = QualName::new(None, ns!(html), local_name!("body"));
|
||||
let tree = DomTree::new();
|
||||
parse_fragment(tree, ParseOpts::default(), context_name, vec![])
|
||||
.from_utf8()
|
||||
.one(html.as_bytes())
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn test_parse_simple_html() {
|
||||
let tree = parse_html("<html><head></head><body><h1>Hello</h1></body></html>");
|
||||
assert!(tree.len() > 3);
|
||||
let text = tree.text_content(tree.document());
|
||||
assert!(text.contains("Hello"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_parse_with_attributes() {
|
||||
let tree = parse_html(r#"<div id="main" class="container">Text</div>"#);
|
||||
let main = tree.get_element_by_id("main");
|
||||
assert!(main.is_some());
|
||||
let node = tree.get_node(main.unwrap()).unwrap();
|
||||
assert_eq!(node.get_attribute("class"), Some("container"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_parse_nested_structure() {
|
||||
let tree = parse_html(
|
||||
r#"<html><body>
|
||||
<div id="outer">
|
||||
<p id="para">Hello <strong>World</strong></p>
|
||||
<ul>
|
||||
<li>Item 1</li>
|
||||
<li>Item 2</li>
|
||||
</ul>
|
||||
</div>
|
||||
</body></html>"#,
|
||||
);
|
||||
|
||||
let outer = tree.get_element_by_id("outer").unwrap();
|
||||
let text = tree.text_content(outer);
|
||||
assert!(text.contains("Hello"));
|
||||
assert!(text.contains("World"));
|
||||
assert!(text.contains("Item 1"));
|
||||
assert!(text.contains("Item 2"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_parse_malformed_html() {
|
||||
let tree = parse_html("<div><p>Unclosed paragraph<p>Another<div>Nested wrong</div>");
|
||||
assert!(tree.len() > 3);
|
||||
let text = tree.text_content(tree.document());
|
||||
assert!(text.contains("Unclosed paragraph"));
|
||||
assert!(text.contains("Another"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_parse_doctype() {
|
||||
let tree = parse_html("<!DOCTYPE html><html><body>Hello</body></html>");
|
||||
let first_child = tree.children(tree.document())[0];
|
||||
let node = tree.get_node(first_child).unwrap();
|
||||
assert!(matches!(node.data, NodeData::Doctype { .. }));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_parse_fragment() {
|
||||
let tree = parse_fragment("<p>Hello</p><p>World</p>");
|
||||
let text = tree.text_content(tree.document());
|
||||
assert!(text.contains("Hello"));
|
||||
assert!(text.contains("World"));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,43 @@
|
||||
[package]
|
||||
name = "obscura-js"
|
||||
version.workspace = true
|
||||
edition.workspace = true
|
||||
|
||||
[features]
|
||||
default = []
|
||||
# Builds obscura-net with its stealth (wreq) HTTP client and lets scripted
|
||||
# fetch()/XHR be routed through it so JS-level requests carry the same Chrome
|
||||
# TLS fingerprint and client hints as stealth navigation.
|
||||
stealth = ["obscura-net/stealth"]
|
||||
|
||||
[build-dependencies]
|
||||
deno_core = "0.350"
|
||||
|
||||
[dependencies]
|
||||
obscura-dom = { workspace = true }
|
||||
obscura-net = { workspace = true }
|
||||
deno_core = "0.350"
|
||||
tokio = { workspace = true }
|
||||
serde = { workspace = true }
|
||||
serde_json = { workspace = true }
|
||||
url = { workspace = true }
|
||||
tracing = { workspace = true }
|
||||
thiserror = { workspace = true }
|
||||
anyhow = { workspace = true }
|
||||
reqwest = { workspace = true }
|
||||
html5ever = { workspace = true }
|
||||
deno_error = "0.6"
|
||||
base64 = { workspace = true }
|
||||
sha1 = "0.10"
|
||||
sha2 = "0.10"
|
||||
# WebCrypto (crypto.subtle) primitives. RustCrypto, pure Rust, no CMake/OpenSSL,
|
||||
# and they unify on the digest 0.10 / cipher 0.4 stack already in the tree.
|
||||
hmac = "0.12"
|
||||
aes = "0.8"
|
||||
aes-gcm = "0.10"
|
||||
cbc = { version = "0.1", features = ["alloc"] }
|
||||
ctr = "0.9"
|
||||
pbkdf2 = { version = "0.12", default-features = false, features = ["hmac"] }
|
||||
hkdf = "0.12"
|
||||
# CSPRNG for getRandomValues / randomUUID (replaces Math.random).
|
||||
getrandom = "0.2"
|
||||
@@ -0,0 +1,38 @@
|
||||
use std::path::PathBuf;
|
||||
|
||||
fn main() {
|
||||
println!("cargo:rerun-if-changed=js/bootstrap.js");
|
||||
println!("cargo:rerun-if-changed=build.rs");
|
||||
|
||||
let out_dir = PathBuf::from(std::env::var("OUT_DIR").unwrap());
|
||||
let snapshot_path = out_dir.join("OBSCURA_SNAPSHOT.bin");
|
||||
|
||||
let bootstrap_js = include_str!("js/bootstrap.js");
|
||||
|
||||
let output = deno_core::snapshot::create_snapshot(
|
||||
deno_core::snapshot::CreateSnapshotOptions {
|
||||
cargo_manifest_dir: env!("CARGO_MANIFEST_DIR"),
|
||||
startup_snapshot: None,
|
||||
skip_op_registration: true,
|
||||
extensions: vec![],
|
||||
extension_transpiler: None,
|
||||
with_runtime_cb: Some(Box::new(move |runtime| {
|
||||
runtime
|
||||
.execute_script("<obscura:bootstrap>", bootstrap_js.to_string())
|
||||
.expect("bootstrap.js should not fail during snapshot creation");
|
||||
})),
|
||||
},
|
||||
None,
|
||||
)
|
||||
.expect("Failed to create V8 snapshot");
|
||||
|
||||
std::fs::write(&snapshot_path, &*output.output).expect("Failed to write snapshot");
|
||||
println!(
|
||||
"cargo:rustc-env=OBSCURA_SNAPSHOT_PATH={}",
|
||||
snapshot_path.display()
|
||||
);
|
||||
|
||||
for file in &output.files_loaded_during_snapshot {
|
||||
println!("cargo:rerun-if-changed={}", file.display());
|
||||
}
|
||||
}
|
||||
Vendored
+8147
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,117 @@
|
||||
//! Shared per-command V8 watchdog for the CDP server.
|
||||
//!
|
||||
//! The CDP dispatcher holds a process-wide V8 lock around every V8-touching
|
||||
//! command, so at most one command's isolate is ever executing at a time. That
|
||||
//! lets a single long-lived watchdog thread bound the current command with a
|
||||
//! deadline, instead of spawning+joining a thread per command (which adds
|
||||
//! ~240us per command on the hot dispatch path). `arm` and `disarm` are just a
|
||||
//! mutex + condvar notify, in the low microseconds.
|
||||
//!
|
||||
//! Correctness rests on the V8-lock serialization: between `arm` and `disarm`
|
||||
//! the caller holds the V8 lock, so no other command can arm, and the single
|
||||
//! slot is exclusively this command's.
|
||||
|
||||
use std::sync::atomic::{AtomicBool, Ordering};
|
||||
use std::sync::{Arc, Condvar, Mutex, OnceLock};
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
use crate::runtime::IsolateHandle;
|
||||
|
||||
struct Slot {
|
||||
deadline: Instant,
|
||||
handle: IsolateHandle,
|
||||
gen: u64,
|
||||
fired: Arc<AtomicBool>,
|
||||
}
|
||||
|
||||
struct Shared {
|
||||
// (current armed slot, monotonic generation counter)
|
||||
state: Mutex<(Option<Slot>, u64)>,
|
||||
cv: Condvar,
|
||||
}
|
||||
|
||||
static SHARED: OnceLock<Arc<Shared>> = OnceLock::new();
|
||||
|
||||
fn shared() -> &'static Arc<Shared> {
|
||||
SHARED.get_or_init(|| {
|
||||
let s = Arc::new(Shared {
|
||||
state: Mutex::new((None, 0)),
|
||||
cv: Condvar::new(),
|
||||
});
|
||||
let worker = s.clone();
|
||||
std::thread::Builder::new()
|
||||
.name("cdp-watchdog".into())
|
||||
.spawn(move || watchdog_loop(worker))
|
||||
.expect("spawn cdp watchdog");
|
||||
s
|
||||
})
|
||||
}
|
||||
|
||||
fn watchdog_loop(s: Arc<Shared>) {
|
||||
let mut guard = s.state.lock().unwrap();
|
||||
loop {
|
||||
let next_deadline = match &guard.0 {
|
||||
None => None,
|
||||
Some(slot) => {
|
||||
let now = Instant::now();
|
||||
if slot.deadline <= now {
|
||||
// Overran: terminate this isolate and clear the slot. The
|
||||
// dispatcher's disarm will observe `fired` and clear the V8
|
||||
// termination flag before the next command.
|
||||
slot.fired.store(true, Ordering::SeqCst);
|
||||
slot.handle.terminate_execution();
|
||||
guard.0 = None;
|
||||
None
|
||||
} else {
|
||||
Some(slot.deadline - now)
|
||||
}
|
||||
}
|
||||
};
|
||||
guard = match next_deadline {
|
||||
// No armed command: block until arm() notifies. The worker holds the
|
||||
// lock until it waits, so arm() (which needs the lock) cannot notify
|
||||
// into the void; no lost wakeup.
|
||||
None => s.cv.wait(guard).unwrap(),
|
||||
Some(dur) => s.cv.wait_timeout(guard, dur).unwrap().0,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
/// Handle to an armed command; pass to [`disarm`].
|
||||
pub struct Armed {
|
||||
gen: u64,
|
||||
fired: Arc<AtomicBool>,
|
||||
}
|
||||
|
||||
/// Arm the shared watchdog for the current command. If the isolate is still
|
||||
/// executing `budget` later, it is terminated. O(1), no thread spawn.
|
||||
pub fn arm(handle: IsolateHandle, budget: Duration) -> Armed {
|
||||
let s = shared();
|
||||
let mut guard = s.state.lock().unwrap();
|
||||
guard.1 += 1;
|
||||
let gen = guard.1;
|
||||
let fired = Arc::new(AtomicBool::new(false));
|
||||
guard.0 = Some(Slot {
|
||||
deadline: Instant::now() + budget,
|
||||
handle,
|
||||
gen,
|
||||
fired: fired.clone(),
|
||||
});
|
||||
s.cv.notify_one();
|
||||
Armed { gen, fired }
|
||||
}
|
||||
|
||||
/// Disarm the command's watchdog. Returns true if it had already fired
|
||||
/// (terminated the isolate), in which case the caller must clear the V8
|
||||
/// termination flag before the next command runs.
|
||||
pub fn disarm(armed: Armed) -> bool {
|
||||
let s = shared();
|
||||
let mut guard = s.state.lock().unwrap();
|
||||
// Only clear the slot if it is still ours (a newer command would have a
|
||||
// higher gen, though the V8 lock prevents that overlap in practice).
|
||||
if guard.0.as_ref().map(|sl| sl.gen) == Some(armed.gen) {
|
||||
guard.0 = None;
|
||||
s.cv.notify_one();
|
||||
}
|
||||
armed.fired.load(Ordering::SeqCst)
|
||||
}
|
||||
@@ -0,0 +1,13 @@
|
||||
#[macro_use]
|
||||
extern crate html5ever;
|
||||
|
||||
pub mod cdp_watchdog;
|
||||
pub mod module_loader;
|
||||
pub mod runtime;
|
||||
pub mod ops;
|
||||
pub mod v8_flags;
|
||||
pub mod v8_lock;
|
||||
pub mod markdown;
|
||||
|
||||
pub use markdown::HTML_TO_MARKDOWN_JS;
|
||||
pub use v8_flags::set_v8_flags;
|
||||
@@ -0,0 +1,71 @@
|
||||
//! Shared markdown extraction script used by the LP.getMarkdown CDP method
|
||||
//! and the CLI `--dump markdown` mode. Lives in obscura-browser so both
|
||||
//! obscura-cdp and obscura-cli can call it without depending on each other.
|
||||
|
||||
/// JS expression that walks `document.body` and returns a markdown string.
|
||||
/// Must be evaluated against a Page that has a fully-bootstrapped JS runtime.
|
||||
pub const HTML_TO_MARKDOWN_JS: &str = r#"
|
||||
(function() {
|
||||
function toMd(el, depth) {
|
||||
if (!el) return '';
|
||||
var out = '';
|
||||
if (el.nodeType === 3) return el.textContent || '';
|
||||
if (el.nodeType !== 1) return '';
|
||||
var tag = (el.tagName || '').toLowerCase();
|
||||
var children = '';
|
||||
var cn = el.childNodes || [];
|
||||
for (var i = 0; i < cn.length; i++) children += toMd(cn[i], depth);
|
||||
children = children.replace(/\n{3,}/g, '\n\n');
|
||||
switch(tag) {
|
||||
case 'h1': return '\n# ' + children.trim() + '\n\n';
|
||||
case 'h2': return '\n## ' + children.trim() + '\n\n';
|
||||
case 'h3': return '\n### ' + children.trim() + '\n\n';
|
||||
case 'h4': return '\n#### ' + children.trim() + '\n\n';
|
||||
case 'h5': return '\n##### ' + children.trim() + '\n\n';
|
||||
case 'h6': return '\n###### ' + children.trim() + '\n\n';
|
||||
case 'p': return '\n' + children.trim() + '\n\n';
|
||||
case 'br': return '\n';
|
||||
case 'hr': return '\n---\n\n';
|
||||
case 'strong': case 'b': return '**' + children + '**';
|
||||
case 'em': case 'i': return '*' + children + '*';
|
||||
case 'code': return '`' + children + '`';
|
||||
case 'pre': return '\n```\n' + children + '\n```\n\n';
|
||||
case 'blockquote': return '\n> ' + children.trim().replace(/\n/g, '\n> ') + '\n\n';
|
||||
case 'a':
|
||||
var href = el.getAttribute('href') || '';
|
||||
if (href && children.trim()) return '[' + children.trim() + '](' + href + ')';
|
||||
return children;
|
||||
case 'img':
|
||||
var src = el.getAttribute('src') || '';
|
||||
var alt = el.getAttribute('alt') || '';
|
||||
return '';
|
||||
case 'ul': case 'ol':
|
||||
return '\n' + children + '\n';
|
||||
case 'li':
|
||||
var parent = el.parentNode;
|
||||
var isOrdered = parent && parent.tagName && parent.tagName.toLowerCase() === 'ol';
|
||||
var bullet = isOrdered ? '1. ' : '- ';
|
||||
return bullet + children.trim() + '\n';
|
||||
case 'table': return '\n' + children + '\n';
|
||||
case 'thead': case 'tbody': case 'tfoot': return children;
|
||||
case 'tr':
|
||||
var cells = [];
|
||||
var tds = el.childNodes || [];
|
||||
for (var j = 0; j < tds.length; j++) {
|
||||
if (tds[j].nodeType === 1) cells.push(toMd(tds[j], depth).trim());
|
||||
}
|
||||
return '| ' + cells.join(' | ') + ' |\n';
|
||||
case 'th': case 'td': return children;
|
||||
case 'script': case 'style': case 'noscript': case 'link': case 'meta': return '';
|
||||
case 'div': case 'section': case 'article': case 'main': case 'aside': case 'nav': case 'header': case 'footer':
|
||||
return '\n' + children;
|
||||
case 'span': return children;
|
||||
default: return children;
|
||||
}
|
||||
}
|
||||
var body = document.body || document.documentElement;
|
||||
var md = toMd(body, 0);
|
||||
md = md.replace(/\n{3,}/g, '\n\n').trim();
|
||||
return md;
|
||||
})()
|
||||
"#;
|
||||
@@ -0,0 +1,115 @@
|
||||
use std::pin::Pin;
|
||||
|
||||
use deno_core::error::ModuleLoaderError;
|
||||
use deno_core::ModuleLoadResponse;
|
||||
use deno_core::ModuleLoader;
|
||||
use deno_core::ModuleSource;
|
||||
use deno_core::ModuleSourceCode;
|
||||
use deno_core::ModuleSpecifier;
|
||||
use deno_core::RequestedModuleType;
|
||||
|
||||
pub struct ObscuraModuleLoader {
|
||||
pub base_url: String,
|
||||
/// Proxy URL threaded through to every dynamic ES-module fetch (#139).
|
||||
/// `None` keeps the pre-#139 direct-connection behaviour for callers
|
||||
/// that haven't been updated.
|
||||
pub proxy_url: Option<String>,
|
||||
}
|
||||
|
||||
impl ObscuraModuleLoader {
|
||||
pub fn new(base_url: &str) -> Self {
|
||||
Self::with_proxy(base_url, None)
|
||||
}
|
||||
|
||||
pub fn with_proxy(base_url: &str, proxy_url: Option<String>) -> Self {
|
||||
ObscuraModuleLoader {
|
||||
base_url: base_url.to_string(),
|
||||
proxy_url,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn io_err(msg: String) -> ModuleLoaderError {
|
||||
std::io::Error::new(std::io::ErrorKind::Other, msg).into()
|
||||
}
|
||||
|
||||
impl ModuleLoader for ObscuraModuleLoader {
|
||||
fn resolve(
|
||||
&self,
|
||||
specifier: &str,
|
||||
referrer: &str,
|
||||
_kind: deno_core::ResolutionKind,
|
||||
) -> Result<ModuleSpecifier, ModuleLoaderError> {
|
||||
let base = if referrer.is_empty()
|
||||
|| referrer.starts_with('<')
|
||||
|| referrer == "."
|
||||
|| referrer == "about:blank"
|
||||
{
|
||||
&self.base_url
|
||||
} else {
|
||||
referrer
|
||||
};
|
||||
|
||||
deno_core::resolve_import(specifier, base).map_err(|e| e.into())
|
||||
}
|
||||
|
||||
fn load(
|
||||
&self,
|
||||
module_specifier: &ModuleSpecifier,
|
||||
_maybe_referrer: Option<&ModuleSpecifier>,
|
||||
_is_dyn_import: bool,
|
||||
_requested_module_type: RequestedModuleType,
|
||||
) -> ModuleLoadResponse {
|
||||
let url = module_specifier.to_string();
|
||||
// Capture the loader's proxy here so the async closure below owns a
|
||||
// plain Option<String> rather than borrowing &self across an `await`.
|
||||
let proxy_url = self.proxy_url.clone();
|
||||
|
||||
ModuleLoadResponse::Async(Pin::from(Box::new(async move {
|
||||
// Reuse the process-wide cached client (same one op_fetch_url
|
||||
// uses). Modern SPAs dynamic-import 20-50 chunks per page; the
|
||||
// old code built a fresh reqwest::Client per import, each with
|
||||
// its own empty connection pool, no reuse, fresh TLS init for
|
||||
// every chunk. The cache means the first import on a given
|
||||
// proxy pays the build cost once and every chunk after reuses
|
||||
// the same warm pool.
|
||||
let client = crate::ops::cached_request_client(proxy_url.as_deref())
|
||||
.map_err(io_err)?;
|
||||
|
||||
tracing::debug!(
|
||||
"Loading ES module: {} (proxy: {})",
|
||||
url,
|
||||
proxy_url.as_deref().unwrap_or("direct")
|
||||
);
|
||||
|
||||
let resp = client
|
||||
.get(&url)
|
||||
.header("Accept", "application/javascript, text/javascript, */*")
|
||||
.send()
|
||||
.await
|
||||
.map_err(|e| io_err(format!("Failed to fetch module {}: {}", url, e)))?;
|
||||
|
||||
if !resp.status().is_success() {
|
||||
return Err(io_err(format!(
|
||||
"Module {} returned HTTP {}",
|
||||
url,
|
||||
resp.status()
|
||||
)));
|
||||
}
|
||||
|
||||
let code = resp.text().await.map_err(|e| {
|
||||
io_err(format!("Failed to read module body {}: {}", url, e))
|
||||
})?;
|
||||
|
||||
let specifier = ModuleSpecifier::parse(&url)
|
||||
.map_err(|e| io_err(format!("Invalid module URL {}: {}", url, e)))?;
|
||||
|
||||
Ok(ModuleSource::new(
|
||||
deno_core::ModuleType::JavaScript,
|
||||
ModuleSourceCode::String(code.into()),
|
||||
&specifier,
|
||||
None,
|
||||
))
|
||||
})))
|
||||
}
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,37 @@
|
||||
use std::sync::Once;
|
||||
|
||||
static INIT: Once = Once::new();
|
||||
|
||||
/// Apply user-supplied V8 flags exactly once, before the first isolate is
|
||||
/// created.
|
||||
///
|
||||
/// `flags` is a raw V8 flag string in the same form V8/Chromium/Node accept
|
||||
/// (e.g. `"--max-old-space-size=4096 --max-semi-space-size=64"`). An empty or
|
||||
/// whitespace-only string is a no-op and does not consume the one-shot guard,
|
||||
/// so a later non-empty call still takes effect.
|
||||
///
|
||||
/// V8 ignores `set_flags_from_string` once the platform is initialized, so the
|
||||
/// first non-empty call must run before any `JsRuntime` is constructed.
|
||||
/// Subsequent calls are silently dropped.
|
||||
pub fn set_v8_flags(flags: &str) {
|
||||
let trimmed = flags.trim();
|
||||
if trimmed.is_empty() {
|
||||
return;
|
||||
}
|
||||
INIT.call_once(|| {
|
||||
deno_core::v8::V8::set_flags_from_string(trimmed);
|
||||
});
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn empty_is_noop() {
|
||||
// Must not panic and must not consume the Once guard.
|
||||
set_v8_flags("");
|
||||
set_v8_flags(" ");
|
||||
set_v8_flags("\t\n");
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,27 @@
|
||||
//! Process-wide async lock that serializes V8 work across all `JsRuntime`s.
|
||||
//!
|
||||
//! V8's invariant: on any given OS thread, only one `Isolate` may be entered
|
||||
//! at a time, and HandleScope/ContextScope stacks must unwind on the Isolate
|
||||
//! they belong to. Obscura's CDP server runs every `JsRuntime` (one per Page)
|
||||
//! on a single OS thread via `tokio::task::LocalSet` + `spawn_local`. As soon
|
||||
//! as two pages' V8-touching futures interleave across an `.await`, V8 trips
|
||||
//! its `heap->isolate() == Isolate::TryGetCurrent()` check and aborts the
|
||||
//! whole process (no Rust panic; `V8_Fatal` calls `abort(3)`).
|
||||
//!
|
||||
//! Acquiring this lock around any block that calls `JsRuntime::execute_script`
|
||||
//! or `JsRuntime::run_event_loop` keeps that block contiguous on the thread:
|
||||
//! V8 fully exits the prior Isolate before the next page is allowed in. This
|
||||
//! converts the abort into latency. It is the issue-19 "Option 1" fix.
|
||||
//!
|
||||
//! The properly concurrent fix is to pin each `JsRuntime` to its own OS
|
||||
//! thread (issue-19 "Option 2"); that's a larger refactor tracked separately.
|
||||
|
||||
use std::sync::OnceLock;
|
||||
use tokio::sync::Mutex;
|
||||
|
||||
static V8_LOCK: OnceLock<Mutex<()>> = OnceLock::new();
|
||||
|
||||
/// Returns the process-wide V8 serialization lock.
|
||||
pub fn global() -> &'static Mutex<()> {
|
||||
V8_LOCK.get_or_init(|| Mutex::new(()))
|
||||
}
|
||||
@@ -0,0 +1,19 @@
|
||||
[package]
|
||||
name = "obscura-mcp"
|
||||
version.workspace = true
|
||||
edition.workspace = true
|
||||
|
||||
[features]
|
||||
default = []
|
||||
stealth = ["obscura-browser/stealth", "obscura-net/stealth"]
|
||||
|
||||
[dependencies]
|
||||
obscura-browser = { workspace = true }
|
||||
obscura-dom = { workspace = true }
|
||||
obscura-net = { workspace = true }
|
||||
tokio = { workspace = true }
|
||||
url = { workspace = true }
|
||||
serde = { workspace = true }
|
||||
serde_json = { workspace = true }
|
||||
anyhow = { workspace = true }
|
||||
tracing = { workspace = true }
|
||||
@@ -0,0 +1,323 @@
|
||||
use anyhow::Result;
|
||||
use serde_json::{json, Value};
|
||||
use tokio::io::{AsyncBufReadExt, AsyncReadExt, AsyncWriteExt, BufReader};
|
||||
use tokio::net::TcpListener;
|
||||
|
||||
use crate::{dispatch, BrowserState};
|
||||
|
||||
/// Hard cap on a single MCP request body. The client-supplied `Content-Length`
|
||||
/// is used to pre-size the read buffer; without a ceiling a request advertising
|
||||
/// e.g. `Content-Length: 4294967296` makes the server allocate and zero-fill
|
||||
/// that many bytes before reading any body — an unauthenticated OOM/DoS. 16 MiB
|
||||
/// is far above any real JSON-RPC tool call.
|
||||
const MAX_BODY_BYTES: usize = 16 * 1024 * 1024;
|
||||
|
||||
/// Origin allowlist for browser callers, read from `OBSCURA_MCP_ALLOWED_ORIGINS`
|
||||
/// (comma-separated). Unset/empty → permissive (unchanged `*`) so hosted
|
||||
/// dashboards keep working (issue #175).
|
||||
fn allowed_origins_env() -> Option<String> {
|
||||
std::env::var("OBSCURA_MCP_ALLOWED_ORIGINS")
|
||||
.ok()
|
||||
.filter(|s| !s.trim().is_empty())
|
||||
}
|
||||
|
||||
/// Whether a request's `Origin` is permitted. A request with no `Origin`
|
||||
/// (native, non-browser MCP clients) is always allowed — the same-origin
|
||||
/// policy only constrains browser callers. When an allowlist is configured, a
|
||||
/// browser `Origin` must match one of its entries (case-insensitive); this
|
||||
/// stops a malicious local web page from driving the loopback MCP port.
|
||||
fn origin_allowed(origin: Option<&str>, allowlist: Option<&str>) -> bool {
|
||||
match allowlist {
|
||||
None => true,
|
||||
Some(list) => match origin {
|
||||
None => true,
|
||||
Some(o) => {
|
||||
let o = o.trim();
|
||||
list.split(',')
|
||||
.map(str::trim)
|
||||
.any(|a| !a.is_empty() && a.eq_ignore_ascii_case(o))
|
||||
}
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
/// CORS `Access-Control-Allow-Origin` value for a response. With no allowlist
|
||||
/// we keep the permissive `*` (issue #175). With an allowlist the request's
|
||||
/// origin has already passed `origin_allowed`, so echo it back plus `Vary:
|
||||
/// Origin` instead of advertising `*`; a native client with no `Origin` needs
|
||||
/// no CORS header at all.
|
||||
fn cors_header(origin: Option<&str>, allowlist: Option<&str>) -> String {
|
||||
match allowlist {
|
||||
None => "Access-Control-Allow-Origin: *\r\n".to_string(),
|
||||
Some(_) => match origin {
|
||||
Some(o) => format!("Access-Control-Allow-Origin: {o}\r\nVary: Origin\r\n"),
|
||||
None => String::new(),
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
/// MCP Streamable HTTP transport (POST /mcp → JSON response).
|
||||
///
|
||||
/// Connections are handled sequentially on the current thread — the browser
|
||||
/// session (including the V8 runtime) is single-threaded and `!Send`, so we
|
||||
/// never need to move state across threads.
|
||||
pub async fn run(host: String, port: u16, proxy: Option<String>, user_agent: Option<String>, stealth: bool) -> Result<()> {
|
||||
let addr: std::net::SocketAddr = format!("{}:{}", host, port).parse()?;
|
||||
let listener = TcpListener::bind(&addr).await?;
|
||||
tracing::info!("MCP HTTP server on http://{}:{}/mcp", host, port);
|
||||
|
||||
let mut state = BrowserState::new(proxy, user_agent, stealth);
|
||||
let allowed_origins = allowed_origins_env();
|
||||
|
||||
loop {
|
||||
let (stream, peer) = listener.accept().await?;
|
||||
tracing::debug!("MCP HTTP connection from {}", peer);
|
||||
if let Err(e) = handle_connection(stream, &mut state, allowed_origins.as_deref()).await {
|
||||
tracing::debug!("connection closed: {}", e);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async fn handle_connection(
|
||||
stream: tokio::net::TcpStream,
|
||||
state: &mut BrowserState,
|
||||
allowed_origins: Option<&str>,
|
||||
) -> Result<()> {
|
||||
let (reader, mut writer) = stream.into_split();
|
||||
let mut reader = BufReader::new(reader);
|
||||
|
||||
loop {
|
||||
// ── request line ─────────────────────────────────────────────────────
|
||||
let mut request_line = String::new();
|
||||
if reader.read_line(&mut request_line).await? == 0 {
|
||||
break;
|
||||
}
|
||||
let request_line = request_line.trim().to_string();
|
||||
if request_line.is_empty() {
|
||||
break;
|
||||
}
|
||||
|
||||
let parts: Vec<&str> = request_line.splitn(3, ' ').collect();
|
||||
if parts.len() < 3 {
|
||||
break;
|
||||
}
|
||||
let method = parts[0];
|
||||
let path = parts[1];
|
||||
|
||||
// ── headers ──────────────────────────────────────────────────────────
|
||||
let mut content_length: Option<usize> = None;
|
||||
let mut accept_sse = false;
|
||||
let mut keep_alive = false;
|
||||
let mut origin: Option<String> = None;
|
||||
|
||||
loop {
|
||||
let mut line = String::new();
|
||||
reader.read_line(&mut line).await?;
|
||||
let trimmed = line.trim_end_matches("\r\n").trim_end_matches('\n');
|
||||
if trimmed.is_empty() {
|
||||
break;
|
||||
}
|
||||
let lower = trimmed.to_lowercase();
|
||||
if let Some(v) = lower.strip_prefix("content-length: ") {
|
||||
content_length = v.trim().parse().ok();
|
||||
}
|
||||
if lower.starts_with("origin:") {
|
||||
if let Some(idx) = trimmed.find(':') {
|
||||
origin = Some(trimmed[idx + 1..].trim().to_string());
|
||||
}
|
||||
}
|
||||
if lower.contains("text/event-stream") {
|
||||
accept_sse = true;
|
||||
}
|
||||
if lower.starts_with("connection: ") && lower.contains("keep-alive") {
|
||||
keep_alive = true;
|
||||
}
|
||||
}
|
||||
|
||||
// ── routing ──────────────────────────────────────────────────────────
|
||||
if path != "/mcp" {
|
||||
respond(&mut writer, 404, b"{\"error\":\"not found\"}").await?;
|
||||
break;
|
||||
}
|
||||
|
||||
// Origin gate: when OBSCURA_MCP_ALLOWED_ORIGINS is configured, a browser
|
||||
// request from a non-listed origin is refused before it can drive the
|
||||
// browser session (mitigates a malicious local web page issuing
|
||||
// cross-origin POSTs to the loopback MCP port). The permissive default
|
||||
// and no-Origin native clients are unaffected.
|
||||
if !origin_allowed(origin.as_deref(), allowed_origins) {
|
||||
respond(&mut writer, 403, b"{\"error\":\"origin not allowed\"}").await?;
|
||||
break;
|
||||
}
|
||||
let cors = cors_header(origin.as_deref(), allowed_origins);
|
||||
|
||||
match method {
|
||||
"OPTIONS" => {
|
||||
// mcp-protocol-version is part of the MCP spec, Authorization /
|
||||
// X-API-Key are common for hosted deployments. Without these
|
||||
// listed the browser preflight check fails and blocks the actual
|
||||
// request.
|
||||
let hdr = format!(
|
||||
"HTTP/1.1 204 No Content\r\n\
|
||||
{cors}\
|
||||
Access-Control-Allow-Methods: GET, POST, OPTIONS\r\n\
|
||||
Access-Control-Allow-Headers: Content-Type, Authorization, X-API-Key, mcp-protocol-version\r\n\
|
||||
Access-Control-Max-Age: 86400\r\n\
|
||||
\r\n"
|
||||
);
|
||||
writer.write_all(hdr.as_bytes()).await?;
|
||||
}
|
||||
|
||||
"GET" if accept_sse => {
|
||||
// SSE stream: hold open and send periodic keep-alive comments
|
||||
let hdr = format!(
|
||||
"HTTP/1.1 200 OK\r\n\
|
||||
Content-Type: text/event-stream\r\n\
|
||||
Cache-Control: no-cache\r\n\
|
||||
Connection: keep-alive\r\n\
|
||||
{cors}\
|
||||
\r\n"
|
||||
);
|
||||
writer.write_all(hdr.as_bytes()).await?;
|
||||
loop {
|
||||
tokio::time::sleep(tokio::time::Duration::from_secs(15)).await;
|
||||
if writer.write_all(b": ping\n\n").await.is_err() {
|
||||
break;
|
||||
}
|
||||
let _ = writer.flush().await;
|
||||
}
|
||||
break;
|
||||
}
|
||||
|
||||
"POST" => {
|
||||
let len = match content_length {
|
||||
Some(n) => n,
|
||||
None => {
|
||||
respond(&mut writer, 400, b"{\"error\":\"missing Content-Length\"}").await?;
|
||||
break;
|
||||
}
|
||||
};
|
||||
|
||||
// Reject oversized bodies BEFORE allocating: `vec![0u8; len]`
|
||||
// commits `len` bytes up front, so an attacker-chosen
|
||||
// Content-Length would otherwise OOM the process (DoS).
|
||||
if len > MAX_BODY_BYTES {
|
||||
respond(&mut writer, 413, b"{\"error\":\"payload too large\"}").await?;
|
||||
break;
|
||||
}
|
||||
|
||||
let mut body = vec![0u8; len];
|
||||
reader.read_exact(&mut body).await?;
|
||||
|
||||
let response = process_body(&body, state).await;
|
||||
let bytes = serde_json::to_vec(&response)?;
|
||||
respond_json(&mut writer, &bytes, &cors).await?;
|
||||
|
||||
if !keep_alive {
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
_ => {
|
||||
respond(&mut writer, 405, b"{\"error\":\"method not allowed\"}").await?;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn process_body(body: &[u8], state: &mut BrowserState) -> Value {
|
||||
let msg: Value = match serde_json::from_slice(body) {
|
||||
Ok(v) => v,
|
||||
Err(_) => return json!({"jsonrpc":"2.0","id":null,"error":{"code":-32700,"message":"Parse error"}}),
|
||||
};
|
||||
|
||||
if let Some(batch) = msg.as_array() {
|
||||
let mut results = Vec::new();
|
||||
for item in batch {
|
||||
if let Some(r) = process_one(item, state).await {
|
||||
results.push(r);
|
||||
}
|
||||
}
|
||||
return Value::Array(results);
|
||||
}
|
||||
|
||||
process_one(&msg, state).await
|
||||
.unwrap_or_else(|| json!({"jsonrpc":"2.0","id":null,"error":{"code":-32600,"message":"Invalid Request"}}))
|
||||
}
|
||||
|
||||
async fn process_one(msg: &Value, state: &mut BrowserState) -> Option<Value> {
|
||||
let id = msg.get("id").cloned()?; // notifications have no id — return None
|
||||
let method = msg.get("method").and_then(Value::as_str).unwrap_or("");
|
||||
let params = msg.get("params").unwrap_or(&Value::Null);
|
||||
let resp = dispatch(method, id, params, state).await;
|
||||
Some(serde_json::to_value(resp).unwrap())
|
||||
}
|
||||
|
||||
async fn respond_json(writer: &mut (impl AsyncWriteExt + Unpin), body: &[u8], cors: &str) -> Result<()> {
|
||||
let hdr = format!(
|
||||
"HTTP/1.1 200 OK\r\n\
|
||||
Content-Type: application/json\r\n\
|
||||
Content-Length: {}\r\n\
|
||||
{cors}\
|
||||
Connection: keep-alive\r\n\
|
||||
\r\n",
|
||||
body.len()
|
||||
);
|
||||
writer.write_all(hdr.as_bytes()).await?;
|
||||
writer.write_all(body).await?;
|
||||
writer.flush().await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn respond(writer: &mut (impl AsyncWriteExt + Unpin), status: u16, body: &[u8]) -> Result<()> {
|
||||
let status_text = match status {
|
||||
400 => "Bad Request",
|
||||
403 => "Forbidden",
|
||||
404 => "Not Found",
|
||||
405 => "Method Not Allowed",
|
||||
413 => "Payload Too Large",
|
||||
_ => "OK",
|
||||
};
|
||||
let hdr = format!(
|
||||
"HTTP/1.1 {status} {status_text}\r\n\
|
||||
Content-Type: application/json\r\n\
|
||||
Content-Length: {}\r\n\
|
||||
\r\n",
|
||||
body.len()
|
||||
);
|
||||
writer.write_all(hdr.as_bytes()).await?;
|
||||
writer.write_all(body).await?;
|
||||
writer.flush().await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod mcp_hardening_tests {
|
||||
use super::{origin_allowed, MAX_BODY_BYTES};
|
||||
|
||||
#[test]
|
||||
fn no_allowlist_is_permissive() {
|
||||
assert!(origin_allowed(Some("https://evil.example"), None));
|
||||
assert!(origin_allowed(None, None));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn allowlist_matches_case_insensitively_and_rejects_others() {
|
||||
let list = Some("http://localhost:3000, https://app.example.com");
|
||||
assert!(origin_allowed(Some("http://localhost:3000"), list));
|
||||
assert!(origin_allowed(Some("https://APP.example.com"), list));
|
||||
assert!(!origin_allowed(Some("https://evil.example"), list));
|
||||
// A native client (no Origin header) is always allowed.
|
||||
assert!(origin_allowed(None, list));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn body_cap_is_sane() {
|
||||
// Far above a real JSON-RPC tool call, far below an OOM-inducing value.
|
||||
assert!(MAX_BODY_BYTES >= 1 << 20);
|
||||
assert!(MAX_BODY_BYTES <= 64 << 20);
|
||||
}
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,137 @@
|
||||
//! Regression test for issue #175: the MCP HTTP server's OPTIONS preflight
|
||||
//! response must list every header a browser MCP client may send, including
|
||||
//! `mcp-protocol-version` (from the MCP spec) and `Authorization` /
|
||||
//! `X-API-Key` (common in hosted deployments). Otherwise the browser blocks
|
||||
//! the actual request with a CORS error.
|
||||
|
||||
use std::net::TcpListener as StdListener;
|
||||
use std::time::Duration;
|
||||
use tokio::io::{AsyncReadExt, AsyncWriteExt};
|
||||
use tokio::net::TcpStream;
|
||||
use tokio::task::LocalSet;
|
||||
use tokio::time::{sleep, timeout};
|
||||
|
||||
fn pick_free_port() -> u16 {
|
||||
let l = StdListener::bind("127.0.0.1:0").unwrap();
|
||||
let p = l.local_addr().unwrap().port();
|
||||
drop(l);
|
||||
p
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "current_thread")]
|
||||
async fn options_preflight_lists_required_browser_headers() {
|
||||
let port = pick_free_port();
|
||||
let local = LocalSet::new();
|
||||
|
||||
// Spawn the MCP HTTP server. It loops forever; we abort the task at the
|
||||
// end of the test. `current_thread` + LocalSet is required because the
|
||||
// browser state is `!Send` (Page holds V8 handles).
|
||||
let server = local.spawn_local(async move {
|
||||
let _ = obscura_mcp::http::run("127.0.0.1".to_string(), port, None, None, false).await;
|
||||
});
|
||||
|
||||
local.run_until(async {
|
||||
// Wait for the listener to bind.
|
||||
for _ in 0..40 {
|
||||
if TcpStream::connect(("127.0.0.1", port)).await.is_ok() {
|
||||
break;
|
||||
}
|
||||
sleep(Duration::from_millis(50)).await;
|
||||
}
|
||||
|
||||
let mut stream = TcpStream::connect(("127.0.0.1", port))
|
||||
.await
|
||||
.expect("MCP server did not come up");
|
||||
let req = b"OPTIONS /mcp HTTP/1.1\r\n\
|
||||
Host: 127.0.0.1\r\n\
|
||||
Origin: https://dashboard.example.com\r\n\
|
||||
Access-Control-Request-Method: POST\r\n\
|
||||
Access-Control-Request-Headers: Content-Type, mcp-protocol-version, Authorization\r\n\
|
||||
\r\n";
|
||||
stream.write_all(req).await.unwrap();
|
||||
stream.flush().await.unwrap();
|
||||
|
||||
let mut buf = [0u8; 4096];
|
||||
let n = timeout(Duration::from_secs(2), stream.read(&mut buf))
|
||||
.await
|
||||
.expect("read timed out")
|
||||
.expect("read failed");
|
||||
let response = String::from_utf8_lossy(&buf[..n]).to_string();
|
||||
|
||||
server.abort();
|
||||
|
||||
assert!(
|
||||
response.starts_with("HTTP/1.1 204"),
|
||||
"expected 204 No Content preflight, got:\n{response}"
|
||||
);
|
||||
let lc = response.to_lowercase();
|
||||
assert!(
|
||||
lc.contains("access-control-allow-headers:"),
|
||||
"preflight must include Access-Control-Allow-Headers; got:\n{response}"
|
||||
);
|
||||
assert!(
|
||||
lc.contains("mcp-protocol-version"),
|
||||
"ACAH must list mcp-protocol-version (per MCP spec); got:\n{response}"
|
||||
);
|
||||
assert!(
|
||||
lc.contains("authorization"),
|
||||
"ACAH must list Authorization for hosted deployments; got:\n{response}"
|
||||
);
|
||||
assert!(
|
||||
lc.contains("x-api-key"),
|
||||
"ACAH must list X-API-Key for hosted deployments; got:\n{response}"
|
||||
);
|
||||
})
|
||||
.await;
|
||||
}
|
||||
|
||||
/// Regression test for the unbounded `Content-Length` allocation: a POST that
|
||||
/// advertises a huge body must be rejected with 413 *before* the server tries
|
||||
/// to allocate `vec![0u8; len]`, rather than committing gigabytes of RAM and
|
||||
/// OOM-ing the process (unauthenticated DoS).
|
||||
#[tokio::test(flavor = "current_thread")]
|
||||
async fn oversized_content_length_is_rejected() {
|
||||
let port = pick_free_port();
|
||||
let local = LocalSet::new();
|
||||
|
||||
let server = local.spawn_local(async move {
|
||||
let _ = obscura_mcp::http::run("127.0.0.1".to_string(), port, None, None, false).await;
|
||||
});
|
||||
|
||||
local
|
||||
.run_until(async {
|
||||
for _ in 0..40 {
|
||||
if TcpStream::connect(("127.0.0.1", port)).await.is_ok() {
|
||||
break;
|
||||
}
|
||||
sleep(Duration::from_millis(50)).await;
|
||||
}
|
||||
|
||||
let mut stream = TcpStream::connect(("127.0.0.1", port))
|
||||
.await
|
||||
.expect("MCP server did not come up");
|
||||
// 8 GiB advertised, zero body sent. Pre-fix the server allocates 8 GiB.
|
||||
let req = b"POST /mcp HTTP/1.1\r\n\
|
||||
Host: 127.0.0.1\r\n\
|
||||
Content-Type: application/json\r\n\
|
||||
Content-Length: 8589934592\r\n\
|
||||
\r\n";
|
||||
stream.write_all(req).await.unwrap();
|
||||
stream.flush().await.unwrap();
|
||||
|
||||
let mut buf = [0u8; 1024];
|
||||
let n = timeout(Duration::from_secs(2), stream.read(&mut buf))
|
||||
.await
|
||||
.expect("read timed out")
|
||||
.expect("read failed");
|
||||
let response = String::from_utf8_lossy(&buf[..n]).to_string();
|
||||
|
||||
server.abort();
|
||||
|
||||
assert!(
|
||||
response.starts_with("HTTP/1.1 413"),
|
||||
"expected 413 Payload Too Large, got:\n{response}"
|
||||
);
|
||||
})
|
||||
.await;
|
||||
}
|
||||
@@ -0,0 +1,36 @@
|
||||
[package]
|
||||
name = "obscura-net"
|
||||
version.workspace = true
|
||||
edition.workspace = true
|
||||
|
||||
[features]
|
||||
default = []
|
||||
stealth = ["wreq", "wreq-util"]
|
||||
|
||||
[dependencies]
|
||||
reqwest = { workspace = true }
|
||||
tokio = { workspace = true }
|
||||
url = { workspace = true }
|
||||
serde = { workspace = true }
|
||||
serde_json = { workspace = true }
|
||||
tracing = { workspace = true }
|
||||
thiserror = { workspace = true }
|
||||
async-trait = "0.1"
|
||||
encoding_rs = "0.8"
|
||||
tempfile = "3"
|
||||
# wreq and wreq-util are pre-release crates whose API changes between rc builds
|
||||
# (CertStore moved out of `tls`, `cert_store` renamed to `tls_cert_store`, the
|
||||
# Emulation builder reshuffled). A caret requirement let a fresh resolve pull a
|
||||
# newer, incompatible rc and broke `cargo build --features stealth` (issue #234).
|
||||
# Pin exact versions so the build always resolves to the API wreq_client.rs targets.
|
||||
wreq-util = { version = "=3.0.0-rc.12", optional = true }
|
||||
|
||||
# `prefix-symbols` renames BoringSSL exports to avoid clashes with system OpenSSL,
|
||||
# but the renaming is only correct on Linux and Android (per wreq's README).
|
||||
# Enabling it on other platforms produces unresolved `build_script_main_*`
|
||||
# symbols at link time (see issue #39).
|
||||
[target.'cfg(any(target_os = "linux", target_os = "android"))'.dependencies]
|
||||
wreq = { version = "=6.0.0-rc.29", optional = true, features = ["prefix-symbols", "socks"] }
|
||||
|
||||
[target.'cfg(not(any(target_os = "linux", target_os = "android")))'.dependencies]
|
||||
wreq = { version = "=6.0.0-rc.29", optional = true, features = ["socks"] }
|
||||
@@ -0,0 +1,77 @@
|
||||
|
||||
use std::collections::HashSet;
|
||||
use std::sync::OnceLock;
|
||||
|
||||
const PGL_LIST: &str = include_str!("pgl_domains.txt");
|
||||
|
||||
fn blocklist() -> &'static HashSet<&'static str> {
|
||||
static BLOCKLIST: OnceLock<HashSet<&str>> = OnceLock::new();
|
||||
BLOCKLIST.get_or_init(|| {
|
||||
let mut set = HashSet::with_capacity(4000);
|
||||
for line in PGL_LIST.lines() {
|
||||
let domain = line.trim();
|
||||
if !domain.is_empty() && !domain.starts_with('#') {
|
||||
set.insert(domain);
|
||||
}
|
||||
}
|
||||
for domain in EXTRA_DOMAINS {
|
||||
set.insert(*domain);
|
||||
}
|
||||
set
|
||||
})
|
||||
}
|
||||
|
||||
pub fn is_blocked(host: &str) -> bool {
|
||||
let bl = blocklist();
|
||||
|
||||
if bl.contains(host) {
|
||||
return true;
|
||||
}
|
||||
|
||||
let mut domain = host;
|
||||
while let Some(pos) = domain.find('.') {
|
||||
domain = &domain[pos + 1..];
|
||||
if bl.contains(domain) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
false
|
||||
}
|
||||
|
||||
static EXTRA_DOMAINS: &[&str] = &[];
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn test_exact_match() {
|
||||
assert!(is_blocked("google-analytics.com"));
|
||||
assert!(is_blocked("doubleclick.net"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_subdomain_match() {
|
||||
assert!(is_blocked("www.google-analytics.com"));
|
||||
assert!(is_blocked("ssl.google-analytics.com"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_not_blocked() {
|
||||
assert!(!is_blocked("google.com"));
|
||||
assert!(!is_blocked("example.com"));
|
||||
assert!(!is_blocked("github.com"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_pgl_domains() {
|
||||
assert!(is_blocked("adnxs.com"));
|
||||
assert!(is_blocked("criteo.com"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_blocklist_size() {
|
||||
assert!(blocklist().len() > 3500);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,726 @@
|
||||
use std::collections::HashMap;
|
||||
use std::sync::Arc;
|
||||
use std::time::Duration;
|
||||
|
||||
use std::net::{IpAddr, SocketAddr};
|
||||
|
||||
use reqwest::dns::{Addrs, Name, Resolve, Resolving};
|
||||
use reqwest::header::{HeaderMap, HeaderName, HeaderValue, USER_AGENT};
|
||||
use reqwest::redirect::Policy;
|
||||
use reqwest::{Client, Method};
|
||||
use tokio::sync::RwLock;
|
||||
use url::Url;
|
||||
|
||||
use crate::cookies::CookieJar;
|
||||
use crate::interceptor::{InterceptAction, RequestInterceptor};
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct Response {
|
||||
pub url: Url,
|
||||
pub status: u16,
|
||||
pub headers: HashMap<String, String>,
|
||||
pub body: Vec<u8>,
|
||||
pub redirected_from: Vec<Url>,
|
||||
}
|
||||
|
||||
impl Response {
|
||||
/// Decode the body as text, honoring the response charset.
|
||||
///
|
||||
/// Uses the HTTP `Content-Type` header's `charset=` parameter, then for
|
||||
/// HTML responses falls back to sniffing `<meta charset>` in the first
|
||||
/// 1KB, then UTF-8. Mirrors browser behaviour per the HTML5 spec.
|
||||
pub fn text(&self) -> String {
|
||||
if self.is_html() {
|
||||
crate::encoding::decode_response(&self.body, self.content_type())
|
||||
} else {
|
||||
crate::encoding::decode_non_html(&self.body, self.content_type())
|
||||
}
|
||||
}
|
||||
|
||||
pub fn header(&self, name: &str) -> Option<&str> {
|
||||
self.headers.get(&name.to_lowercase()).map(|s| s.as_str())
|
||||
}
|
||||
|
||||
pub fn content_type(&self) -> Option<&str> {
|
||||
self.header("content-type")
|
||||
}
|
||||
|
||||
pub fn is_html(&self) -> bool {
|
||||
self.content_type()
|
||||
.map(|ct| ct.contains("text/html"))
|
||||
.unwrap_or(false)
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct RequestInfo {
|
||||
pub url: Url,
|
||||
pub method: String,
|
||||
pub headers: HashMap<String, String>,
|
||||
pub resource_type: ResourceType,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub enum ResourceType {
|
||||
Document,
|
||||
Script,
|
||||
Stylesheet,
|
||||
Image,
|
||||
Font,
|
||||
Xhr,
|
||||
Fetch,
|
||||
Other,
|
||||
}
|
||||
|
||||
pub type RequestCallback = Arc<dyn Fn(&RequestInfo) + Send + Sync>;
|
||||
pub type ResponseCallback = Arc<dyn Fn(&RequestInfo, &Response) + Send + Sync>;
|
||||
|
||||
/// Process-wide opt-in via env var. Older flow that issue #4 introduced. The
|
||||
/// new `--allow-private-network` CLI flag (issue #33) sets a per-client field
|
||||
/// that is OR'd with this so existing scripts and Docker setups that pin the
|
||||
/// env var keep working unchanged.
|
||||
pub fn env_allows_private_network() -> bool {
|
||||
matches!(
|
||||
std::env::var("OBSCURA_ALLOW_PRIVATE_NETWORK")
|
||||
.ok()
|
||||
.as_deref()
|
||||
.map(str::trim)
|
||||
.map(str::to_ascii_lowercase)
|
||||
.as_deref(),
|
||||
Some("1") | Some("true") | Some("yes") | Some("on")
|
||||
)
|
||||
}
|
||||
|
||||
/// True when `ip` must never be the target of an outbound request from the
|
||||
/// engine: loopback, RFC1918 private, link-local (incl. the 169.254.169.254
|
||||
/// cloud-metadata endpoint), broadcast, documentation, the unspecified address
|
||||
/// (0.0.0.0 / ::, which the OS routes to localhost), IPv6 unique-local
|
||||
/// (fc00::/7), and any IPv4-mapped/compatible IPv6 form of the above.
|
||||
/// Centralizes the SSRF deny-set so the literal-host check and the
|
||||
/// DNS-resolution check (`SsrfGuardResolver`) can never disagree.
|
||||
pub fn is_forbidden_ip(ip: IpAddr) -> bool {
|
||||
match ip {
|
||||
IpAddr::V4(v4) => {
|
||||
v4.is_loopback()
|
||||
|| v4.is_private()
|
||||
|| v4.is_link_local()
|
||||
|| v4.is_broadcast()
|
||||
|| v4.is_documentation()
|
||||
|| v4.is_unspecified()
|
||||
}
|
||||
IpAddr::V6(v6) => {
|
||||
if v6.is_loopback()
|
||||
|| v6.is_unspecified()
|
||||
|| v6.is_unique_local()
|
||||
|| v6.is_unicast_link_local()
|
||||
{
|
||||
return true;
|
||||
}
|
||||
// Unwrap IPv4-mapped (::ffff:a.b.c.d) and IPv4-compatible (::a.b.c.d)
|
||||
// forms and re-check the embedded v4 so e.g. [::ffff:127.0.0.1] or
|
||||
// [::ffff:169.254.169.254] cannot slip past the v6 arm.
|
||||
if let Some(v4) = v6.to_ipv4_mapped().or_else(|| v6.to_ipv4()) {
|
||||
return is_forbidden_ip(IpAddr::V4(v4));
|
||||
}
|
||||
false
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// reqwest DNS resolver that performs the lookup and then rejects the whole
|
||||
/// request if ANY resolved address is in the SSRF deny-set. This closes the
|
||||
/// DNS-rebinding bypass a host-string check alone cannot: a public name that
|
||||
/// resolves to 127.0.0.1 / 169.254.169.254 / an RFC1918 address is blocked at
|
||||
/// connect time, using the very addresses reqwest will dial. When private
|
||||
/// access is permitted (`--allow-private-network` or
|
||||
/// `OBSCURA_ALLOW_PRIVATE_NETWORK`) the lookup passes through unfiltered.
|
||||
pub struct SsrfGuardResolver {
|
||||
allow_private: bool,
|
||||
}
|
||||
|
||||
impl SsrfGuardResolver {
|
||||
pub fn new(allow_private: bool) -> Self {
|
||||
Self { allow_private }
|
||||
}
|
||||
}
|
||||
|
||||
impl Resolve for SsrfGuardResolver {
|
||||
fn resolve(&self, name: Name) -> Resolving {
|
||||
let allow = self.allow_private || env_allows_private_network();
|
||||
let host = name.as_str().to_string();
|
||||
Box::pin(async move {
|
||||
let addrs: Vec<SocketAddr> = tokio::net::lookup_host((host.as_str(), 0))
|
||||
.await
|
||||
.map_err(|e| -> Box<dyn std::error::Error + Send + Sync> { Box::new(e) })?
|
||||
.collect();
|
||||
if !allow {
|
||||
if let Some(bad) = addrs.iter().find(|sa| is_forbidden_ip(sa.ip())) {
|
||||
return Err(format!(
|
||||
"SSRF blocked: '{}' resolves to forbidden address {}",
|
||||
host,
|
||||
bad.ip()
|
||||
)
|
||||
.into());
|
||||
}
|
||||
}
|
||||
let iter: Addrs = Box::new(addrs.into_iter());
|
||||
Ok(iter)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
fn validate_url(url: &Url, allow_private_network: bool) -> Result<(), ObscuraNetError> {
|
||||
let allow_private_network = allow_private_network || env_allows_private_network();
|
||||
let scheme = url.scheme();
|
||||
if scheme != "http" && scheme != "https" && scheme != "file" {
|
||||
return Err(ObscuraNetError::Network(format!(
|
||||
"Forbidden URL scheme '{}' - only http, https, and file are allowed",
|
||||
scheme
|
||||
)));
|
||||
}
|
||||
|
||||
if scheme == "file" || allow_private_network {
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
if let Some(host) = url.host() {
|
||||
match host {
|
||||
url::Host::Ipv4(ip) => {
|
||||
if is_forbidden_ip(IpAddr::V4(ip)) {
|
||||
return Err(ObscuraNetError::Network(format!(
|
||||
"Access to private/internal IP address {} is not allowed",
|
||||
ip
|
||||
)));
|
||||
}
|
||||
}
|
||||
url::Host::Ipv6(ip) => {
|
||||
if is_forbidden_ip(IpAddr::V6(ip)) {
|
||||
return Err(ObscuraNetError::Network(format!(
|
||||
"Access to private/internal IPv6 address {} is not allowed",
|
||||
ip
|
||||
)));
|
||||
}
|
||||
}
|
||||
url::Host::Domain(domain) => {
|
||||
let lower_domain = domain.to_lowercase();
|
||||
if lower_domain == "localhost"
|
||||
|| lower_domain.ends_with(".localhost")
|
||||
|| lower_domain == "127.0.0.1"
|
||||
|| lower_domain == "::1"
|
||||
{
|
||||
return Err(ObscuraNetError::Network(format!(
|
||||
"Access to localhost domain '{}' is not allowed",
|
||||
domain
|
||||
)));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn fetch_file_url(url: &Url) -> Result<Response, ObscuraNetError> {
|
||||
let path = url
|
||||
.to_file_path()
|
||||
.map_err(|_| ObscuraNetError::Network("Invalid file URL".to_string()))?;
|
||||
let body = tokio::fs::read(&path)
|
||||
.await
|
||||
.map_err(|e| ObscuraNetError::Network(format!("Failed to read file: {}", e)))?;
|
||||
|
||||
let mut headers = HashMap::new();
|
||||
if let Some(ext) = path.extension().and_then(|e| e.to_str()) {
|
||||
let ct = match ext.to_lowercase().as_str() {
|
||||
"html" | "htm" => "text/html",
|
||||
"css" => "text/css",
|
||||
"js" | "mjs" => "application/javascript",
|
||||
"json" => "application/json",
|
||||
"png" => "image/png",
|
||||
"jpg" | "jpeg" => "image/jpeg",
|
||||
"gif" => "image/gif",
|
||||
"svg" => "image/svg+xml",
|
||||
"webp" => "image/webp",
|
||||
"ico" => "image/x-icon",
|
||||
_ => "application/octet-stream",
|
||||
};
|
||||
headers.insert("content-type".to_string(), ct.to_string());
|
||||
}
|
||||
|
||||
Ok(Response {
|
||||
url: url.clone(),
|
||||
status: 200,
|
||||
headers,
|
||||
body,
|
||||
redirected_from: Vec::new(),
|
||||
})
|
||||
}
|
||||
|
||||
pub struct ObscuraHttpClient {
|
||||
client: tokio::sync::OnceCell<Client>,
|
||||
proxy_url: Option<String>,
|
||||
pub cookie_jar: Arc<CookieJar>,
|
||||
pub user_agent: RwLock<String>,
|
||||
pub extra_headers: RwLock<HashMap<String, String>>,
|
||||
pub interceptor: RwLock<Option<Box<dyn RequestInterceptor + Send + Sync>>>,
|
||||
pub on_request: RwLock<Vec<RequestCallback>>,
|
||||
pub on_response: RwLock<Vec<ResponseCallback>>,
|
||||
pub timeout: Duration,
|
||||
pub in_flight: Arc<std::sync::atomic::AtomicU32>,
|
||||
pub block_trackers: bool,
|
||||
/// When true, `validate_url` lets localhost / RFC1918 / link-local addresses
|
||||
/// through in addition to the `OBSCURA_ALLOW_PRIVATE_NETWORK` env var.
|
||||
/// Set via `--allow-private-network` on the CLI (issue #33).
|
||||
pub allow_private_network: bool,
|
||||
}
|
||||
|
||||
/// Derive the sec-ch-ua and sec-ch-ua-platform client-hint header values from a
|
||||
/// User-Agent string, using Chromium's per-major-version GREASE algorithm so
|
||||
/// the non-stealth HTTP path agrees with navigator.userAgentData instead of
|
||||
/// shipping a fixed Linux/Chrome-145 hint that contradicts a Windows profile.
|
||||
fn chrome_client_hints(ua: &str) -> (String, String) {
|
||||
let major: usize = ua
|
||||
.split("Chrome/")
|
||||
.nth(1)
|
||||
.and_then(|s| s.split('.').next())
|
||||
.and_then(|s| s.parse::<usize>().ok())
|
||||
.unwrap_or(145);
|
||||
const GREASE_CHARS: [char; 11] = [' ', '(', ':', '-', '.', '/', ')', ';', '=', '?', '_'];
|
||||
const GREASE_VER: [&str; 3] = ["8", "99", "24"];
|
||||
const PERMS: [[usize; 3]; 6] = [[0, 1, 2], [0, 2, 1], [1, 0, 2], [1, 2, 0], [2, 0, 1], [2, 1, 0]];
|
||||
let grease_brand = format!(
|
||||
"Not{}A{}Brand",
|
||||
GREASE_CHARS[major % 11],
|
||||
GREASE_CHARS[(major + 1) % 11]
|
||||
);
|
||||
let brands = [
|
||||
(grease_brand, GREASE_VER[major % 3].to_string()),
|
||||
("Chromium".to_string(), major.to_string()),
|
||||
("Google Chrome".to_string(), major.to_string()),
|
||||
];
|
||||
let p = PERMS[major % 6];
|
||||
let sec_ch_ua = p
|
||||
.iter()
|
||||
.map(|&i| format!("\"{}\";v=\"{}\"", brands[i].0, brands[i].1))
|
||||
.collect::<Vec<_>>()
|
||||
.join(", ");
|
||||
let platform = if ua.contains("Windows NT") {
|
||||
"\"Windows\""
|
||||
} else if ua.contains("Macintosh") {
|
||||
"\"macOS\""
|
||||
} else {
|
||||
"\"Linux\""
|
||||
};
|
||||
(sec_ch_ua, platform.to_string())
|
||||
}
|
||||
|
||||
impl ObscuraHttpClient {
|
||||
pub fn new() -> Self {
|
||||
Self::with_cookie_jar(Arc::new(CookieJar::new()))
|
||||
}
|
||||
|
||||
pub fn with_cookie_jar(cookie_jar: Arc<CookieJar>) -> Self {
|
||||
Self::with_options(cookie_jar, None)
|
||||
}
|
||||
|
||||
pub fn with_options(cookie_jar: Arc<CookieJar>, proxy_url: Option<&str>) -> Self {
|
||||
Self::with_full_options(cookie_jar, proxy_url, false)
|
||||
}
|
||||
|
||||
pub fn with_full_options(
|
||||
cookie_jar: Arc<CookieJar>,
|
||||
proxy_url: Option<&str>,
|
||||
allow_private_network: bool,
|
||||
) -> Self {
|
||||
ObscuraHttpClient {
|
||||
client: tokio::sync::OnceCell::new(),
|
||||
proxy_url: proxy_url.map(|s| s.to_string()),
|
||||
cookie_jar,
|
||||
user_agent: RwLock::new(
|
||||
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/145.0.0.0 Safari/537.36".to_string(),
|
||||
),
|
||||
extra_headers: RwLock::new(HashMap::new()),
|
||||
interceptor: RwLock::new(None),
|
||||
on_request: RwLock::new(Vec::new()),
|
||||
on_response: RwLock::new(Vec::new()),
|
||||
in_flight: Arc::new(std::sync::atomic::AtomicU32::new(0)),
|
||||
timeout: Duration::from_secs(30),
|
||||
block_trackers: false,
|
||||
allow_private_network,
|
||||
}
|
||||
}
|
||||
|
||||
async fn get_client(&self) -> &Client {
|
||||
self.client.get_or_init(|| async {
|
||||
let mut builder = Client::builder()
|
||||
.redirect(Policy::none())
|
||||
.timeout(Duration::from_secs(30))
|
||||
.danger_accept_invalid_certs(false)
|
||||
// SSRF guard: reject hostnames that resolve to a private/loopback IP.
|
||||
.dns_resolver(Arc::new(SsrfGuardResolver::new(self.allow_private_network)))
|
||||
;
|
||||
|
||||
if let Some(ref proxy) = self.proxy_url {
|
||||
if let Ok(p) = reqwest::Proxy::all(proxy.as_str()) {
|
||||
builder = builder.proxy(p);
|
||||
}
|
||||
}
|
||||
|
||||
builder.build().expect("failed to build HTTP client")
|
||||
}).await
|
||||
}
|
||||
|
||||
/// Read-only accessor for the proxy URL the client was configured with
|
||||
/// (if any). Exposed so callers outside the `obscura-net` crate — notably
|
||||
/// `op_fetch_url` in `obscura-js` (#139) — can route their own reqwest
|
||||
/// requests through the same upstream proxy.
|
||||
pub fn proxy_url(&self) -> Option<&str> {
|
||||
self.proxy_url.as_deref()
|
||||
}
|
||||
|
||||
pub async fn fetch(&self, url: &Url) -> Result<Response, ObscuraNetError> {
|
||||
self.fetch_with_method(Method::GET, url, None).await
|
||||
}
|
||||
|
||||
pub async fn post_form(&self, url: &Url, body: &str) -> Result<Response, ObscuraNetError> {
|
||||
self.fetch_with_method(Method::POST, url, Some(body.as_bytes().to_vec())).await
|
||||
}
|
||||
|
||||
pub async fn fetch_with_method(
|
||||
&self,
|
||||
initial_method: Method,
|
||||
url: &Url,
|
||||
initial_body: Option<Vec<u8>>,
|
||||
) -> Result<Response, ObscuraNetError> {
|
||||
validate_url(url, self.allow_private_network)?;
|
||||
|
||||
if url.scheme() == "file" {
|
||||
return fetch_file_url(url).await;
|
||||
}
|
||||
|
||||
let mut method = initial_method;
|
||||
let mut body = initial_body;
|
||||
if self.block_trackers {
|
||||
if let Some(host) = url.host_str() {
|
||||
if crate::blocklist::is_blocked(host) {
|
||||
tracing::debug!("Blocked tracker: {}", url);
|
||||
return Ok(Response {
|
||||
status: 0,
|
||||
url: url.clone(),
|
||||
headers: HashMap::new(),
|
||||
body: Vec::new(),
|
||||
redirected_from: Vec::new(),
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let mut current_url = url.clone();
|
||||
let mut redirects = Vec::new();
|
||||
let max_redirects = 20;
|
||||
|
||||
for _redirect_count in 0..max_redirects {
|
||||
let request_info = RequestInfo {
|
||||
url: current_url.clone(),
|
||||
method: method.to_string(),
|
||||
headers: self.extra_headers.read().await.clone(),
|
||||
resource_type: ResourceType::Document,
|
||||
};
|
||||
|
||||
if let Some(interceptor) = self.interceptor.read().await.as_ref() {
|
||||
match interceptor.intercept(&request_info).await {
|
||||
InterceptAction::Continue => {}
|
||||
InterceptAction::Block => {
|
||||
return Err(ObscuraNetError::Blocked(current_url.to_string()));
|
||||
}
|
||||
InterceptAction::Fulfill(response) => {
|
||||
return Ok(response);
|
||||
}
|
||||
InterceptAction::ModifyHeaders(headers) => {
|
||||
let mut extra = self.extra_headers.write().await;
|
||||
extra.extend(headers);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
for cb in self.on_request.read().await.iter() {
|
||||
cb(&request_info);
|
||||
}
|
||||
|
||||
let ua = self.user_agent.read().await.clone();
|
||||
let (sec_ch_ua, sec_ch_ua_platform) = chrome_client_hints(&ua);
|
||||
let mut headers = HeaderMap::new();
|
||||
// Chrome's top-level navigation header order. (reqwest appends
|
||||
// accept-encoding/host after these, so accept-encoding lands after
|
||||
// accept-language rather than before it; the rest matches Chrome.)
|
||||
headers.insert(
|
||||
HeaderName::from_static("sec-ch-ua"),
|
||||
HeaderValue::from_str(&sec_ch_ua)
|
||||
.unwrap_or_else(|_| HeaderValue::from_static("\"Not:A-Brand\";v=\"99\", \"Google Chrome\";v=\"145\", \"Chromium\";v=\"145\"")),
|
||||
);
|
||||
headers.insert(HeaderName::from_static("sec-ch-ua-mobile"), HeaderValue::from_static("?0"));
|
||||
headers.insert(
|
||||
HeaderName::from_static("sec-ch-ua-platform"),
|
||||
HeaderValue::from_str(&sec_ch_ua_platform)
|
||||
.unwrap_or_else(|_| HeaderValue::from_static("\"Windows\"")),
|
||||
);
|
||||
headers.insert(HeaderName::from_static("upgrade-insecure-requests"), HeaderValue::from_static("1"));
|
||||
headers.insert(USER_AGENT, HeaderValue::from_str(&ua).unwrap_or_else(|_| {
|
||||
HeaderValue::from_static("Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/145.0.0.0 Safari/537.36")
|
||||
}));
|
||||
headers.insert(
|
||||
reqwest::header::ACCEPT,
|
||||
HeaderValue::from_static("text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7"),
|
||||
);
|
||||
headers.insert(HeaderName::from_static("sec-fetch-site"), HeaderValue::from_static("none"));
|
||||
headers.insert(HeaderName::from_static("sec-fetch-mode"), HeaderValue::from_static("navigate"));
|
||||
headers.insert(HeaderName::from_static("sec-fetch-user"), HeaderValue::from_static("?1"));
|
||||
headers.insert(HeaderName::from_static("sec-fetch-dest"), HeaderValue::from_static("document"));
|
||||
headers.insert(
|
||||
reqwest::header::ACCEPT_LANGUAGE,
|
||||
HeaderValue::from_static("en-US,en;q=0.9"),
|
||||
);
|
||||
|
||||
let cookie_header = self.cookie_jar.get_cookie_header(¤t_url);
|
||||
tracing::debug!(
|
||||
"Cookie header for {}: {} cookies ({} bytes)",
|
||||
current_url.host_str().unwrap_or("?"),
|
||||
cookie_header.split("; ").filter(|s| !s.is_empty()).count(),
|
||||
cookie_header.len(),
|
||||
);
|
||||
if !cookie_header.is_empty() {
|
||||
match HeaderValue::from_str(&cookie_header) {
|
||||
Ok(val) => {
|
||||
headers.insert(reqwest::header::COOKIE, val);
|
||||
}
|
||||
Err(_) => {
|
||||
let filtered: String = cookie_header
|
||||
.split("; ")
|
||||
.filter(|pair| HeaderValue::from_str(pair).is_ok())
|
||||
.collect::<Vec<_>>()
|
||||
.join("; ");
|
||||
if !filtered.is_empty() {
|
||||
if let Ok(val) = HeaderValue::from_str(&filtered) {
|
||||
headers.insert(reqwest::header::COOKIE, val);
|
||||
}
|
||||
}
|
||||
tracing::debug!(
|
||||
"Cookie header invalid chars, filtered {} -> {} bytes",
|
||||
cookie_header.len(), filtered.len(),
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
for (k, v) in self.extra_headers.read().await.iter() {
|
||||
if let (Ok(name), Ok(val)) = (
|
||||
HeaderName::from_bytes(k.as_bytes()),
|
||||
HeaderValue::from_str(v),
|
||||
) {
|
||||
headers.insert(name, val);
|
||||
}
|
||||
}
|
||||
|
||||
let mut req_builder = self.get_client().await.request(method.clone(), current_url.as_str())
|
||||
.headers(headers);
|
||||
|
||||
if let Some(ref b) = body {
|
||||
if method == Method::POST {
|
||||
req_builder = req_builder.header(
|
||||
reqwest::header::CONTENT_TYPE,
|
||||
"application/x-www-form-urlencoded",
|
||||
);
|
||||
}
|
||||
req_builder = req_builder.body(b.clone());
|
||||
}
|
||||
|
||||
self.in_flight.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
|
||||
let resp = req_builder.send().await.map_err(|e| {
|
||||
self.in_flight.fetch_sub(1, std::sync::atomic::Ordering::Relaxed);
|
||||
ObscuraNetError::Network(format!("{}: {}", current_url, e))
|
||||
})?;
|
||||
self.in_flight.fetch_sub(1, std::sync::atomic::Ordering::Relaxed);
|
||||
|
||||
let status = resp.status();
|
||||
|
||||
for val in resp.headers().get_all(reqwest::header::SET_COOKIE) {
|
||||
if let Ok(s) = val.to_str() {
|
||||
self.cookie_jar.set_cookie(s, ¤t_url);
|
||||
}
|
||||
}
|
||||
|
||||
let response_headers: HashMap<String, String> = resp
|
||||
.headers()
|
||||
.iter()
|
||||
.map(|(k, v)| (k.as_str().to_lowercase(), v.to_str().unwrap_or("").to_string()))
|
||||
.collect();
|
||||
|
||||
if status.is_redirection() {
|
||||
if let Some(location) = resp.headers().get(reqwest::header::LOCATION) {
|
||||
let location_str = location.to_str().map_err(|_| {
|
||||
ObscuraNetError::Network("Invalid redirect Location header".into())
|
||||
})?;
|
||||
let next_url = current_url.join(location_str).map_err(|e| {
|
||||
ObscuraNetError::Network(format!("Invalid redirect URL: {}", e))
|
||||
})?;
|
||||
validate_url(&next_url, self.allow_private_network)?;
|
||||
redirects.push(current_url.clone());
|
||||
current_url = next_url;
|
||||
if status == reqwest::StatusCode::MOVED_PERMANENTLY
|
||||
|| status == reqwest::StatusCode::FOUND
|
||||
|| status == reqwest::StatusCode::SEE_OTHER
|
||||
{
|
||||
method = Method::GET;
|
||||
body = None;
|
||||
}
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
let body_bytes = resp.bytes().await.map_err(|e| {
|
||||
ObscuraNetError::Network(format!("Failed to read body: {}", e))
|
||||
})?.to_vec();
|
||||
|
||||
let response = Response {
|
||||
url: current_url,
|
||||
status: status.as_u16(),
|
||||
headers: response_headers,
|
||||
body: body_bytes,
|
||||
redirected_from: redirects,
|
||||
};
|
||||
|
||||
for cb in self.on_response.read().await.iter() {
|
||||
cb(&request_info, &response);
|
||||
}
|
||||
|
||||
return Ok(response);
|
||||
}
|
||||
|
||||
Err(ObscuraNetError::TooManyRedirects(current_url.to_string()))
|
||||
}
|
||||
|
||||
pub async fn set_user_agent(&self, ua: &str) {
|
||||
*self.user_agent.write().await = ua.to_string();
|
||||
}
|
||||
|
||||
pub async fn set_extra_headers(&self, headers: HashMap<String, String>) {
|
||||
*self.extra_headers.write().await = headers;
|
||||
}
|
||||
|
||||
pub fn active_requests(&self) -> u32 {
|
||||
self.in_flight.load(std::sync::atomic::Ordering::Relaxed)
|
||||
}
|
||||
|
||||
pub fn is_network_idle(&self) -> bool {
|
||||
self.active_requests() == 0
|
||||
}
|
||||
}
|
||||
|
||||
impl Default for ObscuraHttpClient {
|
||||
fn default() -> Self {
|
||||
Self::new()
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, thiserror::Error)]
|
||||
pub enum ObscuraNetError {
|
||||
#[error("Network error: {0}")]
|
||||
Network(String),
|
||||
|
||||
#[error("Too many redirects: {0}")]
|
||||
TooManyRedirects(String),
|
||||
|
||||
#[error("Request blocked: {0}")]
|
||||
Blocked(String),
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod ssrf_tests {
|
||||
use super::{is_forbidden_ip, validate_url, SsrfGuardResolver};
|
||||
use reqwest::dns::{Name, Resolve};
|
||||
use std::net::IpAddr;
|
||||
use std::str::FromStr;
|
||||
use url::Url;
|
||||
|
||||
fn ip(s: &str) -> IpAddr {
|
||||
IpAddr::from_str(s).unwrap()
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ipv4_private_and_special_ranges_are_forbidden() {
|
||||
for s in [
|
||||
"127.0.0.1",
|
||||
"127.5.6.7",
|
||||
"10.0.0.1",
|
||||
"172.16.0.1",
|
||||
"192.168.1.1",
|
||||
"169.254.169.254", // cloud-metadata endpoint
|
||||
"0.0.0.0", // unspecified -> localhost (was a bypass)
|
||||
"255.255.255.255", // broadcast
|
||||
"192.0.2.1", // documentation
|
||||
] {
|
||||
assert!(is_forbidden_ip(ip(s)), "{s} should be forbidden");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn public_ipv4_is_allowed() {
|
||||
for s in ["1.1.1.1", "8.8.8.8", "93.184.216.34"] {
|
||||
assert!(!is_forbidden_ip(ip(s)), "{s} should be allowed");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ipv6_loopback_ula_linklocal_and_mapped_are_forbidden() {
|
||||
for s in [
|
||||
"::1", // loopback
|
||||
"::", // unspecified
|
||||
"fc00::1", // unique-local (was a bypass)
|
||||
"fd12:3456:789a::1", // unique-local
|
||||
"fe80::1", // link-local
|
||||
"::ffff:127.0.0.1", // v4-mapped loopback (was a bypass)
|
||||
"::ffff:169.254.169.254", // v4-mapped metadata
|
||||
] {
|
||||
assert!(is_forbidden_ip(ip(s)), "{s} should be forbidden");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn public_ipv6_is_allowed() {
|
||||
assert!(!is_forbidden_ip(ip("2606:4700:4700::1111"))); // cloudflare dns
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn validate_url_blocks_unspecified_and_allows_public() {
|
||||
// 0.0.0.0 previously slipped through validate_url's literal-host check.
|
||||
assert!(validate_url(&Url::parse("http://0.0.0.0:8080/").unwrap(), false).is_err());
|
||||
assert!(validate_url(&Url::parse("http://127.0.0.1/").unwrap(), false).is_err());
|
||||
assert!(validate_url(&Url::parse("http://example.com/").unwrap(), false).is_ok());
|
||||
// The allow flag bypasses the guard (local-dev escape hatch).
|
||||
assert!(validate_url(&Url::parse("http://127.0.0.1/").unwrap(), true).is_ok());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn resolver_blocks_hostname_that_resolves_to_loopback() {
|
||||
// localtest.me is a public DNS name that resolves to 127.0.0.1 — the
|
||||
// canonical DNS-rebinding test. The guard must reject it. If DNS is
|
||||
// unavailable the lookup itself errors (also Err), so the assertion
|
||||
// holds either way.
|
||||
let r = SsrfGuardResolver::new(false);
|
||||
let res = r.resolve(Name::from_str("localtest.me").unwrap()).await;
|
||||
assert!(res.is_err(), "localtest.me -> 127.0.0.1 must be blocked");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn resolver_does_not_ssrf_block_public_host() {
|
||||
// A public host must not be SSRF-blocked. Tolerate a no-network sandbox
|
||||
// by only failing on an actual SSRF rejection, not a lookup failure.
|
||||
let r = SsrfGuardResolver::new(false);
|
||||
match r.resolve(Name::from_str("example.com").unwrap()).await {
|
||||
Ok(_) => {}
|
||||
Err(e) => assert!(
|
||||
!e.to_string().contains("SSRF blocked"),
|
||||
"example.com wrongly SSRF-blocked: {e}"
|
||||
),
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,989 @@
|
||||
use std::collections::HashMap;
|
||||
use std::sync::RwLock;
|
||||
use url::Url;
|
||||
|
||||
const DEFAULT_SAME_SITE: &str = "Lax";
|
||||
|
||||
/// SameSite is case-insensitive per RFC 6265bis; normalize a present value to
|
||||
/// title-case so stored cookies compare equal regardless of how they were sent.
|
||||
/// Unrecognized values fall back to Lax per spec.
|
||||
fn normalize_same_site(value: &str) -> String {
|
||||
match value.trim().to_ascii_lowercase().as_str() {
|
||||
"strict" => "Strict",
|
||||
"none" => "None",
|
||||
_ => "Lax",
|
||||
}
|
||||
.to_string()
|
||||
}
|
||||
|
||||
pub struct CookieJar {
|
||||
cookies: RwLock<HashMap<String, HashMap<String, CookieEntry>>>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]
|
||||
struct CookieEntry {
|
||||
name: String,
|
||||
value: String,
|
||||
path: String,
|
||||
domain: String,
|
||||
/// Cookies set without a Domain attribute are host-only: sent to the exact
|
||||
/// origin host and never to subdomains. `serde(default)` keeps persisted
|
||||
/// cookie files from before this field existed loadable.
|
||||
#[serde(default)]
|
||||
host_only: bool,
|
||||
secure: bool,
|
||||
http_only: bool,
|
||||
expires: Option<u64>,
|
||||
same_site: String,
|
||||
}
|
||||
|
||||
impl CookieJar {
|
||||
pub fn new() -> Self {
|
||||
CookieJar {
|
||||
cookies: RwLock::new(HashMap::new()),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn set_cookie(&self, set_cookie_str: &str, url: &Url) {
|
||||
let parts: Vec<&str> = set_cookie_str.splitn(2, ';').collect();
|
||||
let name_value = parts[0].trim();
|
||||
let (name, value) = match name_value.split_once('=') {
|
||||
Some((n, v)) => (n.trim().to_string(), v.trim().to_string()),
|
||||
None => return,
|
||||
};
|
||||
|
||||
let request_host = url.host_str().unwrap_or("").to_lowercase();
|
||||
let mut domain_attr: Option<String> = None;
|
||||
let mut path = url.path().to_string();
|
||||
let mut secure = false;
|
||||
let mut http_only = false;
|
||||
let mut expires: Option<u64> = None;
|
||||
let mut same_site = "Lax".to_string();
|
||||
|
||||
if parts.len() > 1 {
|
||||
for attr in parts[1].split(';') {
|
||||
let attr = attr.trim();
|
||||
if let Some((key, val)) = attr.split_once('=') {
|
||||
match key.trim().to_lowercase().as_str() {
|
||||
"domain" => {
|
||||
domain_attr = Some(val.trim().trim_start_matches('.').to_lowercase());
|
||||
}
|
||||
"path" => {
|
||||
path = val.trim().to_string();
|
||||
}
|
||||
"expires" => {
|
||||
if let Ok(ts) = parse_http_date(val.trim()) {
|
||||
expires = Some(ts);
|
||||
}
|
||||
}
|
||||
"max-age" => {
|
||||
if let Ok(secs) = val.trim().parse::<i64>() {
|
||||
if secs <= 0 {
|
||||
expires = Some(0);
|
||||
} else {
|
||||
let now = std::time::SystemTime::now()
|
||||
.duration_since(std::time::UNIX_EPOCH)
|
||||
.unwrap_or_default()
|
||||
.as_secs();
|
||||
expires = Some(now + secs as u64);
|
||||
}
|
||||
}
|
||||
}
|
||||
"samesite" => {
|
||||
same_site = normalize_same_site(val);
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
} else {
|
||||
match attr.to_lowercase().as_str() {
|
||||
"secure" => secure = true,
|
||||
"httponly" => http_only = true,
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Validate Domain against the response origin (RFC 6265): an unrelated
|
||||
// or public-suffix Domain is ignored so a response from attacker.test
|
||||
// cannot scope a cookie to victim.test (GHSA-f22c-8v6q-v6h6).
|
||||
let (domain, host_only) = match resolve_cookie_domain(&request_host, domain_attr.as_deref()) {
|
||||
Some(d) => d,
|
||||
None => return,
|
||||
};
|
||||
|
||||
if let Some(exp) = expires {
|
||||
if exp == 0 {
|
||||
let mut cookies = self.cookies.write().unwrap();
|
||||
if let Some(domain_cookies) = cookies.get_mut(&domain) {
|
||||
domain_cookies.remove(&name);
|
||||
}
|
||||
return;
|
||||
}
|
||||
let now = std::time::SystemTime::now()
|
||||
.duration_since(std::time::UNIX_EPOCH)
|
||||
.unwrap_or_default()
|
||||
.as_secs();
|
||||
if exp < now {
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
let entry = CookieEntry {
|
||||
name: name.clone(),
|
||||
value,
|
||||
path,
|
||||
domain: domain.clone(),
|
||||
host_only,
|
||||
secure,
|
||||
http_only,
|
||||
expires,
|
||||
same_site,
|
||||
};
|
||||
|
||||
let mut cookies = self.cookies.write().unwrap();
|
||||
cookies.entry(domain).or_default().insert(name, entry);
|
||||
}
|
||||
|
||||
pub fn get_cookie_header(&self, url: &Url) -> String {
|
||||
let host = url.host_str().unwrap_or("");
|
||||
let path = url.path();
|
||||
let is_secure = url.scheme() == "https";
|
||||
let cookies = self.cookies.read().unwrap();
|
||||
|
||||
let mut matching: Vec<String> = Vec::new();
|
||||
|
||||
let now = std::time::SystemTime::now()
|
||||
.duration_since(std::time::UNIX_EPOCH)
|
||||
.unwrap_or_default()
|
||||
.as_secs();
|
||||
|
||||
for (domain, domain_cookies) in cookies.iter() {
|
||||
if !domain_matches(host, domain) {
|
||||
continue;
|
||||
}
|
||||
for entry in domain_cookies.values() {
|
||||
if entry.host_only && !host.eq_ignore_ascii_case(domain) {
|
||||
continue;
|
||||
}
|
||||
if let Some(exp) = entry.expires {
|
||||
if exp < now {
|
||||
continue;
|
||||
}
|
||||
}
|
||||
if entry.secure && !is_secure {
|
||||
continue;
|
||||
}
|
||||
if !path_matches(path, &entry.path) {
|
||||
continue;
|
||||
}
|
||||
matching.push(format!("{}={}", entry.name, entry.value));
|
||||
}
|
||||
}
|
||||
|
||||
matching.join("; ")
|
||||
}
|
||||
|
||||
pub fn get_all_cookies(&self) -> Vec<CookieInfo> {
|
||||
let cookies = self.cookies.read().unwrap();
|
||||
let mut result = Vec::new();
|
||||
for domain_cookies in cookies.values() {
|
||||
for entry in domain_cookies.values() {
|
||||
result.push(CookieInfo {
|
||||
name: entry.name.clone(),
|
||||
value: entry.value.clone(),
|
||||
domain: entry.domain.clone(),
|
||||
path: entry.path.clone(),
|
||||
secure: entry.secure,
|
||||
http_only: entry.http_only,
|
||||
same_site: entry.same_site.clone(),
|
||||
expires: entry.expires.map(|e| e as i64),
|
||||
});
|
||||
}
|
||||
}
|
||||
result
|
||||
}
|
||||
|
||||
pub fn set_cookies_from_cdp(&self, cookies: Vec<CookieInfo>) {
|
||||
let mut jar = self.cookies.write().unwrap();
|
||||
for cookie in cookies {
|
||||
let same_site = if cookie.same_site.is_empty() {
|
||||
DEFAULT_SAME_SITE.to_string()
|
||||
} else {
|
||||
normalize_same_site(&cookie.same_site)
|
||||
};
|
||||
let expires = cookie.expires.and_then(|e| if e > 0 { Some(e as u64) } else { None });
|
||||
let entry = CookieEntry {
|
||||
name: cookie.name.clone(),
|
||||
value: cookie.value,
|
||||
path: cookie.path,
|
||||
domain: cookie.domain.clone(),
|
||||
// CDP/persisted import is trusted; honor the explicit domain as
|
||||
// domain-scoped (matches the prior behavior).
|
||||
host_only: false,
|
||||
secure: cookie.secure,
|
||||
http_only: cookie.http_only,
|
||||
expires,
|
||||
same_site,
|
||||
};
|
||||
jar.entry(cookie.domain).or_default().insert(cookie.name, entry);
|
||||
}
|
||||
}
|
||||
|
||||
pub fn get_js_visible_cookies(&self, url: &Url) -> String {
|
||||
let host = url.host_str().unwrap_or("");
|
||||
let path = url.path();
|
||||
let is_secure = url.scheme() == "https";
|
||||
let cookies = self.cookies.read().unwrap();
|
||||
|
||||
let now = std::time::SystemTime::now()
|
||||
.duration_since(std::time::UNIX_EPOCH)
|
||||
.unwrap_or_default()
|
||||
.as_secs();
|
||||
|
||||
let mut matching: Vec<String> = Vec::new();
|
||||
|
||||
for (domain, domain_cookies) in cookies.iter() {
|
||||
if !domain_matches(host, domain) {
|
||||
continue;
|
||||
}
|
||||
for entry in domain_cookies.values() {
|
||||
if entry.host_only && !host.eq_ignore_ascii_case(domain) {
|
||||
continue;
|
||||
}
|
||||
if entry.http_only {
|
||||
continue;
|
||||
}
|
||||
if let Some(exp) = entry.expires {
|
||||
if exp < now {
|
||||
continue;
|
||||
}
|
||||
}
|
||||
if entry.secure && !is_secure {
|
||||
continue;
|
||||
}
|
||||
if !path_matches(path, &entry.path) {
|
||||
continue;
|
||||
}
|
||||
matching.push(format!("{}={}", entry.name, entry.value));
|
||||
}
|
||||
}
|
||||
|
||||
matching.join("; ")
|
||||
}
|
||||
|
||||
pub fn set_cookie_from_js(&self, cookie_str: &str, url: &Url) {
|
||||
let parts: Vec<&str> = cookie_str.splitn(2, ';').collect();
|
||||
let name_value = parts[0].trim();
|
||||
let (name, value) = match name_value.split_once('=') {
|
||||
Some((n, v)) => (n.trim().to_string(), v.trim().to_string()),
|
||||
None => return,
|
||||
};
|
||||
|
||||
let request_host = url.host_str().unwrap_or("").to_lowercase();
|
||||
let mut domain_attr: Option<String> = None;
|
||||
let mut path = url.path().to_string();
|
||||
let mut secure = false;
|
||||
let mut expires: Option<u64> = None;
|
||||
let mut same_site = "Lax".to_string();
|
||||
|
||||
if parts.len() > 1 {
|
||||
for attr in parts[1].split(';') {
|
||||
let attr = attr.trim();
|
||||
if let Some((key, val)) = attr.split_once('=') {
|
||||
match key.trim().to_lowercase().as_str() {
|
||||
"domain" => {
|
||||
domain_attr = Some(val.trim().trim_start_matches('.').to_lowercase());
|
||||
}
|
||||
"path" => {
|
||||
path = val.trim().to_string();
|
||||
}
|
||||
"expires" => {
|
||||
if let Ok(ts) = parse_http_date(val.trim()) {
|
||||
expires = Some(ts);
|
||||
}
|
||||
}
|
||||
"max-age" => {
|
||||
if let Ok(secs) = val.trim().parse::<i64>() {
|
||||
if secs <= 0 {
|
||||
expires = Some(0);
|
||||
} else {
|
||||
let now = std::time::SystemTime::now()
|
||||
.duration_since(std::time::UNIX_EPOCH)
|
||||
.unwrap_or_default()
|
||||
.as_secs();
|
||||
expires = Some(now + secs as u64);
|
||||
}
|
||||
}
|
||||
}
|
||||
"samesite" => {
|
||||
same_site = normalize_same_site(val);
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
} else {
|
||||
match attr.to_lowercase().as_str() {
|
||||
"secure" => secure = true,
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let (domain, host_only) = match resolve_cookie_domain(&request_host, domain_attr.as_deref()) {
|
||||
Some(d) => d,
|
||||
None => return,
|
||||
};
|
||||
|
||||
if let Some(exp) = expires {
|
||||
if exp == 0 {
|
||||
let mut cookies = self.cookies.write().unwrap();
|
||||
if let Some(domain_cookies) = cookies.get_mut(&domain) {
|
||||
domain_cookies.remove(&name);
|
||||
}
|
||||
return;
|
||||
}
|
||||
let now = std::time::SystemTime::now()
|
||||
.duration_since(std::time::UNIX_EPOCH)
|
||||
.unwrap_or_default()
|
||||
.as_secs();
|
||||
if exp < now {
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
let entry = CookieEntry {
|
||||
name: name.clone(),
|
||||
value,
|
||||
path,
|
||||
domain: domain.clone(),
|
||||
host_only,
|
||||
secure,
|
||||
http_only: false,
|
||||
expires,
|
||||
same_site,
|
||||
};
|
||||
|
||||
let mut cookies = self.cookies.write().unwrap();
|
||||
cookies.entry(domain).or_default().insert(name, entry);
|
||||
}
|
||||
|
||||
pub fn delete_cookie(&self, name: &str, domain: &str) {
|
||||
let mut cookies = self.cookies.write().unwrap();
|
||||
if domain.is_empty() {
|
||||
for domain_cookies in cookies.values_mut() {
|
||||
domain_cookies.remove(name);
|
||||
}
|
||||
} else {
|
||||
let domains_to_try = [
|
||||
domain.to_string(),
|
||||
format!(".{}", domain.trim_start_matches('.')),
|
||||
domain.trim_start_matches('.').to_string(),
|
||||
];
|
||||
for d in &domains_to_try {
|
||||
if let Some(domain_cookies) = cookies.get_mut(d.as_str()) {
|
||||
domain_cookies.remove(name);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub fn delete_cookies_filtered(&self, name: &str, domain: &str, path: Option<&str>) {
|
||||
let mut cookies = self.cookies.write().unwrap();
|
||||
let matches_path = |entry_path: &str| match path {
|
||||
Some(p) => entry_path == p,
|
||||
None => true,
|
||||
};
|
||||
if domain.is_empty() {
|
||||
for domain_cookies in cookies.values_mut() {
|
||||
domain_cookies.retain(|n, e| !(n == name && matches_path(&e.path)));
|
||||
}
|
||||
} else {
|
||||
let domains_to_try = [
|
||||
domain.to_string(),
|
||||
format!(".{}", domain.trim_start_matches('.')),
|
||||
domain.trim_start_matches('.').to_string(),
|
||||
];
|
||||
for d in &domains_to_try {
|
||||
if let Some(domain_cookies) = cookies.get_mut(d.as_str()) {
|
||||
domain_cookies.retain(|n, e| !(n == name && matches_path(&e.path)));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub fn clear(&self) {
|
||||
self.cookies.write().unwrap().clear();
|
||||
}
|
||||
|
||||
/// Serialize all non-expired cookies to a JSON file.
|
||||
/// Writes atomically via tempfile then rename.
|
||||
pub fn save_to_file(&self, path: &std::path::Path) -> Result<(), std::io::Error> {
|
||||
use std::io::Write;
|
||||
|
||||
let cookies = self.cookies.read().unwrap();
|
||||
let now = std::time::SystemTime::now()
|
||||
.duration_since(std::time::UNIX_EPOCH)
|
||||
.unwrap_or_default()
|
||||
.as_secs();
|
||||
|
||||
let mut all: Vec<CookieInfo> = Vec::new();
|
||||
for domain_cookies in cookies.values() {
|
||||
for entry in domain_cookies.values() {
|
||||
if let Some(exp) = entry.expires {
|
||||
if exp < now {
|
||||
continue;
|
||||
}
|
||||
}
|
||||
all.push(CookieInfo {
|
||||
name: entry.name.clone(),
|
||||
value: entry.value.clone(),
|
||||
domain: entry.domain.clone(),
|
||||
path: entry.path.clone(),
|
||||
secure: entry.secure,
|
||||
http_only: entry.http_only,
|
||||
same_site: entry.same_site.clone(),
|
||||
expires: entry.expires.map(|e| e as i64),
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
let json = serde_json::to_string_pretty(&all).map_err(|e| {
|
||||
std::io::Error::new(std::io::ErrorKind::InvalidData, e)
|
||||
})?;
|
||||
if let Some(parent) = path.parent() {
|
||||
std::fs::create_dir_all(parent)?;
|
||||
}
|
||||
let mut tmp = tempfile::NamedTempFile::new_in(
|
||||
path.parent().unwrap_or(std::path::Path::new(".")),
|
||||
)?;
|
||||
tmp.write_all(json.as_bytes())?;
|
||||
tmp.persist(path).map_err(|e| e.error)?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Load cookies from a JSON file into the jar.
|
||||
/// Merges with existing cookies (does not clear).
|
||||
/// Returns the number of cookies loaded.
|
||||
pub fn load_from_file(&self, path: &std::path::Path) -> Result<usize, std::io::Error> {
|
||||
if !path.exists() {
|
||||
return Ok(0);
|
||||
}
|
||||
let data = std::fs::read_to_string(path)?;
|
||||
let cookies: Vec<CookieInfo> =
|
||||
serde_json::from_str(&data).map_err(|e| {
|
||||
std::io::Error::new(std::io::ErrorKind::InvalidData, e)
|
||||
})?;
|
||||
let count = cookies.len();
|
||||
self.set_cookies_from_cdp(cookies);
|
||||
Ok(count)
|
||||
}
|
||||
}
|
||||
|
||||
impl Default for CookieJar {
|
||||
fn default() -> Self {
|
||||
Self::new()
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]
|
||||
pub struct CookieInfo {
|
||||
pub name: String,
|
||||
pub value: String,
|
||||
pub domain: String,
|
||||
pub path: String,
|
||||
pub secure: bool,
|
||||
#[serde(rename = "httpOnly")]
|
||||
pub http_only: bool,
|
||||
#[serde(default, rename = "sameSite")]
|
||||
pub same_site: String,
|
||||
#[serde(default)]
|
||||
pub expires: Option<i64>,
|
||||
}
|
||||
|
||||
fn parse_http_date(s: &str) -> Result<u64, ()> {
|
||||
let months = ["jan", "feb", "mar", "apr", "may", "jun", "jul", "aug", "sep", "oct", "nov", "dec"];
|
||||
|
||||
let s = s.replace('-', " ");
|
||||
let parts: Vec<&str> = s.split_whitespace().collect();
|
||||
|
||||
if parts.len() < 5 { return Err(()); }
|
||||
|
||||
let day: u64 = parts[1].parse().map_err(|_| ())?;
|
||||
let month = months.iter().position(|m| parts[2].to_lowercase().starts_with(m))
|
||||
.ok_or(())? as u64 + 1;
|
||||
let year: u64 = parts[3].parse().map_err(|_| ())?;
|
||||
|
||||
let time_parts: Vec<&str> = parts[4].split(':').collect();
|
||||
let hour: u64 = time_parts.first().and_then(|s| s.parse().ok()).unwrap_or(0);
|
||||
let minute: u64 = time_parts.get(1).and_then(|s| s.parse().ok()).unwrap_or(0);
|
||||
let second: u64 = time_parts.get(2).and_then(|s| s.parse().ok()).unwrap_or(0);
|
||||
|
||||
let mut days_total: u64 = 0;
|
||||
for y in 1970..year {
|
||||
days_total += if y % 4 == 0 && (y % 100 != 0 || y % 400 == 0) { 366 } else { 365 };
|
||||
}
|
||||
let days_in_month = [0, 31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31];
|
||||
let is_leap = year % 4 == 0 && (year % 100 != 0 || year % 400 == 0);
|
||||
for m in 1..month {
|
||||
days_total += days_in_month[m as usize] + if m == 2 && is_leap { 1 } else { 0 };
|
||||
}
|
||||
days_total += day - 1;
|
||||
|
||||
Ok(days_total * 86400 + hour * 3600 + minute * 60 + second)
|
||||
}
|
||||
|
||||
/// Resolve the storage domain and host-only flag for a cookie being set from
|
||||
/// `origin_host` (RFC 6265 §5.2/§5.3). With no Domain attribute the cookie is
|
||||
/// host-only: scoped to the exact origin host. A Domain attribute is honored
|
||||
/// only when it domain-matches the origin (equal to it or a parent domain) and
|
||||
/// is not an obvious public suffix; otherwise the attribute is ignored and the
|
||||
/// cookie is stored host-only on the origin. This is what stops a response from
|
||||
/// attacker.test planting a cookie scoped to victim.test.
|
||||
///
|
||||
/// Returns None only when the origin host itself is absent (the cookie cannot
|
||||
/// be scoped and is dropped).
|
||||
///
|
||||
/// Note: a full public suffix list is not bundled, so multi-label public
|
||||
/// suffixes (co.uk, github.io) are not rejected; the domain-match check still
|
||||
/// blocks the reported cross-domain attack, and single-label suffixes (com,
|
||||
/// local) are rejected.
|
||||
fn resolve_cookie_domain(origin_host: &str, domain_attr: Option<&str>) -> Option<(String, bool)> {
|
||||
let origin = origin_host.trim().trim_start_matches('.').to_lowercase();
|
||||
if origin.is_empty() {
|
||||
return None;
|
||||
}
|
||||
let dom = match domain_attr {
|
||||
None => return Some((origin, true)),
|
||||
Some(raw) => raw.trim().trim_start_matches('.').to_lowercase(),
|
||||
};
|
||||
if dom.is_empty() || dom == origin {
|
||||
return Some((origin, true));
|
||||
}
|
||||
if dom.contains('.') && origin.ends_with(&format!(".{dom}")) {
|
||||
Some((dom, false))
|
||||
} else {
|
||||
Some((origin, true))
|
||||
}
|
||||
}
|
||||
|
||||
// RFC 6265 5.1.4 path-match. A bare `starts_with` over-matches sibling paths
|
||||
// that share a string prefix (a Path=/admin cookie leaking to /administrator),
|
||||
// so a prefix match also requires the boundary to fall on a '/'.
|
||||
fn path_matches(request_path: &str, cookie_path: &str) -> bool {
|
||||
if request_path == cookie_path {
|
||||
return true;
|
||||
}
|
||||
if !request_path.starts_with(cookie_path) {
|
||||
return false;
|
||||
}
|
||||
// Prefix match: exact when the cookie-path already ends in '/', otherwise
|
||||
// the next char of the request-path must be the '/' boundary.
|
||||
cookie_path.ends_with('/') || request_path.as_bytes().get(cookie_path.len()) == Some(&b'/')
|
||||
}
|
||||
|
||||
fn domain_matches(host: &str, domain: &str) -> bool {
|
||||
// Avoid allocations on the hot path. Cookie lookup runs per fetch
|
||||
// (every subresource on a page) and walks every domain in the jar.
|
||||
// Previously this allocated 2 lowercase Strings + a "." prefix
|
||||
// per (host, domain) pair.
|
||||
let domain = domain.trim_start_matches('.');
|
||||
if host.len() < domain.len() {
|
||||
return false;
|
||||
}
|
||||
// Exact match (case-insensitive)
|
||||
if host.eq_ignore_ascii_case(domain) {
|
||||
return true;
|
||||
}
|
||||
// Suffix match with a '.' boundary: host = "sub.example.com",
|
||||
// domain = "example.com". The byte before the suffix in host
|
||||
// must be '.'.
|
||||
let prefix_len = host.len() - domain.len();
|
||||
if prefix_len < 1 { return false; }
|
||||
if !host.is_char_boundary(prefix_len) { return false; }
|
||||
if host.as_bytes()[prefix_len - 1] != b'.' { return false; }
|
||||
host[prefix_len..].eq_ignore_ascii_case(domain)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn test_set_and_get_cookie() {
|
||||
let jar = CookieJar::new();
|
||||
let url = Url::parse("https://example.com/path").unwrap();
|
||||
jar.set_cookie("session=abc123; Path=/; Secure; HttpOnly", &url);
|
||||
|
||||
let header = jar.get_cookie_header(&url);
|
||||
assert!(header.contains("session=abc123"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_cookie_domain_matching() {
|
||||
let jar = CookieJar::new();
|
||||
let url = Url::parse("https://www.example.com/").unwrap();
|
||||
jar.set_cookie("token=xyz; Domain=example.com", &url);
|
||||
|
||||
let header = jar.get_cookie_header(&url);
|
||||
assert!(header.contains("token=xyz"));
|
||||
|
||||
let sub_url = Url::parse("https://api.example.com/").unwrap();
|
||||
let header2 = jar.get_cookie_header(&sub_url);
|
||||
assert!(header2.contains("token=xyz"));
|
||||
|
||||
let other_url = Url::parse("https://other.com/").unwrap();
|
||||
let header3 = jar.get_cookie_header(&other_url);
|
||||
assert!(header3.is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_cdp_cookie_with_leading_dot_domain_matches_requests() {
|
||||
let jar = CookieJar::new();
|
||||
jar.set_cookies_from_cdp(vec![CookieInfo {
|
||||
name: "token".to_string(),
|
||||
value: "xyz".to_string(),
|
||||
domain: ".example.com".to_string(),
|
||||
path: "/".to_string(),
|
||||
secure: false,
|
||||
http_only: false,
|
||||
same_site: String::new(),
|
||||
expires: None,
|
||||
}]);
|
||||
|
||||
let apex_url = Url::parse("https://example.com/").unwrap();
|
||||
let apex_header = jar.get_cookie_header(&apex_url);
|
||||
assert!(apex_header.contains("token=xyz"));
|
||||
|
||||
let subdomain_url = Url::parse("https://api.example.com/").unwrap();
|
||||
let subdomain_header = jar.get_cookie_header(&subdomain_url);
|
||||
assert!(subdomain_header.contains("token=xyz"));
|
||||
|
||||
let other_url = Url::parse("https://other.com/").unwrap();
|
||||
let other_header = jar.get_cookie_header(&other_url);
|
||||
assert!(other_header.is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_secure_cookie_not_sent_over_http() {
|
||||
let jar = CookieJar::new();
|
||||
let https_url = Url::parse("https://example.com/").unwrap();
|
||||
jar.set_cookie("secure_token=secret; Secure", &https_url);
|
||||
|
||||
let http_url = Url::parse("http://example.com/").unwrap();
|
||||
let header = jar.get_cookie_header(&http_url);
|
||||
assert!(header.is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_max_age_zero_deletes_cookie() {
|
||||
let jar = CookieJar::new();
|
||||
let url = Url::parse("https://example.com/").unwrap();
|
||||
jar.set_cookie("session=abc", &url);
|
||||
assert!(jar.get_cookie_header(&url).contains("session=abc"));
|
||||
|
||||
jar.set_cookie("session=abc; Max-Age=0", &url);
|
||||
assert!(jar.get_cookie_header(&url).is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_max_age_sets_expiry() {
|
||||
let jar = CookieJar::new();
|
||||
let url = Url::parse("https://example.com/").unwrap();
|
||||
jar.set_cookie("token=xyz; Max-Age=3600", &url);
|
||||
assert!(jar.get_cookie_header(&url).contains("token=xyz"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_expired_cookie_not_sent() {
|
||||
let jar = CookieJar::new();
|
||||
let url = Url::parse("https://example.com/").unwrap();
|
||||
jar.set_cookie("old=gone; Expires=Thu, 01 Jan 2020 00:00:00 GMT", &url);
|
||||
assert!(jar.get_cookie_header(&url).is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_samesite_parsed() {
|
||||
let jar = CookieJar::new();
|
||||
let url = Url::parse("https://example.com/").unwrap();
|
||||
jar.set_cookie("strict_cookie=val; SameSite=Strict", &url);
|
||||
assert!(jar.get_cookie_header(&url).contains("strict_cookie=val"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_clear_cookies() {
|
||||
let jar = CookieJar::new();
|
||||
let url = Url::parse("https://example.com/").unwrap();
|
||||
jar.set_cookie("a=1", &url);
|
||||
assert!(!jar.get_cookie_header(&url).is_empty());
|
||||
|
||||
jar.clear();
|
||||
assert!(jar.get_cookie_header(&url).is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_set_cookies_from_cdp_preserves_same_site_and_expires() {
|
||||
let jar = CookieJar::new();
|
||||
let future_expiry = std::time::SystemTime::now()
|
||||
.duration_since(std::time::UNIX_EPOCH)
|
||||
.unwrap()
|
||||
.as_secs() as i64
|
||||
+ 3600;
|
||||
jar.set_cookies_from_cdp(vec![CookieInfo {
|
||||
name: "sid".to_string(),
|
||||
value: "abc".to_string(),
|
||||
domain: "example.com".to_string(),
|
||||
path: "/".to_string(),
|
||||
secure: true,
|
||||
http_only: true,
|
||||
same_site: "Strict".to_string(),
|
||||
expires: Some(future_expiry),
|
||||
}]);
|
||||
|
||||
let cookies = jar.get_all_cookies();
|
||||
assert_eq!(cookies.len(), 1);
|
||||
assert_eq!(cookies[0].same_site, "Strict");
|
||||
assert_eq!(cookies[0].expires, Some(future_expiry));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_set_cookies_from_cdp_session_when_expires_none() {
|
||||
let jar = CookieJar::new();
|
||||
jar.set_cookies_from_cdp(vec![CookieInfo {
|
||||
name: "n".to_string(),
|
||||
value: "v".to_string(),
|
||||
domain: "example.com".to_string(),
|
||||
path: "/".to_string(),
|
||||
secure: false,
|
||||
http_only: false,
|
||||
same_site: String::new(),
|
||||
expires: None,
|
||||
}]);
|
||||
let cookies = jar.get_all_cookies();
|
||||
assert_eq!(cookies[0].expires, None);
|
||||
assert_eq!(cookies[0].same_site, DEFAULT_SAME_SITE);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_delete_cookies_filtered_path_mismatch_preserves_cookie() {
|
||||
let jar = CookieJar::new();
|
||||
jar.set_cookies_from_cdp(vec![CookieInfo {
|
||||
name: "sid".to_string(),
|
||||
value: "v".to_string(),
|
||||
domain: "example.com".to_string(),
|
||||
path: "/admin".to_string(),
|
||||
secure: false,
|
||||
http_only: false,
|
||||
same_site: String::new(),
|
||||
expires: None,
|
||||
}]);
|
||||
jar.delete_cookies_filtered("sid", "example.com", Some("/other"));
|
||||
assert_eq!(jar.get_all_cookies().len(), 1);
|
||||
|
||||
jar.delete_cookies_filtered("sid", "example.com", Some("/admin"));
|
||||
assert!(jar.get_all_cookies().is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_delete_cookies_filtered_no_path_deletes_regardless() {
|
||||
let jar = CookieJar::new();
|
||||
jar.set_cookies_from_cdp(vec![CookieInfo {
|
||||
name: "sid".to_string(),
|
||||
value: "v".to_string(),
|
||||
domain: "example.com".to_string(),
|
||||
path: "/admin".to_string(),
|
||||
secure: false,
|
||||
http_only: false,
|
||||
same_site: String::new(),
|
||||
expires: None,
|
||||
}]);
|
||||
jar.delete_cookies_filtered("sid", "example.com", None);
|
||||
assert!(jar.get_all_cookies().is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_set_cookies_from_cdp_expired_does_not_persist() {
|
||||
let jar = CookieJar::new();
|
||||
let now = std::time::SystemTime::now()
|
||||
.duration_since(std::time::UNIX_EPOCH)
|
||||
.unwrap()
|
||||
.as_secs() as i64;
|
||||
jar.set_cookies_from_cdp(vec![CookieInfo {
|
||||
name: "old".to_string(),
|
||||
value: "v".to_string(),
|
||||
domain: "example.com".to_string(),
|
||||
path: "/".to_string(),
|
||||
secure: false,
|
||||
http_only: false,
|
||||
same_site: String::new(),
|
||||
expires: Some(now - 1),
|
||||
}]);
|
||||
let url = Url::parse("https://example.com/").unwrap();
|
||||
assert!(jar.get_cookie_header(&url).is_empty());
|
||||
}
|
||||
#[test]
|
||||
fn test_save_load_roundtrip() {
|
||||
let tmp = tempfile::tempdir().unwrap();
|
||||
let path = tmp.path().join("cookies.json");
|
||||
|
||||
let jar = CookieJar::new();
|
||||
let url = Url::parse("https://example.com/").unwrap();
|
||||
jar.set_cookie("session=abc123; Domain=example.com; Path=/", &url);
|
||||
jar.set_cookie("token=xyz; Secure; HttpOnly", &url);
|
||||
|
||||
jar.save_to_file(&path).unwrap();
|
||||
assert!(path.exists());
|
||||
|
||||
let jar2 = CookieJar::new();
|
||||
let count = jar2.load_from_file(&path).unwrap();
|
||||
assert_eq!(count, 2);
|
||||
|
||||
let header = jar2.get_cookie_header(&url);
|
||||
assert!(header.contains("session=abc123"));
|
||||
assert!(header.contains("token=xyz"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_load_nonexistent_file_returns_zero() {
|
||||
let jar = CookieJar::new();
|
||||
let count = jar
|
||||
.load_from_file(std::path::Path::new("/nonexistent/cookies.json"))
|
||||
.unwrap();
|
||||
assert_eq!(count, 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_domain_matches_subdomain_without_leading_dot() {
|
||||
let jar = CookieJar::new();
|
||||
jar.set_cookies_from_cdp(vec![CookieInfo {
|
||||
name: "session".to_string(),
|
||||
value: "abc".to_string(),
|
||||
domain: "xiaohongshu.com".to_string(),
|
||||
path: "/".to_string(),
|
||||
secure: false,
|
||||
http_only: true,
|
||||
same_site: String::new(),
|
||||
expires: None,
|
||||
}]);
|
||||
let url = Url::parse("https://www.xiaohongshu.com/explore").unwrap();
|
||||
let header = jar.get_cookie_header(&url);
|
||||
assert!(header.contains("session=abc"), "Cookie header was: '{}'", header);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_cookie_from_file_load_then_send_in_request() {
|
||||
// Simulate what happens: load cookies from file → navigate → cookie should be in request
|
||||
use std::io::Write;
|
||||
let tmp = tempfile::tempdir().unwrap();
|
||||
let path = tmp.path().join("cookies.json");
|
||||
|
||||
// Write cookies like we exported from Chrome
|
||||
let cookies = serde_json::json!([
|
||||
{"name": "a1", "value": "testval", "domain": "xiaohongshu.com", "path": "/", "secure": false, "httpOnly": false},
|
||||
{"name": "web_session", "value": "sess123", "domain": "xiaohongshu.com", "path": "/", "secure": false, "httpOnly": true},
|
||||
]);
|
||||
std::fs::write(&path, serde_json::to_string(&cookies).unwrap()).unwrap();
|
||||
|
||||
let jar = CookieJar::new();
|
||||
let count = jar.load_from_file(&path).unwrap();
|
||||
assert_eq!(count, 2, "Should load 2 cookies");
|
||||
|
||||
let url = Url::parse("https://www.xiaohongshu.com/explore").unwrap();
|
||||
let header = jar.get_cookie_header(&url);
|
||||
assert!(header.contains("a1=testval"), "Missing a1 in: '{}'", header);
|
||||
assert!(header.contains("web_session=sess123"), "Missing web_session in: '{}'", header);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn attacker_response_cannot_set_unrelated_victim_domain_cookie() {
|
||||
// GHSA-f22c-8v6q-v6h6: a response from attacker.test must not be able to
|
||||
// plant a cookie scoped to victim.test.
|
||||
let jar = CookieJar::new();
|
||||
let attacker = Url::parse("http://attacker.test/").unwrap();
|
||||
jar.set_cookie("sid=attacker; Domain=victim.test; Path=/", &attacker);
|
||||
|
||||
let victim = Url::parse("http://victim.test/account").unwrap();
|
||||
assert!(
|
||||
!jar.get_cookie_header(&victim).contains("sid=attacker"),
|
||||
"cross-domain cookie leaked to victim: {}",
|
||||
jar.get_cookie_header(&victim)
|
||||
);
|
||||
// The cookie is stored host-only on the attacker origin instead.
|
||||
assert!(jar.get_cookie_header(&attacker).contains("sid=attacker"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn document_cookie_cannot_set_unrelated_victim_domain_cookie() {
|
||||
let jar = CookieJar::new();
|
||||
let attacker = Url::parse("http://attacker.test/").unwrap();
|
||||
jar.set_cookie_from_js("js_sid=attacker; Domain=victim.test; Path=/", &attacker);
|
||||
|
||||
let victim = Url::parse("http://victim.test/account").unwrap();
|
||||
assert!(
|
||||
!jar.get_cookie_header(&victim).contains("js_sid=attacker"),
|
||||
"cross-domain JS cookie leaked to victim: {}",
|
||||
jar.get_cookie_header(&victim)
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn public_suffix_domain_attribute_is_ignored() {
|
||||
let jar = CookieJar::new();
|
||||
let url = Url::parse("http://www.example.com/").unwrap();
|
||||
jar.set_cookie("bad=1; Domain=com; Path=/", &url);
|
||||
// "com" is a public suffix; the cookie must not be scoped to it.
|
||||
let other = Url::parse("http://other.com/").unwrap();
|
||||
assert!(!jar.get_cookie_header(&other).contains("bad=1"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn host_only_cookie_not_sent_to_subdomain() {
|
||||
let jar = CookieJar::new();
|
||||
let www = Url::parse("http://www.example.com/").unwrap();
|
||||
jar.set_cookie("hostonly=1; Path=/", &www); // no Domain attribute -> host-only
|
||||
|
||||
assert!(jar.get_cookie_header(&www).contains("hostonly=1"));
|
||||
let sub = Url::parse("http://sub.www.example.com/").unwrap();
|
||||
assert!(
|
||||
!jar.get_cookie_header(&sub).contains("hostonly=1"),
|
||||
"host-only cookie leaked to subdomain: {}",
|
||||
jar.get_cookie_header(&sub)
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn valid_subdomain_can_set_parent_domain_cookie() {
|
||||
// A subdomain setting Domain=<parent> (a legitimate parent) still works.
|
||||
let jar = CookieJar::new();
|
||||
let www = Url::parse("http://www.example.com/").unwrap();
|
||||
jar.set_cookie("token=1; Domain=example.com; Path=/", &www);
|
||||
|
||||
let apex = Url::parse("http://example.com/").unwrap();
|
||||
assert!(jar.get_cookie_header(&apex).contains("token=1"));
|
||||
let api = Url::parse("http://api.example.com/").unwrap();
|
||||
assert!(jar.get_cookie_header(&api).contains("token=1"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn cookie_path_requires_slash_boundary() {
|
||||
// RFC 6265 5.1.4: a Path=/admin cookie must NOT be sent to a sibling
|
||||
// path like /administrator that merely shares the string prefix. It is
|
||||
// sent to /admin, /admin/, and /admin/x.
|
||||
let jar = CookieJar::new();
|
||||
let admin = Url::parse("https://example.com/admin").unwrap();
|
||||
jar.set_cookie("sess=1; Path=/admin", &admin);
|
||||
|
||||
let sibling = Url::parse("https://example.com/administrator").unwrap();
|
||||
assert!(
|
||||
!jar.get_cookie_header(&sibling).contains("sess=1"),
|
||||
"cookie leaked to sibling path /administrator: {}",
|
||||
jar.get_cookie_header(&sibling)
|
||||
);
|
||||
|
||||
assert!(jar.get_cookie_header(&admin).contains("sess=1"));
|
||||
let exact_slash = Url::parse("https://example.com/admin/").unwrap();
|
||||
assert!(jar.get_cookie_header(&exact_slash).contains("sess=1"));
|
||||
let sub = Url::parse("https://example.com/admin/panel").unwrap();
|
||||
assert!(jar.get_cookie_header(&sub).contains("sess=1"));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,351 @@
|
||||
//! Charset detection and decoding for HTTP response bodies.
|
||||
//!
|
||||
//! Issue #113: obscura used to call `String::from_utf8_lossy` on every
|
||||
//! response body, which silently corrupts every non-UTF-8 page (GBK, Big5,
|
||||
//! Shift-JIS, Windows-125x, EUC-KR, ISO-8859-x). Picking the right decoder
|
||||
//! is required for scraping non-Latin sites at all.
|
||||
//!
|
||||
//! Detection order, mirroring real browsers (HTML5 spec § 8.2.2.4):
|
||||
//! 1. `Content-Type: text/html; charset=...` from the HTTP response header.
|
||||
//! 2. `<meta charset="...">` or `<meta http-equiv="Content-Type" content="text/html; charset=...">`
|
||||
//! sniffed from the first 1024 bytes of the body.
|
||||
//! 3. Default UTF-8.
|
||||
//!
|
||||
//! For non-HTML resources (JS, CSS, JSON), only steps 1 and 3 apply.
|
||||
|
||||
use encoding_rs::{DecoderResult, EncoderResult, Encoding, UTF_8};
|
||||
|
||||
/// WHATWG canonical (lowercased) name for an encoding label, or None if the
|
||||
/// label is not a known encoding. Backs `TextDecoder`'s label validation and
|
||||
/// its `.encoding` property.
|
||||
pub fn label_name(label: &str) -> Option<String> {
|
||||
Encoding::for_label(label.as_bytes()).map(|e| e.name().to_ascii_lowercase())
|
||||
}
|
||||
|
||||
/// Decode `bytes` with an explicit encoding label, with TextDecoder semantics.
|
||||
/// Returns None when the label is unknown, or (when `fatal`) when the input is
|
||||
/// not valid in that encoding. Non-fatal decoding replaces errors with U+FFFD.
|
||||
pub fn decode_with_label(label: &str, bytes: &[u8], fatal: bool, ignore_bom: bool) -> Option<String> {
|
||||
let enc = Encoding::for_label(label.as_bytes())?;
|
||||
let mut dec = if ignore_bom {
|
||||
enc.new_decoder_without_bom_handling()
|
||||
} else {
|
||||
enc.new_decoder()
|
||||
};
|
||||
if fatal {
|
||||
let mut out = String::with_capacity(bytes.len() + 1);
|
||||
let (res, _) = dec.decode_to_string_without_replacement(bytes, &mut out, true);
|
||||
match res {
|
||||
DecoderResult::InputEmpty => Some(out),
|
||||
_ => None,
|
||||
}
|
||||
} else {
|
||||
let mut out = String::with_capacity(bytes.len() * 2 + 1);
|
||||
let _ = dec.decode_to_string(bytes, &mut out, true);
|
||||
Some(out)
|
||||
}
|
||||
}
|
||||
|
||||
/// Decode an HTTP response body. `content_type_header` is the raw header
|
||||
/// value if present (e.g. `text/html; charset=gbk`). For HTML resources,
|
||||
/// the parser also sniffs `<meta charset>` in the first 1KB.
|
||||
pub fn decode_response(bytes: &[u8], content_type_header: Option<&str>) -> String {
|
||||
let (encoding, _) = detect_encoding(bytes, content_type_header);
|
||||
let (cow, _, _) = encoding.decode(bytes);
|
||||
cow.into_owned()
|
||||
}
|
||||
|
||||
/// Like `decode_response`, but also returns the WHATWG canonical name of the
|
||||
/// encoding that was used (e.g. "EUC-JP", "Shift_JIS", "UTF-8"). Callers use
|
||||
/// the name to expose `document.characterSet` and to do document-encoding-aware
|
||||
/// URL query serialization (the WHATWG "encoding override").
|
||||
pub fn decode_response_with_name(
|
||||
bytes: &[u8],
|
||||
content_type_header: Option<&str>,
|
||||
) -> (String, &'static str) {
|
||||
let (encoding, _) = detect_encoding(bytes, content_type_header);
|
||||
let (cow, _, _) = encoding.decode(bytes);
|
||||
(cow.into_owned(), encoding.name())
|
||||
}
|
||||
|
||||
const PCT_HEX: &[u8; 16] = b"0123456789ABCDEF";
|
||||
|
||||
fn push_pct(out: &mut String, b: u8) {
|
||||
out.push('%');
|
||||
out.push(PCT_HEX[(b >> 4) as usize] as char);
|
||||
out.push(PCT_HEX[(b & 0x0F) as usize] as char);
|
||||
}
|
||||
|
||||
/// Append an ASCII `byte` to a URL query string, percent-encoding it when it is
|
||||
/// in the WHATWG query percent-encode set (C0 controls, space, `"`, `#`, `<`,
|
||||
/// `>`, 0x7F), plus `'` for special schemes (the special-query set). ASCII
|
||||
/// delimiters like `=` and `&` are left literal, so structured queries survive.
|
||||
fn push_query_ascii(out: &mut String, b: u8, special: bool) {
|
||||
let must_encode = b <= 0x20
|
||||
|| b == 0x7F
|
||||
|| matches!(b, 0x22 | 0x23 | 0x3C | 0x3E)
|
||||
|| (special && b == 0x27);
|
||||
if must_encode {
|
||||
push_pct(out, b);
|
||||
} else {
|
||||
out.push(b as char);
|
||||
}
|
||||
}
|
||||
|
||||
/// Encode a run of non-ASCII code points to the target charset and percent-
|
||||
/// encode EVERY resulting byte. The bytes serialize a non-ASCII character, so
|
||||
/// all of them are escaped (this is what the WPT legacy-mb encode-href tests
|
||||
/// expect, e.g. Big5 `一` -> `%A4%40` even though the 0x40 trail byte is ASCII).
|
||||
/// Unmappable code points become the literal `%26%23<decimal>%3B` sequence (a
|
||||
/// percent-encoded `&#NNN;` numeric character reference), per the URL spec.
|
||||
fn encode_run_pct(out: &mut String, run: &str, enc: &'static Encoding) {
|
||||
let mut encoder = enc.new_encoder();
|
||||
let mut input = run;
|
||||
let mut buf = [0u8; 256];
|
||||
loop {
|
||||
let (result, read, written) =
|
||||
encoder.encode_from_utf8_without_replacement(input, &mut buf, true);
|
||||
for &b in &buf[..written] {
|
||||
push_pct(out, b);
|
||||
}
|
||||
input = &input[read..];
|
||||
match result {
|
||||
EncoderResult::InputEmpty => break,
|
||||
EncoderResult::OutputFull => continue,
|
||||
EncoderResult::Unmappable(c) => {
|
||||
out.push_str("%26%23");
|
||||
out.push_str(&(c as u32).to_string());
|
||||
out.push_str("%3B");
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// WHATWG URL "percent-encode after encoding" for the query component, using a
|
||||
/// non-UTF-8 document encoding override (`label`). `query` is the already
|
||||
/// UTF-8-percent-decoded query string. ASCII code points use the (special-)
|
||||
/// query percent-encode set so real query delimiters (`=`, `&`) stay literal;
|
||||
/// runs of non-ASCII code points are encoded to the target charset with every
|
||||
/// byte percent-encoded. Returns None when the label is unknown.
|
||||
pub fn url_encode_query(query: &str, label: &str, special: bool) -> Option<String> {
|
||||
let enc = Encoding::for_label(label.as_bytes())?;
|
||||
let mut out = String::with_capacity(query.len() * 3);
|
||||
let mut run_start: Option<usize> = None;
|
||||
for (idx, c) in query.char_indices() {
|
||||
if c.is_ascii() {
|
||||
if let Some(s) = run_start.take() {
|
||||
encode_run_pct(&mut out, &query[s..idx], enc);
|
||||
}
|
||||
push_query_ascii(&mut out, c as u8, special);
|
||||
} else if run_start.is_none() {
|
||||
run_start = Some(idx);
|
||||
}
|
||||
}
|
||||
if let Some(s) = run_start {
|
||||
encode_run_pct(&mut out, &query[s..], enc);
|
||||
}
|
||||
Some(out)
|
||||
}
|
||||
|
||||
/// Same as `decode_response` but skips the `<meta charset>` sniff. Use for
|
||||
/// non-HTML resources where embedded HTML meta tags are not authoritative
|
||||
/// (script and style bodies, JSON, plain text).
|
||||
pub fn decode_non_html(bytes: &[u8], content_type_header: Option<&str>) -> String {
|
||||
let encoding = content_type_header
|
||||
.and_then(charset_from_content_type)
|
||||
.and_then(|name| Encoding::for_label(name.as_bytes()))
|
||||
.unwrap_or(UTF_8);
|
||||
let (cow, _, _) = encoding.decode(bytes);
|
||||
cow.into_owned()
|
||||
}
|
||||
|
||||
/// Resolve the encoding to use for an HTML response, mirroring the HTML5
|
||||
/// detection order. Returns the encoding and a tag describing where it was
|
||||
/// picked from (for logging / tests).
|
||||
pub fn detect_encoding<'a>(
|
||||
bytes: &'a [u8],
|
||||
content_type_header: Option<&str>,
|
||||
) -> (&'static Encoding, &'static str) {
|
||||
if let Some(charset) = content_type_header.and_then(charset_from_content_type) {
|
||||
if let Some(enc) = Encoding::for_label(charset.as_bytes()) {
|
||||
return (enc, "content-type");
|
||||
}
|
||||
}
|
||||
if let Some(enc) = sniff_meta_charset(bytes) {
|
||||
return (enc, "meta-charset");
|
||||
}
|
||||
(UTF_8, "default-utf8")
|
||||
}
|
||||
|
||||
/// Pull the `charset=` parameter out of a Content-Type header value.
|
||||
fn charset_from_content_type(header: &str) -> Option<String> {
|
||||
for part in header.split(';') {
|
||||
let trimmed = part.trim();
|
||||
if let Some(rest) = trimmed.strip_prefix("charset=").or_else(|| trimmed.strip_prefix("Charset=")) {
|
||||
// Strip surrounding quotes if present.
|
||||
let value = rest.trim_matches(|c: char| c == '"' || c == '\'').trim();
|
||||
if !value.is_empty() {
|
||||
return Some(value.to_ascii_lowercase());
|
||||
}
|
||||
}
|
||||
// Some servers send `Content-Type: text/html; CHARSET = gbk`.
|
||||
let lower = trimmed.to_ascii_lowercase();
|
||||
if let Some(rest) = lower.strip_prefix("charset") {
|
||||
let rest = rest.trim_start();
|
||||
if let Some(rest) = rest.strip_prefix('=') {
|
||||
let value = rest.trim().trim_matches(|c: char| c == '"' || c == '\'');
|
||||
if !value.is_empty() {
|
||||
return Some(value.to_ascii_lowercase());
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
None
|
||||
}
|
||||
|
||||
/// Scan the first 1024 bytes for a `<meta charset="...">` or
|
||||
/// `<meta http-equiv="Content-Type" content="...; charset=...">` declaration.
|
||||
/// We only look at ASCII bytes; valid meta-charset declarations are always
|
||||
/// ASCII regardless of the page's actual encoding.
|
||||
fn sniff_meta_charset(bytes: &[u8]) -> Option<&'static Encoding> {
|
||||
let prefix_len = bytes.len().min(1024);
|
||||
let prefix = &bytes[..prefix_len];
|
||||
// Lossy is fine: any meta charset attribute is ASCII, even on a non-UTF-8 page.
|
||||
let s = String::from_utf8_lossy(prefix).to_ascii_lowercase();
|
||||
|
||||
// Look for any `<meta ... charset=...>` pattern in the first 1KB. We
|
||||
// intentionally accept both the modern shorthand (`<meta charset=gbk>`)
|
||||
// and the legacy http-equiv form (`<meta http-equiv="content-type" content="text/html; charset=gbk">`).
|
||||
let mut pos = 0;
|
||||
while let Some(meta_start) = s[pos..].find("<meta") {
|
||||
let abs = pos + meta_start;
|
||||
// Find the closing `>` for this meta tag.
|
||||
let end = s[abs..].find('>').map(|e| abs + e).unwrap_or(s.len());
|
||||
let tag = &s[abs..end];
|
||||
|
||||
if let Some(charset_pos) = tag.find("charset") {
|
||||
let after = &tag[charset_pos + "charset".len()..];
|
||||
let after = after.trim_start();
|
||||
if let Some(eq_rest) = after.strip_prefix('=') {
|
||||
let value = eq_rest
|
||||
.trim_start()
|
||||
.trim_start_matches(|c: char| c == '"' || c == '\'')
|
||||
.split(|c: char| c == '"' || c == '\'' || c == ';' || c.is_whitespace() || c == '/')
|
||||
.next()
|
||||
.unwrap_or("");
|
||||
if !value.is_empty() {
|
||||
if let Some(enc) = Encoding::for_label(value.as_bytes()) {
|
||||
return Some(enc);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pos = end + 1;
|
||||
if pos >= s.len() {
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
None
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn content_type_charset_wins() {
|
||||
let bytes = b"<html><head><meta charset=\"utf-8\"></head><body></body></html>";
|
||||
let (enc, source) = detect_encoding(bytes, Some("text/html; charset=gbk"));
|
||||
assert_eq!(enc.name(), "GBK");
|
||||
assert_eq!(source, "content-type");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn content_type_quoted_charset_is_parsed() {
|
||||
let (enc, _) = detect_encoding(b"", Some("text/html; charset=\"Shift_JIS\""));
|
||||
assert_eq!(enc.name(), "Shift_JIS");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn meta_charset_used_when_header_missing() {
|
||||
let bytes = b"<!doctype html><html><head><meta charset=\"big5\"></head></html>";
|
||||
let (enc, source) = detect_encoding(bytes, None);
|
||||
assert_eq!(enc.name(), "Big5");
|
||||
assert_eq!(source, "meta-charset");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn meta_http_equiv_charset_is_recognized() {
|
||||
let bytes = b"<html><head><meta http-equiv=\"Content-Type\" content=\"text/html; charset=EUC-KR\"></head></html>";
|
||||
let (enc, _) = detect_encoding(bytes, None);
|
||||
assert_eq!(enc.name(), "EUC-KR");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn no_charset_anywhere_falls_back_to_utf8() {
|
||||
let bytes = b"<html><body>hello</body></html>";
|
||||
let (enc, source) = detect_encoding(bytes, None);
|
||||
assert_eq!(enc.name(), "UTF-8");
|
||||
assert_eq!(source, "default-utf8");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn decode_response_gbk_bytes_roundtrip() {
|
||||
// "你好" (ni hao) encoded as GBK = C4 E3 BA C3
|
||||
let bytes: &[u8] = &[0xC4, 0xE3, 0xBA, 0xC3];
|
||||
let s = decode_response(bytes, Some("text/html; charset=gbk"));
|
||||
assert_eq!(s, "你好");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn decode_non_html_skips_meta_sniff() {
|
||||
// A JS body that happens to contain a string `<meta charset="gbk">`
|
||||
// must NOT be decoded as GBK — non-HTML resources only honor the
|
||||
// HTTP header.
|
||||
let bytes = br#"var x = '<meta charset="gbk">'; // not the real charset"#;
|
||||
let s = decode_non_html(bytes, Some("application/javascript"));
|
||||
assert!(s.contains("<meta charset="));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn url_encode_query_eucjp_high_bytes() {
|
||||
// U+8108 (脈) is EUC-JP CC AE; both bytes are above 0x7E so both encode.
|
||||
assert_eq!(url_encode_query("\u{8108}", "euc-jp", true).unwrap(), "%CC%AE");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn url_encode_query_unmappable_becomes_ncr() {
|
||||
// A code point not in shift_jis becomes the percent-encoded &#NNN;.
|
||||
// U+3402 is a CJK ext-A han char not in shift_jis.
|
||||
let got = url_encode_query("\u{3402}", "shift_jis", true).unwrap();
|
||||
assert_eq!(got, "%26%2313314%3B");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn url_encode_query_big5_low_trail_byte_is_escaped() {
|
||||
// U+4E00 (一) is Big5 A4 40; the 0x40 trail byte is ASCII '@' but must
|
||||
// still be percent-encoded because it serializes a non-ASCII char.
|
||||
assert_eq!(url_encode_query("\u{4e00}", "big5", true).unwrap(), "%A4%40");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn url_encode_query_keeps_ascii_structure() {
|
||||
// ASCII delimiters in a real query stay literal (standard query set):
|
||||
// only the non-ASCII value is re-encoded to the target charset.
|
||||
assert_eq!(
|
||||
url_encode_query("a=\u{8108}&b=c", "euc-jp", true).unwrap(),
|
||||
"a=%CC%AE&b=c"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn meta_sniff_only_scans_first_1kb() {
|
||||
let mut bytes = vec![b' '; 2048];
|
||||
bytes.extend_from_slice(b"<meta charset=\"gbk\">");
|
||||
let (enc, _) = detect_encoding(&bytes, None);
|
||||
// Beyond 1KB: ignored, fall back to UTF-8.
|
||||
assert_eq!(enc.name(), "UTF-8");
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,15 @@
|
||||
use std::collections::HashMap;
|
||||
|
||||
use crate::client::{RequestInfo, Response};
|
||||
|
||||
pub enum InterceptAction {
|
||||
Continue,
|
||||
Block,
|
||||
Fulfill(Response),
|
||||
ModifyHeaders(HashMap<String, String>),
|
||||
}
|
||||
|
||||
#[async_trait::async_trait]
|
||||
pub trait RequestInterceptor {
|
||||
async fn intercept(&self, request: &RequestInfo) -> InterceptAction;
|
||||
}
|
||||
@@ -0,0 +1,25 @@
|
||||
pub mod client;
|
||||
pub mod cookies;
|
||||
pub mod encoding;
|
||||
pub mod interceptor;
|
||||
pub mod robots;
|
||||
pub mod blocklist;
|
||||
#[cfg(feature = "stealth")]
|
||||
pub mod wreq_client;
|
||||
|
||||
pub use client::{
|
||||
env_allows_private_network, is_forbidden_ip, ObscuraHttpClient, ObscuraNetError, RequestCallback,
|
||||
RequestInfo, ResourceType, Response, ResponseCallback, SsrfGuardResolver,
|
||||
};
|
||||
pub use cookies::{CookieInfo, CookieJar};
|
||||
pub use encoding::{
|
||||
decode_non_html, decode_response, decode_response_with_name, decode_with_label, label_name,
|
||||
url_encode_query,
|
||||
};
|
||||
pub use robots::RobotsCache;
|
||||
pub use blocklist::is_blocked as is_tracker_blocked;
|
||||
#[cfg(feature = "stealth")]
|
||||
pub use wreq_client::{
|
||||
StealthHttpClient, STEALTH_NAVIGATOR_PLATFORM, STEALTH_UA_PLATFORM,
|
||||
STEALTH_UA_PLATFORM_VERSION, STEALTH_USER_AGENT,
|
||||
};
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,163 @@
|
||||
use std::collections::HashMap;
|
||||
use std::sync::RwLock;
|
||||
|
||||
pub struct RobotsCache {
|
||||
cache: RwLock<HashMap<String, RobotsRules>>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
struct RobotsRules {
|
||||
disallowed: Vec<String>,
|
||||
allowed: Vec<String>,
|
||||
}
|
||||
|
||||
impl RobotsCache {
|
||||
pub fn new() -> Self {
|
||||
RobotsCache {
|
||||
cache: RwLock::new(HashMap::new()),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn parse_and_store(&self, domain: &str, body: &str, our_agent: &str) {
|
||||
let rules = parse_robots_txt(body, our_agent);
|
||||
self.cache.write().unwrap().insert(domain.to_string(), rules);
|
||||
}
|
||||
|
||||
pub fn is_allowed(&self, domain: &str, path: &str) -> bool {
|
||||
let cache = self.cache.read().unwrap();
|
||||
let rules = match cache.get(domain) {
|
||||
Some(r) => r,
|
||||
None => return true,
|
||||
};
|
||||
|
||||
for pattern in &rules.allowed {
|
||||
if path_matches(path, pattern) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
for pattern in &rules.disallowed {
|
||||
if path_matches(path, pattern) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
true
|
||||
}
|
||||
}
|
||||
|
||||
impl Default for RobotsCache {
|
||||
fn default() -> Self {
|
||||
Self::new()
|
||||
}
|
||||
}
|
||||
|
||||
fn parse_robots_txt(body: &str, our_agent: &str) -> RobotsRules {
|
||||
let our_agent_lower = our_agent.to_lowercase();
|
||||
let mut disallowed = Vec::new();
|
||||
let mut allowed = Vec::new();
|
||||
let mut in_matching_section = false;
|
||||
let mut found_specific = false;
|
||||
|
||||
for line in body.lines() {
|
||||
let line = line.trim();
|
||||
if line.is_empty() || line.starts_with('#') {
|
||||
continue;
|
||||
}
|
||||
|
||||
if let Some((key, value)) = line.split_once(':') {
|
||||
let key = key.trim().to_lowercase();
|
||||
let value = value.trim();
|
||||
|
||||
match key.as_str() {
|
||||
"user-agent" => {
|
||||
let agent = value.to_lowercase();
|
||||
in_matching_section = agent == "*"
|
||||
|| our_agent_lower.contains(&agent)
|
||||
|| agent.contains(&our_agent_lower);
|
||||
if agent != "*" && in_matching_section {
|
||||
found_specific = true;
|
||||
}
|
||||
}
|
||||
"disallow" if in_matching_section && !value.is_empty() => {
|
||||
disallowed.push(value.to_string());
|
||||
}
|
||||
"allow" if in_matching_section && !value.is_empty() => {
|
||||
allowed.push(value.to_string());
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if !found_specific {
|
||||
disallowed.clear();
|
||||
allowed.clear();
|
||||
in_matching_section = false;
|
||||
|
||||
for line in body.lines() {
|
||||
let line = line.trim();
|
||||
if line.is_empty() || line.starts_with('#') { continue; }
|
||||
if let Some((key, value)) = line.split_once(':') {
|
||||
let key = key.trim().to_lowercase();
|
||||
let value = value.trim();
|
||||
match key.as_str() {
|
||||
"user-agent" => {
|
||||
in_matching_section = value.trim() == "*";
|
||||
}
|
||||
"disallow" if in_matching_section && !value.is_empty() => {
|
||||
disallowed.push(value.to_string());
|
||||
}
|
||||
"allow" if in_matching_section && !value.is_empty() => {
|
||||
allowed.push(value.to_string());
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
RobotsRules { disallowed, allowed }
|
||||
}
|
||||
|
||||
fn path_matches(path: &str, pattern: &str) -> bool {
|
||||
if pattern.ends_with('*') {
|
||||
path.starts_with(&pattern[..pattern.len() - 1])
|
||||
} else if pattern.ends_with('$') {
|
||||
path == &pattern[..pattern.len() - 1]
|
||||
} else {
|
||||
path.starts_with(pattern)
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn test_parse_basic_robots() {
|
||||
let body = "User-agent: *\nDisallow: /private/\nDisallow: /admin\nAllow: /admin/public\n";
|
||||
let cache = RobotsCache::new();
|
||||
cache.parse_and_store("example.com", body, "Obscura");
|
||||
assert!(cache.is_allowed("example.com", "/"));
|
||||
assert!(cache.is_allowed("example.com", "/page"));
|
||||
assert!(!cache.is_allowed("example.com", "/private/secret"));
|
||||
assert!(!cache.is_allowed("example.com", "/admin"));
|
||||
assert!(cache.is_allowed("example.com", "/admin/public"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_no_rules_means_allowed() {
|
||||
let cache = RobotsCache::new();
|
||||
assert!(cache.is_allowed("unknown.com", "/anything"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_disallow_all() {
|
||||
let body = "User-agent: *\nDisallow: /\n";
|
||||
let cache = RobotsCache::new();
|
||||
cache.parse_and_store("blocked.com", body, "Obscura");
|
||||
assert!(!cache.is_allowed("blocked.com", "/"));
|
||||
assert!(!cache.is_allowed("blocked.com", "/page"));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,247 @@
|
||||
#[cfg(feature = "stealth")]
|
||||
use std::collections::HashMap;
|
||||
#[cfg(feature = "stealth")]
|
||||
use std::error::Error;
|
||||
#[cfg(feature = "stealth")]
|
||||
use std::sync::Arc;
|
||||
#[cfg(feature = "stealth")]
|
||||
use std::time::Duration;
|
||||
|
||||
#[cfg(feature = "stealth")]
|
||||
use tokio::sync::RwLock;
|
||||
#[cfg(feature = "stealth")]
|
||||
use url::Url;
|
||||
|
||||
#[cfg(feature = "stealth")]
|
||||
use crate::cookies::CookieJar;
|
||||
#[cfg(feature = "stealth")]
|
||||
use crate::client::{Response, ObscuraNetError};
|
||||
|
||||
#[cfg(feature = "stealth")]
|
||||
pub const STEALTH_USER_AGENT: &str =
|
||||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/145.0.0.0 Safari/537.36";
|
||||
|
||||
// The wreq emulation (Profile::Chrome145, Platform::Windows) sends this exact
|
||||
// UA and sec-ch-ua-platform "Windows" on the wire. navigator has to report the
|
||||
// same identity, otherwise the TLS/HTTP layer and the JS layer disagree and a
|
||||
// site cross-checks the mismatch as a bot signal.
|
||||
#[cfg(feature = "stealth")]
|
||||
pub const STEALTH_NAVIGATOR_PLATFORM: &str = "Win32";
|
||||
#[cfg(feature = "stealth")]
|
||||
pub const STEALTH_UA_PLATFORM: &str = "Windows";
|
||||
#[cfg(feature = "stealth")]
|
||||
pub const STEALTH_UA_PLATFORM_VERSION: &str = "15.0.0";
|
||||
|
||||
#[cfg(feature = "stealth")]
|
||||
pub struct StealthHttpClient {
|
||||
client: wreq::Client,
|
||||
pub cookie_jar: Arc<CookieJar>,
|
||||
pub extra_headers: RwLock<HashMap<String, String>>,
|
||||
pub in_flight: Arc<std::sync::atomic::AtomicU32>,
|
||||
}
|
||||
|
||||
#[cfg(feature = "stealth")]
|
||||
impl StealthHttpClient {
|
||||
pub fn new(cookie_jar: Arc<CookieJar>) -> Self {
|
||||
Self::with_proxy(cookie_jar, None)
|
||||
}
|
||||
|
||||
pub fn with_proxy(cookie_jar: Arc<CookieJar>, proxy_url: Option<&str>) -> Self {
|
||||
let emulation_opts = wreq_util::Emulation::builder()
|
||||
.profile(wreq_util::Profile::Chrome145)
|
||||
.platform(wreq_util::Platform::Windows)
|
||||
.build();
|
||||
|
||||
let mut builder = wreq::Client::builder()
|
||||
.emulation(emulation_opts)
|
||||
.timeout(Duration::from_secs(30))
|
||||
.redirect(wreq::redirect::Policy::none());
|
||||
|
||||
if let Some(proxy) = proxy_url {
|
||||
if let Ok(p) = wreq::Proxy::all(proxy) {
|
||||
builder = builder.proxy(p);
|
||||
}
|
||||
}
|
||||
|
||||
let client = builder.build().expect("failed to build wreq stealth client");
|
||||
|
||||
StealthHttpClient {
|
||||
client,
|
||||
cookie_jar,
|
||||
extra_headers: RwLock::new(HashMap::new()),
|
||||
in_flight: Arc::new(std::sync::atomic::AtomicU32::new(0)),
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn fetch(&self, url: &Url) -> Result<Response, ObscuraNetError> {
|
||||
let mut current_url = url.clone();
|
||||
|
||||
if let Some(host) = current_url.host_str() {
|
||||
if crate::blocklist::is_blocked(host) {
|
||||
tracing::debug!("Blocked tracker: {}", current_url);
|
||||
return Ok(Response {
|
||||
status: 0,
|
||||
url: current_url,
|
||||
headers: HashMap::new(),
|
||||
body: Vec::new(),
|
||||
redirected_from: Vec::new(),
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
let mut redirects = Vec::new();
|
||||
|
||||
for _ in 0..20 {
|
||||
let mut req = self.client.get(current_url.as_str());
|
||||
|
||||
let cookie_header = self.cookie_jar.get_cookie_header(¤t_url);
|
||||
if !cookie_header.is_empty() {
|
||||
req = req.header("Cookie", &cookie_header);
|
||||
}
|
||||
|
||||
for (k, v) in self.extra_headers.read().await.iter() {
|
||||
req = req.header(k.as_str(), v.as_str());
|
||||
}
|
||||
|
||||
self.in_flight.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
|
||||
let resp = req.send().await.map_err(|e| {
|
||||
self.in_flight.fetch_sub(1, std::sync::atomic::Ordering::Relaxed);
|
||||
ObscuraNetError::Network(format!("{}: {} (source: {:?})", current_url, e, e.source()))
|
||||
})?;
|
||||
self.in_flight.fetch_sub(1, std::sync::atomic::Ordering::Relaxed);
|
||||
|
||||
let status = resp.status();
|
||||
|
||||
for val in resp.headers().get_all("set-cookie") {
|
||||
if let Ok(s) = val.to_str() {
|
||||
self.cookie_jar.set_cookie(s, ¤t_url);
|
||||
}
|
||||
}
|
||||
|
||||
let response_headers: HashMap<String, String> = resp
|
||||
.headers()
|
||||
.iter()
|
||||
.map(|(k, v)| (k.as_str().to_lowercase(), v.to_str().unwrap_or("").to_string()))
|
||||
.collect();
|
||||
|
||||
if status.is_redirection() {
|
||||
if let Some(location) = resp.headers().get("location") {
|
||||
let location_str = location.to_str().map_err(|_| {
|
||||
ObscuraNetError::Network("Invalid redirect Location".into())
|
||||
})?;
|
||||
let next_url = current_url.join(location_str).map_err(|e| {
|
||||
ObscuraNetError::Network(format!("Invalid redirect URL: {}", e))
|
||||
})?;
|
||||
redirects.push(current_url.clone());
|
||||
current_url = next_url;
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
let body = resp.bytes().await.map_err(|e| {
|
||||
ObscuraNetError::Network(format!("Failed to read body: {}", e))
|
||||
})?.to_vec();
|
||||
|
||||
return Ok(Response {
|
||||
url: current_url,
|
||||
status: status.as_u16(),
|
||||
headers: response_headers,
|
||||
body,
|
||||
redirected_from: redirects,
|
||||
});
|
||||
}
|
||||
|
||||
Err(ObscuraNetError::TooManyRedirects(url.to_string()))
|
||||
}
|
||||
|
||||
/// One request with no redirect following, for scripted fetch()/XHR. Reads
|
||||
/// the cookie jar for the Cookie header and stores Set-Cookie back into it,
|
||||
/// so the caller only owns redirect hops and SSRF re-validation. Used in
|
||||
/// stealth mode so JS-level requests carry the same Chrome TLS fingerprint
|
||||
/// and client hints as the main navigation instead of the rustls ClientHello
|
||||
/// that op_fetch_url would otherwise send (which bot managers read as a
|
||||
/// non-browser script and reject, e.g. the AWS WAF challenge verify call).
|
||||
pub async fn send_single(
|
||||
&self,
|
||||
method: &str,
|
||||
url: &Url,
|
||||
headers: &HashMap<String, String>,
|
||||
body: &str,
|
||||
) -> Result<Response, ObscuraNetError> {
|
||||
if let Some(host) = url.host_str() {
|
||||
if crate::blocklist::is_blocked(host) {
|
||||
tracing::debug!("Blocked tracker: {}", url);
|
||||
return Ok(Response {
|
||||
status: 0,
|
||||
url: url.clone(),
|
||||
headers: HashMap::new(),
|
||||
body: Vec::new(),
|
||||
redirected_from: Vec::new(),
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
let req_method = method
|
||||
.parse::<wreq::Method>()
|
||||
.map_err(|e| ObscuraNetError::Network(format!("invalid method '{}': {}", method, e)))?;
|
||||
let mut req = self.client.request(req_method, url.as_str());
|
||||
|
||||
let cookie_header = self.cookie_jar.get_cookie_header(url);
|
||||
if !cookie_header.is_empty() {
|
||||
req = req.header("cookie", &cookie_header);
|
||||
}
|
||||
for (k, v) in self.extra_headers.read().await.iter() {
|
||||
req = req.header(k.as_str(), v.as_str());
|
||||
}
|
||||
for (k, v) in headers.iter() {
|
||||
req = req.header(k.as_str(), v.as_str());
|
||||
}
|
||||
if !body.is_empty() {
|
||||
req = req.body(body.to_string());
|
||||
}
|
||||
|
||||
self.in_flight.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
|
||||
let resp = req.send().await.map_err(|e| {
|
||||
self.in_flight.fetch_sub(1, std::sync::atomic::Ordering::Relaxed);
|
||||
ObscuraNetError::Network(format!("{}: {}", url, e))
|
||||
})?;
|
||||
self.in_flight.fetch_sub(1, std::sync::atomic::Ordering::Relaxed);
|
||||
|
||||
let status = resp.status();
|
||||
for val in resp.headers().get_all("set-cookie") {
|
||||
if let Ok(s) = val.to_str() {
|
||||
self.cookie_jar.set_cookie(s, url);
|
||||
}
|
||||
}
|
||||
let response_headers: HashMap<String, String> = resp
|
||||
.headers()
|
||||
.iter()
|
||||
.map(|(k, v)| (k.as_str().to_lowercase(), v.to_str().unwrap_or("").to_string()))
|
||||
.collect();
|
||||
let resp_body = resp
|
||||
.bytes()
|
||||
.await
|
||||
.map_err(|e| ObscuraNetError::Network(format!("Failed to read body: {}", e)))?
|
||||
.to_vec();
|
||||
|
||||
Ok(Response {
|
||||
url: url.clone(),
|
||||
status: status.as_u16(),
|
||||
headers: response_headers,
|
||||
body: resp_body,
|
||||
redirected_from: Vec::new(),
|
||||
})
|
||||
}
|
||||
|
||||
pub async fn set_extra_headers(&self, headers: HashMap<String, String>) {
|
||||
*self.extra_headers.write().await = headers;
|
||||
}
|
||||
|
||||
pub fn active_requests(&self) -> u32 {
|
||||
self.in_flight.load(std::sync::atomic::Ordering::Relaxed)
|
||||
}
|
||||
|
||||
pub fn is_network_idle(&self) -> bool {
|
||||
self.active_requests() == 0
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,25 @@
|
||||
[package]
|
||||
name = "obscura"
|
||||
version = "0.1.0"
|
||||
edition = "2021"
|
||||
description = "Rust API for the Obscura headless browser"
|
||||
readme = "README.md"
|
||||
repository = "https://github.com/h4ckf0r0day/obscura"
|
||||
license = "Apache-2.0"
|
||||
|
||||
[features]
|
||||
default = ["api"]
|
||||
api = ["obscura-browser", "obscura-net", "tokio"]
|
||||
|
||||
[dependencies]
|
||||
obscura-browser = { path = "../obscura-browser", optional = true }
|
||||
obscura-net = { path = "../obscura-net", optional = true }
|
||||
anyhow = "1"
|
||||
tokio = { version = "1", features = ["rt"], optional = true }
|
||||
thiserror = "1"
|
||||
serde_json = "1"
|
||||
serde = { version = "1", features = ["derive"] }
|
||||
url = "2"
|
||||
|
||||
[dev-dependencies]
|
||||
tokio = { version = "1", features = ["rt-multi-thread", "macros", "sync", "time"] }
|
||||
@@ -0,0 +1,43 @@
|
||||
# obscura
|
||||
|
||||
Embeddable Rust API for the [Obscura](https://github.com/h4ckf0r0day/obscura)
|
||||
headless browser. Drive a real V8 + DOM browser (`Browser`, `Page`, `Element`,
|
||||
`CookieStore`) directly from Rust, with no separate process or CDP round-trips.
|
||||
|
||||
## Install
|
||||
|
||||
This crate is not published to crates.io, so depend on it via git. Building it
|
||||
compiles Obscura from source, including its embedded V8 (`deno_core`), so the
|
||||
first build is large and slow.
|
||||
|
||||
```toml
|
||||
[dependencies]
|
||||
obscura = { git = "https://github.com/h4ckf0r0day/obscura", features = ["api"] }
|
||||
tokio = { version = "1", features = ["rt", "macros"] }
|
||||
anyhow = "1"
|
||||
```
|
||||
|
||||
## Usage
|
||||
|
||||
```rust,no_run
|
||||
use obscura::Browser;
|
||||
use std::time::Duration;
|
||||
|
||||
#[tokio::main]
|
||||
async fn main() -> anyhow::Result<()> {
|
||||
let browser = Browser::builder()
|
||||
.stealth(true)
|
||||
.storage_dir("/tmp/cookies")
|
||||
.build()?;
|
||||
|
||||
let mut page = browser.new_page().await?;
|
||||
page.goto("https://example.com").await?;
|
||||
|
||||
let el = page.wait_for_selector("a", Duration::from_secs(5)).await?;
|
||||
println!("{} -> {:?}", el.text(), el.attribute("href"));
|
||||
|
||||
Ok(())
|
||||
}
|
||||
```
|
||||
|
||||
See `examples/basic.rs` for a runnable version (`cargo run --example basic`).
|
||||
@@ -0,0 +1,22 @@
|
||||
/// Full example: launch, navigate, interact, check cookies.
|
||||
use obscura::Browser;
|
||||
use std::time::Duration;
|
||||
|
||||
#[tokio::main]
|
||||
async fn main() -> anyhow::Result<()> {
|
||||
let browser = Browser::builder()
|
||||
.stealth(true)
|
||||
.storage_dir("/tmp/obscura-api-test")
|
||||
.build()?;
|
||||
|
||||
let mut page = browser.new_page().await?;
|
||||
page.goto("https://example.com").await?;
|
||||
println!("URL: {}", page.url());
|
||||
println!("Content length: {}", page.content().len());
|
||||
|
||||
let el = page.wait_for_selector("a", Duration::from_secs(5)).await?;
|
||||
println!("Link text: {}", el.text());
|
||||
println!("Link href: {:?}", el.attribute("href"));
|
||||
|
||||
Ok(())
|
||||
}
|
||||
@@ -0,0 +1,94 @@
|
||||
use std::sync::Arc;
|
||||
use std::sync::atomic::{AtomicU64, Ordering};
|
||||
|
||||
use obscura_browser::BrowserContext;
|
||||
use obscura_net::CookieJar;
|
||||
|
||||
use crate::config::BrowserConfig;
|
||||
use crate::cookie::CookieStore;
|
||||
use crate::error::Error;
|
||||
use crate::page::Page;
|
||||
|
||||
static NEXT_PAGE_ID: AtomicU64 = AtomicU64::new(1);
|
||||
|
||||
pub struct Browser {
|
||||
context: Arc<BrowserContext>,
|
||||
cookie_jar: Arc<CookieJar>,
|
||||
}
|
||||
|
||||
impl Browser {
|
||||
pub fn new() -> Result<Self, Error> {
|
||||
Self::build(BrowserConfig::default())
|
||||
}
|
||||
|
||||
pub fn build(config: BrowserConfig) -> Result<Self, Error> {
|
||||
let context = if let Some(ref dir) = config.storage_dir {
|
||||
BrowserContext::with_storage_full(
|
||||
"api".to_string(),
|
||||
config.proxy,
|
||||
config.stealth,
|
||||
config.user_agent,
|
||||
Some(dir.clone()),
|
||||
)
|
||||
} else {
|
||||
BrowserContext::with_full_options(
|
||||
"api".to_string(),
|
||||
config.proxy,
|
||||
config.stealth,
|
||||
config.user_agent,
|
||||
)
|
||||
};
|
||||
|
||||
let context = Arc::new(context);
|
||||
let cookie_jar = context.cookie_jar.clone();
|
||||
|
||||
Ok(Browser { context, cookie_jar })
|
||||
}
|
||||
|
||||
pub fn builder() -> BrowserBuilder {
|
||||
BrowserBuilder::default()
|
||||
}
|
||||
|
||||
pub async fn new_page(&self) -> Result<Page, Error> {
|
||||
let id = NEXT_PAGE_ID.fetch_add(1, Ordering::Relaxed);
|
||||
let page = obscura_browser::Page::new(
|
||||
format!("page-{}", id),
|
||||
self.context.clone(),
|
||||
);
|
||||
Ok(Page {
|
||||
inner: page,
|
||||
})
|
||||
}
|
||||
|
||||
/// Access the cookie store for this browser session.
|
||||
pub fn cookies(&self) -> CookieStore {
|
||||
CookieStore::new(self.cookie_jar.clone())
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
pub struct BrowserBuilder {
|
||||
config: BrowserConfig,
|
||||
}
|
||||
|
||||
impl BrowserBuilder {
|
||||
pub fn proxy(mut self, proxy: impl Into<String>) -> Self {
|
||||
self.config.proxy = Some(proxy.into());
|
||||
self
|
||||
}
|
||||
pub fn stealth(mut self, stealth: bool) -> Self {
|
||||
self.config.stealth = stealth;
|
||||
self
|
||||
}
|
||||
pub fn user_agent(mut self, ua: impl Into<String>) -> Self {
|
||||
self.config.user_agent = Some(ua.into());
|
||||
self
|
||||
}
|
||||
pub fn storage_dir(mut self, dir: impl Into<std::path::PathBuf>) -> Self {
|
||||
self.config.storage_dir = Some(dir.into());
|
||||
self
|
||||
}
|
||||
pub fn build(self) -> Result<Browser, Error> {
|
||||
Browser::build(self.config)
|
||||
}
|
||||
}
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user