chore: import upstream snapshot with attribution

.claude/CLAUDE.md

@@ -0,0 +1,105 @@
+# CLAUDE.md
+
+This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository.
+
+## General Rules
+
+Keep changes minimal and scoped. Do not fix unrelated issues, touch unrelated files, or 'clean up' code outside the scope of the current task unless explicitly asked.
+
+## Git Conventions
+
+- Always use `chore:` prefix for non-functional commits (config changes, CI fixes, changelog edits). Only use `fix:` for actual bug fixes in source code. Only use `feat:` for new features.
+- Use `--no-verify` flag with git push when husky/lint-staged hooks fail due to PATH issues in this environment.
+
+## Pre-Commit Checks
+
+Always run `yarn tsc-check-tests` (or the project's type-check command) before committing any TypeScript changes. Never assume type safety — verify it.
+
+## Code Editing Rules
+
+When reviewing or editing code, do NOT remove code (assertions, type casts, etc.) unless you have verified it's safe by running the build/type checker. Never claim code is 'redundant' without evidence.
+
+## Testing
+
+- When fixing bugs, write the test FIRST that reproduces the issue, then implement the fix. Do not implement fixes before having a failing test.
+- In test files, never leave debug artifacts (console.log, debug mode flags, commented-out code). Clean up before committing.
+
+## PRs
+
+When opening PRs, write concise descriptions focused on what changed and why. Avoid boilerplate templates or overly verbose descriptions. Skip the "Test plan" section completely, don't state the obvious (e.g., tests pass or other stuff visible from the CI checks).
+
+## Build & Test Commands
+
+```bash
+# Setup (uses Yarn v4 via Corepack)
+corepack enable
+yarn install
+
+# Build
+yarn build                    # Build all packages (Turbo + TypeScript)
+
+# Test
+yarn test                     # Run all tests (vitest)
+yarn test:full                # Include difficult tests (CRAWLEE_DIFFICULT_TESTS=1)
+yarn vitest run path/to/test.ts    # Run specific test file
+
+# Code Quality
+yarn lint                     # ESLint
+yarn lint:fix                 # ESLint with auto-fix
+yarn format                   # Format with Biome
+yarn tsc-check-tests          # Type-check test files
+```
+
+## Architecture
+
+Crawlee is a **Yarn workspaces monorepo** with Turbo build orchestration. All packages are in `/packages/`.
+
+### Package Hierarchy
+
+```
+@crawlee/types          # Shared TypeScript interfaces
+@crawlee/utils          # Shared utilities
+@crawlee/memory-storage # In-memory storage (default for testing)
+       ↓
+@crawlee/core           # Request, RequestQueue, RequestList, Dataset
+       ↓
+@crawlee/basic          # BasicCrawler (foundation for all crawlers)
+       ↓
+@crawlee/http           # HttpCrawler
+       ↓
+┌──────┴──────┬─────────────┐
+↓             ↓             ↓
+@crawlee/cheerio  @crawlee/jsdom  @crawlee/linkedom
+(HTML parsing variants)
+
+@crawlee/browser-pool   # Browser instance management
+       ↓
+@crawlee/browser        # BrowserCrawler base
+       ↓
+┌──────┴──────┐
+↓             ↓
+@crawlee/playwright  @crawlee/puppeteer
+
+crawlee                 # Meta-package re-exporting most @crawlee/* packages
+```
+
+### Test Location
+
+Tests are in `/test/` at the repo root (not inside packages). E2E tests are in `/test/e2e/`.
+
+## Vitest Notes (vs Jest)
+
+- Mocks are per-test-file (no need for `afterAll` unmocking)
+- Use `vitest.mock()` and `vitest.mocked()` for type casting
+- Module mocking must match import style (default vs named exports)
+- Spies are separate instances - reuse the same spy for multiple operations
+- `vitest.setConfig()` for runtime configuration changes
+- Avoid importing `const enum` from external packages (won't inline like tsc)
+
+## macOS Setup for Proxy Tests
+
+```bash
+sudo ifconfig lo0 alias 127.0.0.2 up
+sudo ifconfig lo0 alias 127.0.0.3 up
+sudo ifconfig lo0 alias 127.0.0.4 up
+```

.editorconfig

@@ -0,0 +1,10 @@
+root = true
+
+[*]
+indent_style = space
+indent_size = 4
+charset = utf-8
+trim_trailing_whitespace = true
+insert_final_newline = true
+end_of_line = lf
+quote_type = single

.github/ISSUE_TEMPLATE/bug_report.yml

@@ -0,0 +1,106 @@
+name: Bug report
+description: Report incorrect or unexpected behavior of a module
+labels: [bug]
+body:
+  - type: markdown
+    attributes:
+      value: |
+        If you have a question, please ask it on [GitHub Discussions](https://github.com/apify/crawlee/discussions) or in our [Discord server](https://discord.gg/jyEM2PRvMU)
+
+  - type: dropdown
+    id: package
+    attributes:
+      label: Which package is this bug report for? If unsure which one to select, leave blank
+      multiple: false
+      options:
+        - "@crawlee/core"
+        - "@crawlee/cheerio (CheerioCrawler)"
+        - "@crawlee/jsdom (JSDOMCrawler)"
+        - "@crawlee/playwright (PlaywrightCrawler)"
+        - "@crawlee/puppeteer (PuppeteerCrawler)"
+        - "@crawlee/basic (BasicCrawler)"
+        - "@crawlee/http (HttpCrawler)"
+        - "@crawlee/browser (BrowserCrawler)"
+        - "@crawlee/utils"
+        - "@crawlee/browser-pool"
+        - "@crawlee/memory-storage"
+    validations:
+      required: false
+
+  - type: textarea
+    id: description
+    attributes:
+      label: Issue description
+      description: |
+        Describe the issue in as much detail as possible.
+
+        Tip: You can attach images or log files by clicking this area to highlight it and then dragging files into it.
+      placeholder: |
+        Steps to reproduce with the code sample below:
+        1. do thing
+        2. call x method
+        3. observe behavior
+        4. see error logs below
+    validations:
+      required: true
+
+  - type: textarea
+    id: codesample
+    attributes:
+      label: Code sample
+      description: Include a reproducible, minimal code sample. This will be automatically formatted into code, so no need for backticks.
+      render: TypeScript
+      placeholder: |
+        // Your code sample should be...
+        // ... Minimal - use as little code as possible that still produces the same problem  (and is understandable)
+        // ... Complete - Provide all parts someone else needs to reproduce your problem
+        // ... Reproducible - Test the code you're about to provide to make sure it reproduces the problem
+
+  - type: input
+    id: package-version
+    attributes:
+      label: Package version
+      description: Which version of the package are you using? Run `npm list <package>` in your project directory and paste the output.
+    validations:
+      required: true
+
+  - type: input
+    id: node-version
+    attributes:
+      label: Node.js version
+      description: |
+        Which version of Node.js are you using? Run `node --version` in your project directory and paste the output.
+        If you are using TypeScript, please include its version (`npm list typescript`) as well.
+      placeholder: Node.js version 16+ is required for Crawlee
+    validations:
+      required: true
+
+  - type: input
+    id: os
+    attributes:
+      label: Operating system
+      description: Which OS does your application run on?
+
+  - type: checkboxes
+    id: apify
+    attributes:
+      label: Apify platform
+      description: Did you also encounter this issue when running your code on Apify's platform? (if unsure what that is, leave unticked)
+      options:
+        - label: Tick me if you encountered this issue on the Apify platform
+
+  - type: input
+    id: next-release
+    attributes:
+      label: I have tested this on the `next` release
+      description: |
+        This issue might already be fixed in a development release (tagged as `next` on npm). This is not required, but it helps us greatly.
+        To install the latest development release run `npm i <package>@next` in your project directory.
+        Run `npm list <package>` and provide the full version that is printed.
+      placeholder: 3.0.5-beta.50
+
+  - type: textarea
+    id: additional-context
+    attributes:
+      label: Other context
+      description: Any other context, screenshots, or file uploads that help us understand your bug report.

.github/ISSUE_TEMPLATE/config.yml

@@ -0,0 +1,12 @@
+blank_issues_enabled: false
+contact_links:
+  - name: GitHub Discussions
+    url: https://github.com/apify/crawlee/discussions
+    about: Got a question? Want to leave some feedback for the Crawlee team? Make a post in GitHub Discussions about it!
+  - name: Discord
+    url: https://discord.gg/jyEM2PRvMU
+    about: Join our Discord server where you can talk with the Crawlee community, as well as the Apify team and get help if you have a question or are unsure of how to do something!
+  - name: Crawlee Documentation
+    url: https://crawlee.dev
+    about: Check out Crawlee's documentation, maybe you'll find your solution there!
+

.github/ISSUE_TEMPLATE/feature_request.yml

@@ -0,0 +1,60 @@
+name: Feature request
+description: Request a new feature in Crawlee
+labels: [feature]
+body:
+  - type: dropdown
+    id: package
+    attributes:
+      label: Which package is the feature request for? If unsure which one to select, leave blank
+      multiple: false
+      options:
+        - "@crawlee/core"
+        - "@crawlee/cheerio (CheerioCrawler)"
+        - "@crawlee/jsdom (JSDOMCrawler)"
+        - "@crawlee/playwright (PlaywrightCrawler)"
+        - "@crawlee/puppeteer (PuppeteerCrawler)"
+        - "@crawlee/basic (BasicCrawler)"
+        - "@crawlee/http (HttpCrawler)"
+        - "@crawlee/browser (BrowserCrawler)"
+        - "@crawlee/utils"
+        - "@crawlee/browser-pool"
+        - "@crawlee/memory-storage"
+    validations:
+      required: false
+
+  - type: textarea
+    id: description
+    attributes:
+      label: Feature
+      description: A clear and concise description of what the problem is, or what feature you want to be implemented.
+      placeholder: I'm always frustrated when..., I would love if there was a shortcut for..., A good addition would be...
+    validations:
+      required: true
+
+  - type: textarea
+    id: motive
+    attributes:
+      label: Motivation
+      description: What made you want to suggest this feature? How would you use it in Crawlee and how will it help you?
+    validations:
+      required: true
+
+  - type: textarea
+    id: solution
+    attributes:
+      label: Ideal solution or implementation, and any additional constraints
+      description: What constraints would we need to ensure? How do you see this feature implemented or used by you?
+    validations:
+      required: true
+
+  - type: textarea
+    id: alternatives
+    attributes:
+      label: Alternative solutions or implementations
+      description: Have you considered any alternative solutions or features that would help you with this issue?
+
+  - type: textarea
+    id: additional-context
+    attributes:
+      label: Other context
+      description: Any other context, screenshots, or file uploads that help us understand your feature request.

.github/workflows/check-pr-title.yml

@@ -0,0 +1,12 @@
+name: Check PR title
+
+on:
+    pull_request_target:
+        types: [ opened, edited, synchronize ]
+
+jobs:
+    check_pr_title:
+        name: 'Check PR title'
+        runs-on: ubuntu-22.04
+        steps:
+            -   uses: apify/actions/pr-title-check@v1.4.0

.github/workflows/deploy-nginx.yml

@@ -0,0 +1,24 @@
+# Triggers deployment of the Nginx reverse proxy configuration for crawlee.dev
+# when the configuration file is updated in this repository.
+name: Deploy Nginx Configuration
+
+on:
+  push:
+    branches:
+      - master
+    paths:
+      - 'website/nginx.conf'
+
+jobs:
+  trigger-deployment:
+    name: Trigger Nginx deployment
+    runs-on: ubuntu-latest
+    steps:
+      - name: Trigger deployment workflow in apify-docs-private
+        run: |
+          gh workflow run deploy-nginx.yml \
+            --repo apify/apify-docs-private \
+            --field deployment=crawlee-web
+          echo "✅ Deployment workflow triggered successfully"
+        env:
+          GITHUB_TOKEN: ${{ secrets.APIFY_SERVICE_ACCOUNT_GITHUB_TOKEN }}

.github/workflows/docs.yml

@@ -0,0 +1,65 @@
+name: docs
+
+on:
+    push:
+        branches:
+            - master
+    workflow_dispatch:
+
+jobs:
+    build:
+        environment:
+            name: github-pages
+        permissions:
+            contents: write
+            pages: write
+            id-token: write
+        runs-on: ubuntu-22.04
+
+        steps:
+            -   uses: actions/checkout@v7
+
+            -   name: Use Node.js 24
+                uses: actions/setup-node@v6
+                with:
+                    node-version: 24
+                    package-manager-cache: false
+
+            -   name: Enable corepack
+                run: |
+                    corepack enable
+                    corepack prepare yarn@stable --activate
+
+            -   name: Build & deploy docs
+                run: |
+                    # install project deps
+                    yarn
+                    # go to website dir
+                    cd website
+                    # install website deps
+                    yarn
+                    # build the docs
+                    yarn build
+                env:
+                    APIFY_SIGNING_TOKEN: ${{ secrets.APIFY_SIGNING_TOKEN }}
+                    SEGMENT_TOKEN: ${{ secrets.SEGMENT_TOKEN }}
+
+            -   name: Set up GitHub Pages
+                uses: actions/configure-pages@v6
+
+            -   name: Upload GitHub Pages artifact
+                uses: actions/upload-pages-artifact@v5
+                with:
+                    path: ./website/build
+
+            -   name: Deploy artifact to GitHub Pages
+                uses: actions/deploy-pages@v5
+
+            -   name: Invalidate CloudFront cache
+                run: |
+                    gh workflow run invalidate-cloudfront.yml \
+                        --repo apify/apify-docs-private \
+                        --field deployment=crawlee-web
+                    echo "✅ CloudFront cache invalidation workflow triggered successfully"
+                env:
+                    GITHUB_TOKEN: ${{ secrets.APIFY_SERVICE_ACCOUNT_GITHUB_TOKEN }}

.github/workflows/publish-to-npm.yml

@@ -0,0 +1,112 @@
+name: Publish to NPM
+
+on:
+    workflow_call:
+        inputs:
+            ref:
+                description: Git ref to publish (branch, tag, or commit SHA)
+                required: true
+                type: string
+            dist-tag:
+                description: NPM dist-tag (prod or next)
+                required: false
+                type: string
+                default: 'prod'
+    workflow_dispatch:
+        inputs:
+            ref:
+                description: Git ref to publish (branch, tag, or commit SHA)
+                required: true
+                type: string
+            dist-tag:
+                description: NPM dist-tag (prod or next)
+                required: false
+                type: string
+                default: 'prod'
+
+concurrency:
+    group: publish-to-npm
+    cancel-in-progress: false
+
+permissions:
+    id-token: write # Required for OIDC
+    contents: write # Required to push updates
+
+jobs:
+    publish_to_npm:
+        name: Publish to NPM (${{ inputs.dist-tag }})
+        runs-on: ubuntu-22.04
+        steps:
+            - uses: actions/checkout@v7
+              with:
+                  ref: ${{ inputs.ref }}
+                  token: ${{ secrets.APIFY_SERVICE_ACCOUNT_GITHUB_TOKEN }}
+                  fetch-depth: 0
+
+            - name: Use Node.js 24
+              uses: actions/setup-node@v6
+              with:
+                  node-version: 24
+                  package-manager-cache: false
+
+            - name: Enable corepack
+              run: |
+                  corepack enable
+                  corepack prepare yarn@stable --activate
+
+            - name: Activate cache for Node.js 24
+              uses: actions/setup-node@v6
+              with:
+                  cache: 'yarn'
+
+            - name: Turbo cache
+              id: turbo-cache
+              uses: actions/cache@v6
+              with:
+                  path: .turbo
+                  key: turbo-${{ github.job }}-${{ github.ref_name }}-${{ github.sha }}
+                  restore-keys: |
+                      turbo-${{ github.job }}-${{ github.ref_name }}-
+
+            - name: Install dependencies
+              run: yarn
+
+            - name: Build packages
+              run: yarn ci:build
+
+            - name: Bump canary versions
+              if: inputs.dist-tag == 'next'
+              run: |
+                    yarn turbo copy --force -- --canary --preid=beta
+
+            - name: Commit changes
+              if: inputs.dist-tag == 'next'
+              uses: EndBug/add-and-commit@v10
+              id: commit
+              with:
+                author_name: Apify Release Bot
+                author_email: noreply@apify.com
+                message: 'chore: bump canary versions [skip ci]'
+                push: false
+
+            - name: Publish to NPM (@latest)
+              if: inputs.dist-tag == 'prod'
+              uses: nick-fields/retry@v4
+              with:
+                  timeout_minutes: 10
+                  max_attempts: 5
+                  retry_wait_seconds: 30
+                  command: git checkout . && yarn publish:prod --yes
+              env:
+                  GH_TOKEN: ${{ secrets.APIFY_SERVICE_ACCOUNT_GITHUB_TOKEN }}
+
+            - name: Publish to NPM (@next)
+              if: inputs.dist-tag == 'next'
+              uses: nick-fields/retry@v4
+              with:
+                  timeout_minutes: 10
+                  max_attempts: 5
+                  retry_wait_seconds: 30
+                  command: git checkout . && yarn publish:next --yes
+              env:
+                  GH_TOKEN: ${{ secrets.APIFY_SERVICE_ACCOUNT_GITHUB_TOKEN }}

.github/workflows/release.yml

@@ -0,0 +1,248 @@
+name: Release @latest
+
+env:
+    YARN_IGNORE_NODE: 1
+    RETRY_TESTS: 1
+
+on:
+    workflow_dispatch:
+        inputs:
+            version:
+                description: The version to bump (if you choose custom, please include it under custom version)
+                required: true
+                default: 'patch'
+                type: choice
+                options:
+                    - 'patch'
+                    - 'minor'
+                    - 'major'
+                    - 'custom'
+            custom_version:
+                description: The custom version to bump to (only for "custom" type)
+                required: false
+                type: string
+                default: ''
+
+concurrency:
+    group: ${{ github.workflow }}-${{ github.ref }}
+    cancel-in-progress: true
+
+jobs:
+    # Test before releasing to ensure we don't release a broken version
+    build_and_test:
+        name: Build & Test
+        if: (!contains(github.event.head_commit.message, '[skip ci]') && !contains(github.event.head_commit.message, 'docs:'))
+        strategy:
+            fail-fast: true
+            matrix:
+                # We don't test on Windows as the tests are flaky
+                os: [ ubuntu-22.04 ]
+                node-version: [ 20, 22, 24 ]
+
+        runs-on: ${{ matrix.os }}
+
+        steps:
+            -   name: Checkout repository
+                uses: actions/checkout@v7
+
+            -   name: Use Node.js ${{ matrix.node-version }}
+                uses: actions/setup-node@v6
+                with:
+                    node-version: ${{ matrix.node-version }}
+                    package-manager-cache: false
+
+            -   name: Enable corepack
+                run: |
+                    corepack enable
+                    corepack prepare yarn@stable --activate
+
+            -   name: Activate cache for Node.js ${{ matrix.node-version }}
+                uses: actions/setup-node@v6
+                with:
+                    cache: 'yarn'
+
+            -   name: Turbo cache
+                id: turbo-cache
+                uses: actions/cache@v6
+                with:
+                    path: .turbo
+                    key: turbo-${{ github.job }}-${{ github.ref_name }}-${{ github.sha }}
+                    restore-keys: |
+                        turbo-${{ github.job }}-${{ github.ref_name }}-
+
+            -   name: Install Dependencies
+                run: yarn
+
+            -   name: Cache Playwright browsers
+                uses: actions/cache@v6
+                with:
+                    path: ~/.cache/ms-playwright
+                    key: playwright-${{ runner.os }}-${{ hashFiles('**/yarn.lock') }}
+                    restore-keys: |
+                        playwright-${{ runner.os }}-
+
+            -   name: Install Playwright browsers
+                timeout-minutes: 10
+                run: yarn playwright install --with-deps
+
+            -   name: Build
+                run: yarn ci:build
+
+            -   name: Tests
+                run: yarn test
+
+    release:
+        name: "Bump Crawlee: ${{ inputs.version }} version (${{ inputs.custom_version || 'n/a' }} custom version)"
+        if: (!contains(github.event.head_commit.message, '[skip ci]') && !contains(github.event.head_commit.message, 'docs:'))
+        needs: build_and_test
+        runs-on: ubuntu-22.04
+
+        steps:
+            -   name: Checkout repository
+                uses: actions/checkout@v7
+                with:
+                    token: ${{ secrets.APIFY_SERVICE_ACCOUNT_GITHUB_TOKEN }}
+                    fetch-depth: 0
+
+            -   name: Use Node.js 24
+                uses: actions/setup-node@v6
+                with:
+                    node-version: 24
+                    package-manager-cache: false
+
+            -   name: Enable corepack
+                run: |
+                    corepack enable
+                    corepack prepare yarn@stable --activate
+
+            -   name: Activate cache for Node.js 24
+                uses: actions/setup-node@v6
+                with:
+                    cache: 'yarn'
+
+            -   name: Turbo cache
+                id: turbo-cache
+                uses: actions/cache@v6
+                with:
+                    path: .turbo
+                    key: turbo-${{ github.job }}-${{ github.ref_name }}-${{ github.sha }}
+                    restore-keys: |
+                        turbo-${{ github.job }}-${{ github.ref_name }}-
+
+            -   name: Install Dependencies
+                run: yarn
+
+            -   name: Build
+                run: yarn ci:build
+
+            -   name: Bump version to custom version
+                if: ${{ github.event.inputs.version == 'custom' && github.event.inputs.custom_version != '' }}
+                run: yarn lerna version ${{ github.event.inputs.custom_version }} --force-publish --yes
+                env:
+                    GH_TOKEN: ${{ secrets.APIFY_SERVICE_ACCOUNT_GITHUB_TOKEN }}
+                    GIT_AUTHOR_NAME: Apify Release Bot
+                    GIT_COMMITTER_NAME: Apify Release Bot
+                    GIT_AUTHOR_EMAIL: noreply@apify.com
+                    GIT_COMMITTER_EMAIL: noreply@apify.com
+
+            -   name: Bump version to ${{ github.event.inputs.version }} version
+                if: ${{ github.event.inputs.version != 'custom' }}
+                run: yarn lerna version ${{ github.event.inputs.version }} --force-publish --yes
+                env:
+                    GH_TOKEN: ${{ secrets.APIFY_SERVICE_ACCOUNT_GITHUB_TOKEN }}
+                    GIT_AUTHOR_NAME: Apify Release Bot
+                    GIT_COMMITTER_NAME: Apify Release Bot
+                    GIT_AUTHOR_EMAIL: noreply@apify.com
+                    GIT_COMMITTER_EMAIL: noreply@apify.com
+
+            -   name: Pin versions in internal dependencies and update lockfile
+                run: |
+                    yarn release:pin-versions
+                    yarn install --no-immutable
+
+            -   name: Commit changes
+                id: commit
+                uses: EndBug/add-and-commit@v10
+                with:
+                  author_name: Apify Release Bot
+                  author_email: noreply@apify.com
+                  message: 'chore(release): update internal dependencies [skip ci]'
+                  pull: '--rebase --autostash'
+
+            -   name: Publish packages
+                uses: apify/actions/execute-workflow@v1.4.0
+                with:
+                    workflow: publish-to-npm.yml
+                    inputs: >
+                      {
+                        "ref": "${{ steps.commit.outputs.commit_long_sha || github.sha }}",
+                        "dist-tag": "prod"
+                      }
+
+            -   name: Trigger Docker image builds
+                uses: peter-evans/repository-dispatch@v4
+                with:
+                    token: ${{ secrets.APIFY_SERVICE_ACCOUNT_GITHUB_TOKEN }}
+                    repository: apify/apify-actor-docker
+                    event-type: build-node-images
+
+    version-docs:
+        needs: release
+        runs-on: ubuntu-22.04
+        if: (github.event.inputs.version == 'minor' || github.event.inputs.version == 'major')
+
+        steps:
+            -   name: Checkout repository
+                uses: actions/checkout@v7
+                with:
+                    ref: ${{ github.event.repository.default_branch }}
+                    token: ${{ secrets.APIFY_SERVICE_ACCOUNT_GITHUB_TOKEN }}
+                    fetch-depth: 0
+
+            -   name: Use Node.js 24
+                uses: actions/setup-node@v6
+                with:
+                    node-version: 24
+                    package-manager-cache: false
+
+            -   name: Install jq
+                run: sudo apt-get install jq
+
+            -   name: Enable corepack
+                run: |
+                    corepack enable
+                    corepack prepare yarn@stable --activate
+
+            -   name: Activate cache for Node.js 24
+                uses: actions/setup-node@v6
+                with:
+                    cache: 'yarn'
+
+            -   name: Install dependencies
+                run: |
+                    # install project deps
+                    yarn
+                    # install website deps
+                    cd website
+                    yarn
+
+            -   name: Snapshot the current version
+                run: |
+                    cd website
+                    VERSION=$(jq -r '.version' ../lerna.json)
+                    if [ -z "$VERSION" ]; then
+                        echo "Error: VERSION is empty. Ensure lerna.json exists and contains a valid .version field."
+                        exit 1
+                    fi
+                    echo "VERSION=$VERSION" >> "$GITHUB_ENV"
+                    MAJOR_MINOR=$(echo $VERSION | cut -d. -f1,2)
+                    yarn docusaurus docs:version $MAJOR_MINOR
+                    yarn docusaurus api:version $MAJOR_MINOR
+
+            -   name: Commit and push the version snapshot
+                uses: EndBug/add-and-commit@v10
+                with:
+                    author_name: Apify Release Bot
+                    author_email: noreply@apify.com
+                    message: 'docs: update docs for ${{ env.VERSION }} version'
+                    pull: '--rebase --autostash'

.github/workflows/test-ci.yml

@@ -0,0 +1,369 @@
+name: Check
+
+on:
+    push:
+        branches: [ master, renovate/** ]
+    pull_request:
+        branches: [ master ]
+
+env:
+    YARN_IGNORE_NODE: 1
+    RETRY_TESTS: 1
+
+concurrency:
+    group: ${{ github.workflow }}-${{ github.ref }}
+    cancel-in-progress: true
+
+jobs:
+    # `yarn install` is done in a separate job and cached to speed up the following jobs.
+    build_and_test:
+        name: Build & Test
+        if: (!contains(github.event.head_commit.message, '[skip ci]') && !contains(github.event.head_commit.message, 'docs:'))
+        runs-on: ${{ matrix.os }}
+
+        strategy:
+            fail-fast: false
+            matrix:
+                # tests on windows are extremely unstable
+                # os: [ ubuntu-22.04, windows-2019 ]
+                os: [ ubuntu-22.04 ]
+                node-version: [ 20, 22, 24 ]
+
+        steps:
+            -   name: Checkout repository
+                uses: actions/checkout@v7
+
+            -   name: Use Node.js ${{ matrix.node-version }}
+                uses: actions/setup-node@v6
+                with:
+                    node-version: ${{ matrix.node-version }}
+                    package-manager-cache: false
+
+            -   name: Enable corepack
+                run: |
+                    corepack enable
+                    corepack prepare yarn@stable --activate
+
+            -   name: Activate cache for Node.js ${{ matrix.node-version }}
+                uses: actions/setup-node@v6
+                with:
+                    cache: 'yarn'
+
+            -   name: Turbo cache
+                id: turbo-cache
+                uses: actions/cache@v6
+                with:
+                    path: .turbo
+                    key: turbo-${{ github.job }}-${{ matrix.node-version }}-${{ github.ref_name }}-${{ github.sha }}
+                    restore-keys: |
+                        turbo-${{ github.job }}-${{ matrix.node-version }}-${{ github.ref_name }}-
+
+            -   name: Install Dependencies
+                run: yarn
+                env:
+                    YARN_IGNORE_NODE: 1
+
+            -   name: Cache Playwright browsers
+                uses: actions/cache@v6
+                with:
+                    path: ~/.cache/ms-playwright
+                    key: playwright-${{ runner.os }}-${{ hashFiles('**/yarn.lock') }}
+                    restore-keys: |
+                        playwright-${{ runner.os }}-
+
+            -   name: Install Playwright browsers
+                timeout-minutes: 10
+                run: yarn playwright install --with-deps
+                env:
+                    YARN_IGNORE_NODE: 1
+
+            -   name: Build
+                run: yarn ci:build
+                env:
+                    YARN_IGNORE_NODE: 1
+
+            -   name: Test TS
+                run: yarn tsc-check-tests
+                env:
+                    YARN_IGNORE_NODE: 1
+
+            -   name: Typecheck documentation examples
+                working-directory: ./docs
+                run: |
+                    yarn
+                    yarn typecheck
+                env:
+                    YARN_IGNORE_NODE: 1
+
+            -   name: Tests
+                run: yarn test
+                env:
+                    YARN_IGNORE_NODE: 1
+
+    docs:
+        name: Docs build
+        if: (!contains(github.event.head_commit.message, '[skip ci]') && github.ref != 'refs/heads/master')
+        runs-on: ubuntu-22.04
+        steps:
+            -   name: Checkout Source code
+                uses: actions/checkout@v7
+
+            -   name: Use Node.js 24
+                uses: actions/setup-node@v6
+                with:
+                    node-version: 24
+                    package-manager-cache: false
+
+            -   name: Enable corepack
+                run: |
+                    corepack enable
+                    corepack prepare yarn@stable --activate
+
+            -   name: Activate cache for Node.js 24
+                uses: actions/setup-node@v6
+                with:
+                    cache: 'yarn'
+
+            -   name: Turbo cache
+                id: turbo-cache
+                uses: actions/cache@v6
+                with:
+                    path: .turbo
+                    key: turbo-${{ github.job }}-${{ github.ref_name }}-${{ github.sha }}
+                    restore-keys: |
+                        turbo-${{ github.job }}-${{ github.ref_name }}-
+
+            -   name: Install Dependencies
+                run: yarn
+
+            -   name: Build & deploy docs
+                run: |
+                    cd website
+                    yarn
+                    yarn build
+                env:
+                    APIFY_SIGNING_TOKEN: ${{ secrets.APIFY_SIGNING_TOKEN }}
+                    SEGMENT_TOKEN: ${{ secrets.SEGMENT_TOKEN }}
+
+            -   name: Install Nginx
+                run: |
+                    sudo apt-get update
+                    sudo apt-get install -y nginx
+
+            -   name: Start Docusaurus server
+                run: |
+                    cd website
+                    nohup yarn docusaurus serve --port 3000 --no-open &
+                    sleep 5
+                    curl -f http://localhost:3000 > /dev/null
+
+            -   name: Start Nginx with project config
+                run: |
+                    PWD_PATH="$(pwd)"
+                    cat > default.conf <<EOF
+                    worker_processes auto;
+                    error_log ${PWD_PATH}/logs/error.log;
+                    pid ${PWD_PATH}/logs/nginx.pid;
+                    events {}
+                    http {
+                        access_log ${PWD_PATH}/logs/access.log;
+                        include ${PWD_PATH}/website/nginx.conf;
+                    }
+                    EOF
+                    sed -i 's|https://apify.github.io/crawlee|http://localhost:3000|g' default.conf
+                    mkdir -p "${PWD_PATH}/logs"
+                    nginx -c "${PWD_PATH}/default.conf"
+                    sleep 1
+
+            -   name: Run header assertions
+                run: |
+                    set -euo pipefail
+                    function assert_header() {
+                        url=$1
+                        header=$2
+                        expected=$3
+                        shift 3
+                        extra_args=("$@")
+                        actual=$(curl -s -D - -o /dev/null "${extra_args[@]}" "$url" | grep -i "^$header" | tr -d '\r' || true)
+                        echo "→ $url → $actual"
+                        echo "$actual" | grep -q "$expected" || (echo "❌ Expected '$expected' in '$header' for $url" && exit 1)
+                    }
+
+                    function assert_status() {
+                        url=$1
+                        expected=$2
+                        shift 2
+                        extra_args=("$@")
+                        actual=$(curl -s -o /dev/null -w "%{http_code}" "${extra_args[@]}" "$url")
+                        echo "→ $url → HTTP $actual"
+                        [ "$actual" = "$expected" ] || (echo "❌ Expected HTTP $expected but got $actual for $url" && exit 1)
+                    }
+
+                    function assert_no_redirect() {
+                        url=$1
+                        shift
+                        extra_args=("$@")
+                        response=$(curl -s -D - -o /dev/null -w "\n%{http_code}" "${extra_args[@]}" "$url" 2>/dev/null)
+                        status=$(echo "$response" | tail -1)
+                        location=$(echo "$response" | grep -i "^location:" | tr -d '\r' || true)
+                        echo "→ $url → HTTP $status ${location:+(${location})}"
+                        if [ "$status" = "301" ] || [ "$status" = "302" ]; then
+                            echo "❌ Got redirect for $url: $location" && exit 1
+                        fi
+                    }
+
+                    echo "🧪 Checking open redirect protection..."
+                    assert_no_redirect "http://localhost:8080///%5Cevil.com/"
+                    assert_no_redirect "http://localhost:8080/%5Cevil.com/"
+                    assert_no_redirect "http://localhost:8080///%5cevil.com/"
+                    assert_no_redirect "http://localhost:8080" --request-target '/\evil.com/'
+                    assert_no_redirect "http://localhost:8080" --request-target '///\evil.com/'
+                    assert_status "http://localhost:8080/js/docs/quick-start/" "302"
+
+                    echo "🧪 Checking Nginx responses... (crawlee JS)"
+                    assert_header "http://localhost:8080/" "Content-Type" "text/html"
+                    assert_header "http://localhost:8080/" "Content-Type" "text/markdown" -H "Accept: text/markdown"
+                    assert_header "http://localhost:8080/js/docs/quick-start" "Content-Type" "text/html"
+                    assert_header "http://localhost:8080/js/docs/quick-start.md" "Content-Type" "text/markdown"
+                    assert_header "http://localhost:8080/js/docs/quick-start" "Content-Type" "text/markdown" -H "Accept: text/markdown"
+                    assert_header "http://localhost:8080/llms.txt" "Content-Type" "text/markdown"
+                    assert_header "http://localhost:8080/llms-full.txt" "Content-Type" "text/markdown"
+
+                    echo "🧪 Checking Nginx responses... (crawlee Python)"
+                    assert_header "http://localhost:8080/python/docs/quick-start" "Content-Type" "text/html"
+                    assert_header "http://localhost:8080/python/docs/quick-start.md" "Content-Type" "text/markdown"
+                    assert_header "http://localhost:8080/python/docs/quick-start" "Content-Type" "text/markdown" -H "Accept: text/markdown"
+                    assert_header "http://localhost:8080/python/llms.txt" "Content-Type" "text/markdown"
+                    assert_header "http://localhost:8080/python/llms-full.txt" "Content-Type" "text/markdown"
+
+                    echo "✅ All Nginx header checks passed."
+
+            -   name: Stop Nginx
+                if: always()
+                run: nginx -c "$(pwd)/default.conf" -s stop
+
+    lint:
+        name: Lint
+        runs-on: ubuntu-22.04
+
+        steps:
+            -   name: Checkout repository
+                uses: actions/checkout@v7
+
+            -   name: Use Node.js 24
+                uses: actions/setup-node@v6
+                with:
+                    node-version: 24
+                    package-manager-cache: false
+
+            -   name: Enable corepack
+                run: |
+                    corepack enable
+                    corepack prepare yarn@stable --activate
+
+            -   name: Activate cache for Node.js 24
+                uses: actions/setup-node@v6
+                with:
+                    cache: 'yarn'
+
+            -   name: Turbo cache
+                id: turbo-cache
+                uses: actions/cache@v6
+                with:
+                    path: .turbo
+                    key: turbo-${{ github.job }}-${{ github.ref_name }}-${{ github.sha }}
+                    restore-keys: |
+                        turbo-${{ github.job }}-${{ github.ref_name }}-
+
+            -   name: Install Dependencies
+                run: yarn
+
+            -   name: ESLint
+                run: yarn lint
+
+            -   name: Biome format
+                run: yarn format:check
+
+    release_next:
+        name: Release @next
+        if: github.event_name == 'push' && contains(github.event.ref, 'master') && (!contains(github.event.head_commit.message, '[skip ci]') && !contains(github.event.head_commit.message, 'docs:'))
+        needs: build_and_test
+        runs-on: ubuntu-22.04
+
+        steps:
+            -   name: Checkout repository
+                uses: actions/checkout@v7
+                with:
+                    token: ${{ secrets.APIFY_SERVICE_ACCOUNT_GITHUB_TOKEN }}
+                    fetch-depth: 0
+
+            -   name: Use Node.js 24
+                uses: actions/setup-node@v6
+                with:
+                    node-version: 24
+                    package-manager-cache: false
+
+            -   name: Enable corepack
+                run: |
+                    corepack enable
+                    corepack prepare yarn@stable --activate
+
+            -   name: Activate cache for Node.js 24
+                uses: actions/setup-node@v6
+                with:
+                    cache: 'yarn'
+
+            -   name: Turbo cache
+                id: turbo-cache
+                uses: actions/cache@v6
+                with:
+                    path: .turbo
+                    key: turbo-${{ github.job }}-${{ github.ref_name }}-${{ github.sha }}
+                    restore-keys: |
+                        turbo-${{ github.job }}-${{ github.ref_name }}-
+
+            -   name: Install Dependencies
+                run: yarn
+
+            -   name: Build
+                run: yarn ci:build
+
+            -   name: Generate changed packages list
+                id: changed-packages
+                run: |
+                    echo "changed_packages=$(node ./node_modules/.bin/lerna changed -p | wc -l | xargs)" | tee -a $GITHUB_OUTPUT
+
+            -   name: Report nothing to release
+                if: steps.changed-packages.outputs.changed_packages == '0'
+                run: echo "Nothing to release"
+
+            -   name: Publish packages
+                if: steps.changed-packages.outputs.changed_packages != '0'
+                uses: apify/actions/execute-workflow@v1.4.0
+                with:
+                    workflow: publish-to-npm.yml
+                    inputs: >
+                      {
+                        "ref": "${{ steps.commit.outputs.commit_long_sha || github.sha }}",
+                        "dist-tag": "next"
+                      }
+
+            -   name: Collect versions for Docker images
+                id: versions
+                run: |
+                    crawlee=`node -p "require('./packages/crawlee/package.json').version"`
+                    echo "crawlee=$crawlee" | tee -a $GITHUB_OUTPUT
+
+            -   name: Trigger Docker image builds
+                uses: peter-evans/repository-dispatch@v4
+                # Trigger next images only if we have something new pushed
+                if: steps.changed-packages.outputs.changed_packages != '0'
+                with:
+                    token: ${{ secrets.APIFY_SERVICE_ACCOUNT_GITHUB_TOKEN }}
+                    repository: apify/apify-actor-docker
+                    event-type: build-node-images
+                    client-payload: >
+                        {
+                          "crawlee_version": "${{ steps.versions.outputs.crawlee }}",
+                          "release_tag": "beta"
+                        }

.github/workflows/test-e2e.yml

@@ -0,0 +1,92 @@
+name: E2E tests
+
+on:
+    workflow_dispatch:
+    schedule:
+        # Runs at 2 am every day
+        -   cron: '0 2 * * *'
+
+env:
+    YARN_IGNORE_NODE: 1
+
+concurrency:
+    group: ${{ github.workflow }}-${{ github.ref }}
+    cancel-in-progress: true
+
+jobs:
+    # NPM install is done in a separate job and cached to speed up the following jobs.
+    build_and_test:
+        name: Build & Test
+        runs-on: ubuntu-22.04
+
+        strategy:
+            fail-fast: false
+            matrix:
+                storage: [ LOCAL, MEMORY, PLATFORM ]
+
+        steps:
+            -   name: Checkout repository
+                uses: actions/checkout@v7
+
+            -   name: Use Node.js 24
+                uses: actions/setup-node@v6
+                with:
+                    node-version: 24
+                    package-manager-cache: false
+
+            -   name: Enable corepack
+                run: |
+                    corepack enable
+                    corepack prepare yarn@stable --activate
+
+            -   name: Activate cache for Node.js 24
+                uses: actions/setup-node@v6
+                with:
+                    cache: 'yarn'
+
+            -   name: Turbo cache
+                id: turbo-cache
+                uses: actions/cache@v6
+                with:
+                    path: .turbo
+                    key: turbo-${{ github.job }}-${{ github.ref_name }}-${{ github.sha }}
+                    restore-keys: |
+                        turbo-${{ github.job }}-${{ github.ref_name }}-
+
+
+            -   name: Setup Apify CLI
+                uses: apify/setup-apify-cli-action@main
+                with:
+                    version: 'beta'
+                    token: ${{ secrets.APIFY_SCRAPER_TESTS_API_TOKEN }}
+
+            -   name: Add Apify secrets for E2E tests
+                run: apify secrets add anthropicApiKey ${{ secrets.ANTHROPIC_API_KEY }}
+
+            -   name: Install Dependencies
+                run: yarn
+
+            -   name: Cache Playwright browsers
+                if: (matrix.storage != 'PLATFORM')
+                uses: actions/cache@v6
+                with:
+                    path: ~/.cache/ms-playwright
+                    key: playwright-${{ runner.os }}-${{ hashFiles('**/yarn.lock') }}
+                    restore-keys: |
+                        playwright-${{ runner.os }}-
+
+            -   name: Install Playwright Dependencies
+                if: (matrix.storage != 'PLATFORM')
+                timeout-minutes: 10
+                run: yarn playwright install --with-deps
+
+            -   name: Build
+                run: yarn ci:build
+
+            -   name: Test with storage ${{ matrix.storage }}
+                run: yarn test:e2e
+                env:
+                    STORAGE_IMPLEMENTATION: ${{ matrix.storage }}
+                    APIFY_HTTPBIN_TOKEN: ${{ secrets.APIFY_HTTPBIN_TOKEN }}
+                    ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
+                    GITHUB_TOKEN: ${{ github.token }}

.github/workflows/update_new_issue.yml

@@ -0,0 +1,25 @@
+name: Update new issue
+
+on:
+    issues:
+        types:
+            - opened
+
+jobs:
+    label_issues:
+        name: Label issues
+        runs-on: ubuntu-22.04
+        permissions:
+            issues: write
+
+        steps:
+            # Add the "t-tooling" label to all new issues
+            -   uses: actions/github-script@v9
+                with:
+                    script: |
+                        github.rest.issues.addLabels({
+                          issue_number: context.issue.number,
+                          owner: context.repo.owner,
+                          repo: context.repo.repo,
+                          labels: ["t-tooling"]
+                        })

.gitignore

@@ -0,0 +1,36 @@
+build
+dist
+build-docs
+node_modules
+*.log
+*.pid
+*.seed
+.DS_Store
+lib
+coverage
+.nyc_output
+logs
+pids
+.idea
+.vscode
+.zed
+package-lock.json
+tmp
+jsconfig.json
+docs/api
+docs/typedefs
+.history
+.docusaurus
+**/*.tsbuildinfo
+apify_storage
+crawlee_storage
+storage
+.turbo
+.npmrc
+test/e2e/**/packages
+
+# we use corepack, no need to commit yarn binary
+.yarn
+
+# Local vitest config overrides
+vitest.config.local.mts

.husky/pre-commit

@@ -0,0 +1 @@
+yarn lint-staged

.yarnrc.yml

@@ -0,0 +1,14 @@
+nodeLinker: node-modules
+enableGlobalCache: true
+npmMinimalAgeGate: 1440
+
+npmPreapprovedPackages:
+    - "@apify/*"
+    - "@crawlee/*"
+    - "apify-client"
+    - "apify"
+    - "crawlee"
+    - "camoufox-js"
+    - "got-scraping"
+    - "impit"
+    - "impit-*"

CONTRIBUTING.md

@@ -0,0 +1,364 @@
+# Contributing
+
+Please note we have a code of conduct, please follow it in all your interactions with the project.
+
+## Submission guidelines
+
+### Submitting an issue
+
+Before you submit an issue, please search the issue tracker, maybe an issue for your problem already exists, and the discussion might inform you of workarounds readily available.
+
+Please provide steps to reproduce if you found a bug or ideally fork the repository and add a failing test that demonstrates what is wrong. This will help us understand and fix the issue faster.
+
+### Submitting a pull request
+
+Before you submit your pull request, consider the following guidelines:
+
+- Search [GitHub](https://github.com/apify/crawlee/pulls) for an open or closed PR that relates to your submission. You don't want to duplicate effort.
+
+- Fork the project and install NPM dependencies.
+
+- Run tests before you start working, to be sure they all pass and your setup is working correctly:
+
+    ```sh
+    yarn test
+    ```
+
+- Be sure to **include appropriate test cases**. Tests help make it clear what the PR is fixing and also make sure the changes won't break over time.
+- Commit your changes using a descriptive commit message that follows defined [commit message conventions](https://docs.google.com/document/d/1QrDFcIiPjSLDn3EL15IJygNPiHORgU1_OOAqWjiDU5Y/edit#heading=h.uyo6cb12dt6w). Adherence to these conventions is necessary because release notes are automatically generated from these messages.
+- Push the code to your forked repository and create a pull request on GitHub.
+- If somebody from project contributors suggests changes:
+    - Make the required updates.
+    - Re-run all test suites to ensure tests are still passing.
+    - Commit them and push. Don't rebase after you get a review, so it is clear what changes you did in the last commit. The PR will be squash merged, so its history is irrelevant.
+
+That's it! Thank you for your contribution!
+
+### Yarn
+
+This project now uses yarn v4 to manage dependencies. You will need to install it, the easiest way is by using `corepack`:
+
+```shell
+corepack enable
+```
+
+### macOS
+
+Our proxy tests use different loopback addresses to ensure traffic correctness.
+In contrary to Windows and Linux, macOS comes with only one loopback address - `127.0.0.1`.
+Therefore it is necessary to run the following once per system startup:
+
+```
+sudo ifconfig lo0 alias 127.0.0.2 up
+sudo ifconfig lo0 alias 127.0.0.3 up
+sudo ifconfig lo0 alias 127.0.0.4 up
+```
+
+### Arch linux
+
+Arch linux is [not officially supported](https://github.com/microsoft/playwright/issues/8100) by Playwright, which causes problems in tests. You need to install dependencies manually:
+
+```
+yay -S libffi7 icu66 libwebp052 flite-unpatched
+sudo ln -s /usr/lib/libpcre.so /usr/lib/libpcre.so.3
+```
+
+## Testing in Crawlee with vitest
+
+There are a few small differences between how testing in jest and vitest works. Mostly, they relate to what to do, and not do anymore.
+
+### Configuration file for tests created in the package they are for
+
+You will need to use this tsconfig.json in the `test` folder in the package (say, if you were adding a test to `packages/core` and there wasn't a `tsconfig.json` file already there)
+
+```json
+{
+    "extends": "../../../tsconfig.json",
+    "include": ["**/*", "../../**/*"],
+    "compilerOptions": {
+        "types": ["vitest/globals"]
+    }
+}
+```
+
+### Mocking modules
+
+Mocks are pretty much the same when it comes to jest vs vitest. One crucial difference is that you no longer need to unmock modules in an afterAll block, as they are mocked per test file.
+
+#### Previous
+
+```ts
+jest.mock('node:os', () => {
+    const original: typeof import('node:os') = jest.requireActual('node:os');
+    return {
+        ...original,
+        platform: () => 'darwin',
+        freemem: jest.fn(),
+    };
+});
+
+afterAll(() => {
+    jest.unmock('node:os');
+});
+```
+
+#### Now
+
+```ts
+vitest.mock('node:os', async (importActual) => {
+    const original = await importActual<typeof import('node:os')>();
+    return {
+        ...original,
+        platform: () => 'darwin',
+        freemem: jest.fn(),
+    };
+});
+```
+
+### Mocking based on imports
+
+Given the following two samples:
+
+#### 1
+
+```ts
+import os from 'node:os';
+
+console.log(os.platform());
+```
+
+#### 2
+
+```ts
+import { platform } from 'node:os';
+
+console.log(platform());
+```
+
+You will need to mock the module based on how you import it in the source code. This means, if you will import the default export, you will need to add a `default` property to the mocked object. Otherwise, you will need to mock the module as is.
+
+So, for example 1:
+
+```ts
+vitest.mock('node:os', async (importActual) => {
+    const original = await importActual<
+        typeof import('node:os') & { default: typeof import('node:os') }
+    >();
+
+    const platformMock = () => 'darwin';
+    const freememMock = vitest.fn();
+
+    return {
+        ...original,
+        platform: platformMock,
+        freemem: freememMock,
+        // Specifically, you'll need to also mock the `default` property of the module, as seen below
+        default: {
+            ...original.default,
+            platform: platformMock,
+            freemem: freememMock,
+        },
+    };
+});
+```
+
+And for example 2:
+
+```ts
+vitest.mock('node:os', async (importActual) => {
+    const original = await importActual<typeof import('node:os')>();
+
+    const platformMock = () => 'darwin';
+    const freememMock = vitest.fn();
+
+    return {
+        ...original,
+        platform: platformMock,
+        freemem: freememMock,
+    };
+});
+```
+
+### Mocked functions
+
+In previous jest code, we had to cast mocked functions as `jest.MockedFunction`. This is _technically_ still needed, but vitest gives us a utility function that casts it for us: `vitest.mocked()`. It doesn't do anything runtime wise, but it helps with type inference.
+
+```ts
+import os from 'node:os';
+
+const mockedPlatform = vitest.mocked(os.platform);
+```
+
+### Resetting spies to original implementation
+
+You no longer need to reset spies to their original implementation. This is done automatically for you via vitest's `restoreMocks` option.
+
+With that said, if you create spies in a `beforeAll`/`beforeEach` hook, you might need to call this at the start of your file: `vitest.setConfig({ restoreMocks: false });`, as otherwise your spies will be reset before your tests run.
+
+### Separate spy instances for methods track their own calls
+
+In previous jest code, you could do something like this:
+
+```ts
+const spy = jest.spyOn(os, 'platform').mockReturnValueOnce('darwin');
+
+expect(os.platform()).toBe('darwin');
+expect(spy).toHaveBeenCalledTimes(1);
+
+const spy2 = jest.spyOn(os, 'platform').mockReturnValueOnce('linux');
+
+expect(os.platform()).toBe('linux');
+expect(spy).toHaveBeenCalledTimes(2);
+```
+
+This is no longer valid in vitest. You will need to re-use the same spy instance.
+
+```ts
+const spy = vitest.spyOn(os, 'platform').mockReturnValueOnce('darwin');
+
+expect(os.platform()).toBe('darwin');
+expect(spy).toHaveBeenCalledTimes(1);
+
+spy.mockReturnValueOnce('linux');
+
+expect(os.platform()).toBe('linux');
+expect(spy).toHaveBeenCalledTimes(2);
+```
+
+## Changing test settings
+
+In jest, we were able to do the following to adjust timeouts at runtime:
+
+```ts
+if (os.platform() === 'win32') {
+    jest.setTimeout(100_000);
+}
+```
+
+In vitest, you need to call the `vitest.setConfig` function instead (and specify what to change):
+
+```ts
+if (os.platform() === 'win32') {
+    vitest.setConfig({
+        testTimeout: 100_000,
+    });
+}
+```
+
+## Hook callbacks
+
+In jest, we were able to call the callback provided in the hooks to signal the hook has executed successfully:
+
+```ts
+beforeAll((done) => {
+    // Do something
+    done();
+});
+```
+
+In vitest, this is no longer provided, but _can_ be substituted with a promise:
+
+```ts
+beforeAll(async () => {
+    await new Promise((resolve) => {
+        // Do something
+        resolve();
+    });
+});
+```
+
+## `const enums`
+
+> [!IMPORTANT]
+> Certain projects, like `puppeteer` declare `const enum`s in their typings. These are enums that do not actually exist at runtime, but enums that `tsc` (which is what we're currently using to compile Crawlee) can inline the values of
+> directly into the compiled code. You should avoid importing `const enums` as `vitest` will not inline them like `tsc` does and will throw an error, unless the enum is also present at runtime (check by importing the module and seeing if it's exported anywhere).
+
+## Testing for class names in stack traces
+
+Some tests may want to check for error stack traces and the presence of class names (a prime example is our tests for logging the stack traces for certain logger levels). In `jest`, you were able to do this:
+
+```ts
+expect(/at BasicCrawler\.requestHandler/.test(stackTrace)).toBe(true);
+```
+
+In `vitest`, at the time of writing this (2023/10/12), class names get an `_` prepended to them. In order to solve this, just add `_?` to your regular expression test (this will match both with and without the `_`).
+
+```ts
+expect(/at _?BasicCrawler\.requestHandler/.test(stackTrace)).toBe(true);
+```
+
+## Code of Conduct
+
+### Our Pledge
+
+In the interest of fostering an open and welcoming environment, we as
+contributors and maintainers pledge to making participation in our project and
+our community a harassment-free experience for everyone, regardless of age, body
+size, disability, ethnicity, gender identity and expression, level of experience,
+nationality, personal appearance, race, religion, or sexual identity and
+orientation.
+
+### Our Standards
+
+Examples of behavior that contributes to creating a positive environment
+include:
+
+-   Using welcoming and inclusive language
+-   Being respectful of differing viewpoints and experiences
+-   Gracefully accepting constructive criticism
+-   Focusing on what is best for the community
+-   Showing empathy towards other community members
+
+Examples of unacceptable behavior by participants include:
+
+-   The use of sexualized language or imagery and unwelcome sexual attention or
+    advances
+-   Trolling, insulting/derogatory comments, and personal or political attacks
+-   Public or private harassment
+-   Publishing others' private information, such as a physical or electronic
+    address, without explicit permission
+-   Other conduct which could reasonably be considered inappropriate in a
+    professional setting
+
+### Our Responsibilities
+
+Project maintainers are responsible for clarifying the standards of acceptable
+behavior and are expected to take appropriate and fair corrective action in
+response to any instances of unacceptable behavior.
+
+Project maintainers have the right and responsibility to remove, edit, or
+reject comments, commits, code, wiki edits, issues, and other contributions
+that are not aligned to this Code of Conduct, or to ban temporarily or
+permanently any contributor for other behaviors that they deem inappropriate,
+threatening, offensive, or harmful.
+
+### Scope
+
+This Code of Conduct applies both within project spaces and in public spaces
+when an individual is representing the project or its community. Examples of
+representing a project or community include using an official project e-mail
+address, posting via an official social media account, or acting as an appointed
+representative at an online or offline event. Representation of a project may be
+further defined and clarified by project maintainers.
+
+### Enforcement
+
+Instances of abusive, harassing, or otherwise unacceptable behavior may be
+reported by contacting the project team at <support@apify.com>. All
+complaints will be reviewed and investigated and will result in a response that
+is deemed necessary and appropriate to the circumstances. The project team is
+obligated to maintain confidentiality with regard to the reporter of an incident.
+Further details of specific enforcement policies may be posted separately.
+
+Project maintainers who do not follow or enforce the Code of Conduct in good
+faith may face temporary or permanent repercussions as determined by other
+members of the project's leadership.
+
+### Attribution
+
+This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4,
+available at [http://contributor-covenant.org/version/1/4][version],
+and from [PurpleBooth](https://gist.github.com/PurpleBooth/b24679402957c63ec426).
+
+[homepage]: http://contributor-covenant.org
+[version]: http://contributor-covenant.org/version/1/4/

LICENSE.md

@@ -0,0 +1,201 @@
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "{}"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright 2018 Apify Technologies s.r.o.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

MIGRATIONS.md

@@ -0,0 +1,440 @@
+# Migration from 2.x.x to 3.0.0
+Check the v3 [upgrading guide](https://crawlee.dev/js/docs/upgrading/upgrading-to-v3).
+
+# Migration from 1.x.x to 2.0.0
+There should be no changes needed apart from upgrading your Node.js version to >= 15.10. If you encounter issues with `cheerio`, [read their CHANGELOG](https://github.com/cheeriojs/cheerio/releases). We bumped it from `rc.3` to `rc.10`.
+
+# Migration from 0.2x.x to 1.0.0
+There are a lot of breaking changes in the v1.0.0 release, but we're confident that
+updating your code will be a matter of minutes. Below, you'll find examples how to do it
+and also short tutorials how to use many of the new features.
+
+If you hadn't yet, we suggest reading the [CHANGELOG](https://github.com/apify/apify-js/blob/master/CHANGELOG.md)
+for a high level view of the changes.
+
+> Many of the new features are made with power users in mind,
+> so don't worry if something looks complicated. You don't need to use it.
+
+<!-- toc -->
+
+- [Installation](#installation)
+- [Running on Apify Platform](#running-on-apify-platform)
+- [Handler arguments are now Crawling Context](#handler-arguments-are-now-crawling-context)
+  * [`Map` of crawling contexts and their IDs](#map-of-crawling-contexts-and-their-ids)
+  * [`autoscaledPool` was moved under `crawlingContext.crawler`](#autoscaledpool-was-moved-under-crawlingcontextcrawler)
+- [Replacement of `PuppeteerPool` with `BrowserPool`](#replacement-of-puppeteerpool-with-browserpool)
+  * [Access to running `BrowserPool`](#access-to-running-browserpool)
+  * [Pages now have IDs](#pages-now-have-ids)
+  * [Configuration and lifecycle hooks](#configuration-and-lifecycle-hooks)
+  * [Introduction of `BrowserController`](#introduction-of-browsercontroller)
+  * [`BrowserPool` methods vs `PuppeteerPool`](#browserpool-methods-vs-puppeteerpool)
+- [Updated `PuppeteerCrawlerOptions`](#updated-puppeteercrawleroptions)
+  * [Removal of `gotoFunction`](#removal-of-gotofunction)
+  * [`launchPuppeteerOptions` => `launchContext`](#launchpuppeteeroptions--launchcontext)
+  * [Removal of `launchPuppeteerFunction`](#removal-of-launchpuppeteerfunction)
+- [Launch functions](#launch-functions)
+  * [Updated arguments](#updated-arguments)
+  * [Custom modules](#custom-modules)
+
+<!-- tocstop -->
+
+## Installation
+Previous versions of the SDK bundled the `puppeteer` package, so you did not have to install
+it. SDK v1 supports also `playwright` and we don't want to force users to install both.
+To install SDK v1 with Puppeteer (same as previous versions), run:
+
+```bash
+npm install apify puppeteer
+```
+
+To install SDK v1 with Playwright run:
+```bash
+npm install apify playwright
+```
+
+> While we tried to add the most important functionality in the initial release,
+> you may find that there are still some utilities or options that are only
+> supported by Puppeteer and not Playwright.
+
+## Running on Apify Platform
+If you want to make use of Playwright on the Apify Platform, you need to use a Docker image
+that supports Playwright. We've created them for you, so head over to the new
+[Docker image guide](https://sdk.apify.com/docs/guides/docker-images) and pick the one
+that best suits your needs.
+
+Note that your `package.json` **MUST** include `puppeteer` and/or `playwright` as dependencies.
+If you don't list them, the libraries will be uninstalled from your `node_modules` folder
+when you build your actors.
+
+## Handler arguments are now Crawling Context
+Previously, arguments of user provided handler functions were provided in separate
+objects. This made it difficult to track values across function invocations.
+
+```js
+const handlePageFunction = async (args1) => {
+    args1.hasOwnProperty('proxyInfo') // true
+}
+
+const handleFailedRequestFunction = async (args2) => {
+    args2.hasOwnProperty('proxyInfo') // false
+}
+
+args1 === args2 // false
+```
+
+This happened because a new arguments object was created for each function.
+With SDK v1 we now have a single object called Crawling Context.
+
+```js
+const handlePageFunction = async (crawlingContext1) => {
+    crawlingContext1.hasOwnProperty('proxyInfo') // true
+}
+
+const handleFailedRequestFunction = async (crawlingContext2) => {
+    crawlingContext2.hasOwnProperty('proxyInfo') // true
+}
+
+// All contexts are the same object.
+crawlingContext1 === crawlingContext2 // true
+```
+
+### `Map` of crawling contexts and their IDs
+Now that all the objects are the same, we can keep track of all running crawling contexts.
+We can do that by working with the new `id` property of `crawlingContext`
+This is useful when you need cross-context access.
+
+```js
+let masterContextId;
+const handlePageFunction = async ({ id, page, request, crawler }) => {
+    if (request.userData.masterPage) {
+        masterContextId = id;
+        // Prepare the master page.
+    } else {
+        const masterContext = crawler.crawlingContexts.get(masterContextId);
+        const masterPage = masterContext.page;
+        const masterRequest = masterContext.request;
+        // Now we can manipulate the master data from another handlePageFunction.
+    }
+}
+```
+
+### `autoscaledPool` was moved under `crawlingContext.crawler`
+To prevent bloat and to make access to certain key objects easier, we exposed a `crawler`
+property on the handle page arguments.
+
+```js
+const handlePageFunction = async ({ request, page, crawler }) => {
+    await crawler.requestQueue.addRequest({ url: 'https://example.com' });
+    await crawler.autoscaledPool.pause();
+}
+```
+
+This also means that some shorthands like `puppeteerPool` or `autoscaledPool` were
+no longer necessary.
+
+```js
+const handlePageFunction = async (crawlingContext) => {
+    crawlingContext.autoscaledPool // does NOT exist anymore
+    crawlingContext.crawler.autoscaledPool // <= this is correct usage
+}
+```
+
+## Replacement of `PuppeteerPool` with `BrowserPool`
+`BrowserPool` was created to extend `PuppeteerPool` with the ability to manage other
+browser automation libraries. The API is similar, but not the same.
+
+### Access to running `BrowserPool`
+Only `PuppeteerCrawler` and `PlaywrightCrawler` use `BrowserPool`. You can access it
+on the `crawler` object.
+
+```js
+const crawler = new Apify.PlaywrightCrawler({
+    handlePageFunction: async ({ page, crawler }) => {
+        crawler.browserPool // <-----
+    }
+});
+
+crawler.browserPool // <-----
+```
+
+### Pages now have IDs
+And they're equal to `crawlingContext.id` which gives you access to full `crawlingContext`
+in hooks. See [Lifecycle hooks](#configuration-and-lifecycle-hooks) below.
+
+```js
+const pageId = browserPool.getPageId
+```
+
+### Configuration and lifecycle hooks
+The most important addition with `BrowserPool` are the
+[lifecycle hooks](https://github.com/apify/browser-pool#browserpool).
+You can access them via `browserPoolOptions` in both crawlers. A full list of `browserPoolOptions`
+can be found in [`browser-pool` readme](https://github.com/apify/browser-pool#new-browserpooloptions).
+
+```js
+const crawler = new Apify.PuppeteerCrawler({
+    browserPoolOptions: {
+        retireBrowserAfterPageCount: 10,
+        preLaunchHooks: [
+            async (pageId, launchContext) => {
+                const { request } = crawler.crawlingContexts.get(pageId);
+                if (request.userData.useHeadful === true) {
+                    launchContext.launchOptions.headless = false;
+                }
+            }
+        ]
+    }
+})
+```
+
+### Introduction of `BrowserController`
+[`BrowserController`](https://github.com/apify/browser-pool#browsercontroller)
+is a class of `browser-pool` that's responsible for browser management.
+Its purpose is to provide a single API for working with both Puppeteer and Playwright browsers.
+It works automatically in the background, but if you ever wanted to close a browser properly,
+you should use a `browserController` to do it. You can find it in the handle page arguments.
+
+```js
+const handlePageFunction = async ({ page, browserController }) => {
+    // Wrong usage. Could backfire because it bypasses BrowserPool.
+    page.browser().close();
+
+    // Correct usage. Allows graceful shutdown.
+    browserController.close();
+
+    const cookies = [/* some cookie objects */];
+    // Wrong usage. Will only work in Puppeteer and not Playwright.
+    page.setCookies(...cookies);
+
+    // Correct usage. Will work in both.
+    browserController.setCookies(page, cookies);
+}
+```
+
+The `BrowserController` also includes important information about the browser, such as
+the context it was launched with. This was difficult to do before SDK v1.
+
+```js
+const handlePageFunction = async ({ browserController }) => {
+    // Information about the proxy used by the browser
+    browserController.launchContext.proxyInfo
+
+    // Session used by the browser
+    browserController.launchContext.session
+}
+```
+
+### `BrowserPool` methods vs `PuppeteerPool`
+Some functions were removed (in line with earlier deprecations), and some were changed a bit:
+
+```js
+// OLD
+puppeteerPool.recyclePage(page);
+
+// NEW
+page.close();
+```
+
+```js
+// OLD
+puppeteerPool.retire(page.browser());
+
+// NEW
+browserPool.retireBrowserByPage(page);
+```
+
+```js
+// OLD
+puppeteerPool.serveLiveViewSnapshot();
+
+// NEW
+// There's no LiveView in BrowserPool
+```
+
+## Updated `PuppeteerCrawlerOptions`
+To keep `PuppeteerCrawler` and `PlaywrightCrawler` consistent, we updated the options.
+
+### Removal of `gotoFunction`
+The concept of a configurable `gotoFunction` is not ideal. Especially since we use a modified
+`gotoExtended`. Users have to know this when they override `gotoFunction` if they want to
+extend default behavior. We decided to replace `gotoFunction` with `preNavigationHooks` and
+`postNavigationHooks`.
+
+The following example illustrates how `gotoFunction` makes things complicated.
+```js
+const gotoFunction = async ({ request, page }) => {
+    // pre-processing
+    await makePageStealthy(page);
+
+    // Have to remember how to do this:
+    const response = gotoExtended(page, request, {/* have to remember the defaults */});
+
+    // post-processing
+    await page.evaluate(() => {
+        window.foo = 'bar';
+    });
+
+    // Must not forget!
+    return response;
+}
+
+const crawler = new Apify.PuppeteerCrawler({
+    gotoFunction,
+    // ...
+})
+```
+
+With `preNavigationHooks` and `postNavigationHooks` it's much easier. `preNavigationHooks`
+are called with two arguments: `crawlingContext` and `gotoOptions`. `postNavigationHooks`
+are called only with `crawlingContext`.
+
+```js
+const preNavigationHooks = [
+    async ({ page }) => makePageStealthy(page)
+];
+
+const postNavigationHooks = [
+    async ({ page }) => page.evaluate(() => {
+        window.foo = 'bar'
+    })
+]
+
+const crawler = new Apify.PuppeteerCrawler({
+    preNavigationHooks,
+    postNavigationHooks,
+    // ...
+})
+```
+
+### `launchPuppeteerOptions` => `launchContext`
+Those were always a point of confusion because they merged custom Apify options with
+`launchOptions` of Puppeteer.
+
+```js
+const launchPuppeteerOptions = {
+    useChrome: true, // Apify option
+    headless: false, // Puppeteer option
+}
+```
+
+Use the new `launchContext` object, which explicitly defines `launchOptions`.
+`launchPuppeteerOptions` were removed.
+
+```js
+const crawler = new Apify.PuppeteerCrawler({
+    launchContext: {
+        useChrome: true, // Apify option
+        launchOptions: {
+            headless: false // Puppeteer option
+        }
+    }
+})
+```
+
+> LaunchContext is also a type of [`browser-pool`](https://github.com/apify/browser-pool) and
+> the structure is exactly the same there. SDK only adds extra options.
+
+### Removal of `launchPuppeteerFunction`
+`browser-pool` introduces the idea of [lifecycle hooks](https://github.com/apify/browser-pool#browserpool),
+which are functions that are executed when a certain event in the browser lifecycle happens.
+
+```js
+const launchPuppeteerFunction = async (launchPuppeteerOptions) => {
+    if (someVariable === 'chrome') {
+        launchPuppeteerOptions.useChrome = true;
+    }
+    return Apify.launchPuppeteer(launchPuppeteerOptions);
+}
+
+const crawler = new Apify.PuppeteerCrawler({
+    launchPuppeteerFunction,
+    // ...
+})
+```
+
+Now you can recreate the same functionality with a `preLaunchHook`:
+
+```js
+const maybeLaunchChrome = (pageId, launchContext) => {
+    if (someVariable === 'chrome') {
+        launchContext.useChrome = true;
+    }
+}
+
+const crawler = new Apify.PuppeteerCrawler({
+    browserPoolOptions: {
+        preLaunchHooks: [maybeLaunchChrome]
+    },
+    // ...
+})
+```
+
+This is better in multiple ways. It is consistent across both Puppeteer and Playwright.
+It allows you to easily construct your browsers with pre-defined behavior:
+
+```js
+const preLaunchHooks = [
+    maybeLaunchChrome,
+    useHeadfulIfNeeded,
+    injectNewFingerprint,
+]
+```
+
+And thanks to the addition of [`crawler.crawlingContexts`](#handler-arguments-are-now-crawling-context)
+the functions also have access to the `crawlingContext` of the `request` that triggered the launch.
+
+```js
+const preLaunchHooks = [
+    async function maybeLaunchChrome(pageId, launchContext) {
+        const { request } = crawler.crawlingContexts.get(pageId);
+        if (request.userData.useHeadful === true) {
+            launchContext.launchOptions.headless = false;
+        }
+    }
+]
+```
+
+## Launch functions
+In addition to `Apify.launchPuppeteer()` we now also have `Apify.launchPlaywright()`.
+
+### Updated arguments
+We [updated the launch options object](#launchpuppeteeroptions--launchcontext) because
+it was a frequent source of confusion.
+
+```js
+// OLD
+await Apify.launchPuppeteer({
+    useChrome: true,
+    headless: true,
+})
+
+// NEW
+await Apify.launchPuppeteer({
+    useChrome: true,
+    launchOptions: {
+        headless: true,
+    }
+})
+```
+
+### Custom modules
+`Apify.launchPuppeteer` already supported the `puppeteerModule` option. With Playwright,
+we normalized the name to `launcher` because the `playwright` module itself does not
+launch browsers.
+
+```js
+const puppeteer = require('puppeteer');
+const playwright = require('playwright');
+
+await Apify.launchPuppeteer();
+// Is the same as:
+await Apify.launchPuppeteer({
+    launcher: puppeteer
+})
+
+await Apify.launchPlaywright();
+// Is the same as:
+await Apify.launchPlaywright({
+    launcher: playwright.chromium
+})
+```

README.md

@@ -0,0 +1,153 @@
+<h1 align="center">
+    <a href="https://crawlee.dev">
+        <picture>
+          <source media="(prefers-color-scheme: dark)" srcset="https://raw.githubusercontent.com/apify/crawlee/master/website/static/img/crawlee-dark.svg?sanitize=true">
+          <img alt="Crawlee" src="https://raw.githubusercontent.com/apify/crawlee/master/website/static/img/crawlee-light.svg?sanitize=true" width="500">
+        </picture>
+    </a>
+    <br>
+    <small>A web scraping and browser automation library</small>
+</h1>
+
+<p align=center>
+    <a href="https://trendshift.io/repositories/5179" target="_blank"><img src="https://trendshift.io/api/badge/repositories/5179" alt="apify%2Fcrawlee | Trendshift" style="width: 250px; height: 55px;" width="250" height="55"/></a>
+</p>
+
+<p align=center>
+    <a href="https://www.npmjs.com/package/@crawlee/core" rel="nofollow"><img src="https://img.shields.io/npm/v/@crawlee/core.svg" alt="NPM latest version" data-canonical-src="https://img.shields.io/npm/v/@crawlee/core/next.svg" style="max-width: 100%;"></a>
+    <a href="https://www.npmjs.com/package/@crawlee/core" rel="nofollow"><img src="https://img.shields.io/npm/dm/@crawlee/core.svg" alt="Downloads" data-canonical-src="https://img.shields.io/npm/dm/@crawlee/core.svg" style="max-width: 100%;"></a>
+    <a href="https://discord.gg/jyEM2PRvMU" rel="nofollow"><img src="https://img.shields.io/discord/801163717915574323?label=discord" alt="Chat on discord" data-canonical-src="https://img.shields.io/discord/801163717915574323?label=discord" style="max-width: 100%;"></a>
+    <a href="https://github.com/apify/crawlee/actions/workflows/test-ci.yml"><img src="https://github.com/apify/crawlee/actions/workflows/test-ci.yml/badge.svg?branch=master" alt="Build Status" style="max-width: 100%;"></a>
+</p>
+
+Crawlee covers your crawling and scraping end-to-end and **helps you build reliable scrapers. Fast.**
+
+Your crawlers will appear human-like and fly under the radar of modern bot protections even with the default configuration. Crawlee gives you the tools to crawl the web for links, scrape data, and store it to disk or cloud while staying configurable to suit your project's needs.
+
+Crawlee is available as the [`crawlee`](https://www.npmjs.com/package/crawlee) NPM package.
+
+> 👉 **View full documentation, guides and examples on the [Crawlee project website](https://crawlee.dev)** 👈
+
+> Do you prefer 🐍 Python instead of JavaScript? [👉 Checkout Crawlee for Python 👈](https://github.com/apify/crawlee-python).
+
+## Installation
+
+We recommend visiting the [Introduction tutorial](https://crawlee.dev/js/docs/introduction) in Crawlee documentation for more information.
+
+> Crawlee requires **Node.js 16 or higher**.
+
+### With Crawlee CLI
+
+The fastest way to try Crawlee out is to use the **Crawlee CLI** and choose the **Getting started example**. The CLI will install all the necessary dependencies and add boilerplate code for you to play with.
+
+```bash
+npx crawlee create my-crawler
+```
+
+```bash
+cd my-crawler
+npm start
+```
+
+### Manual installation
+If you prefer adding Crawlee **into your own project**, try the example below. Because it uses `PlaywrightCrawler` we also need to install [Playwright](https://playwright.dev). It's not bundled with Crawlee to reduce install size.
+
+```bash
+npm install crawlee playwright
+```
+
+```js
+import { PlaywrightCrawler, Dataset } from 'crawlee';
+
+// PlaywrightCrawler crawls the web using a headless
+// browser controlled by the Playwright library.
+const crawler = new PlaywrightCrawler({
+    // Use the requestHandler to process each of the crawled pages.
+    async requestHandler({ request, page, enqueueLinks, log }) {
+        const title = await page.title();
+        log.info(`Title of ${request.loadedUrl} is '${title}'`);
+
+        // Save results as JSON to ./storage/datasets/default
+        await Dataset.pushData({ title, url: request.loadedUrl });
+
+        // Extract links from the current page
+        // and add them to the crawling queue.
+        await enqueueLinks();
+    },
+    // Uncomment this option to see the browser window.
+    // headless: false,
+});
+
+// Add first URL to the queue and start the crawl.
+await crawler.run(['https://crawlee.dev']);
+```
+
+By default, Crawlee stores data to `./storage` in the current working directory. You can override this directory via Crawlee configuration. For details, see [Configuration guide](https://crawlee.dev/js/docs/guides/configuration), [Request storage](https://crawlee.dev/js/docs/guides/request-storage) and [Result storage](https://crawlee.dev/js/docs/guides/result-storage).
+
+### Installing pre-release versions
+
+We provide automated beta builds for every merged code change in Crawlee. You can find them in the npm [list of releases](https://www.npmjs.com/package/crawlee?activeTab=versions). If you want to test new features or bug fixes before we release them, feel free to install a beta build like this:
+
+```bash
+npm install crawlee@next
+```
+
+If you also use the [Apify SDK](https://github.com/apify/apify-sdk-js), you need to specify dependency overrides in your `package.json` file so that you don't end up with multiple versions of Crawlee installed:
+
+```json
+{
+    "overrides": {
+       "apify": {
+           "@crawlee/core": "$crawlee",
+           "@crawlee/types": "$crawlee",
+           "@crawlee/utils": "$crawlee"
+       }
+    }
+}
+```
+
+## 🛠 Features
+
+- Single interface for **HTTP and headless browser** crawling
+- Persistent **queue** for URLs to crawl (breadth & depth first)
+- Pluggable **storage** of both tabular data and files
+- Automatic **scaling** with available system resources
+- Integrated **proxy rotation** and session management
+- Lifecycles customizable with **hooks**
+- **CLI** to bootstrap your projects
+- Configurable **routing**, **error handling** and **retries**
+- **Dockerfiles** ready to deploy
+- Written in **TypeScript** with generics
+
+### 👾 HTTP crawling
+
+- Zero config **HTTP2 support**, even for proxies
+- Automatic generation of **browser-like headers**
+- Replication of browser **TLS fingerprints**
+- Integrated fast **HTML parsers**. Cheerio and JSDOM
+- Yes, you can scrape **JSON APIs** as well
+
+### 💻 Real browser crawling
+
+- JavaScript **rendering** and **screenshots**
+- **Headless** and **headful** support
+- Zero-config generation of **human-like fingerprints**
+- Automatic **browser management**
+- Use **Playwright** and **Puppeteer** with the same interface
+- **Chrome**, **Firefox**, **Webkit** and many others
+
+## Usage on the Apify platform
+
+Crawlee is open-source and runs anywhere, but since it's developed by [Apify](https://apify.com), it's easy to set up on the Apify platform and run in the cloud. Visit the [Apify SDK website](https://sdk.apify.com) to learn more about deploying Crawlee to the Apify platform.
+
+## Support
+
+If you find any bug or issue with Crawlee, please [submit an issue on GitHub](https://github.com/apify/crawlee/issues). For questions, you can ask on [Stack Overflow](https://stackoverflow.com/questions/tagged/apify), in GitHub Discussions or you can join our [Discord server](https://discord.com/invite/jyEM2PRvMU).
+
+## Contributing
+
+Your code contributions are welcome, and you'll be praised to eternity! If you have any ideas for improvements, either submit an issue or create a pull request. For contribution guidelines and the code of conduct, see [CONTRIBUTING.md](https://github.com/apify/crawlee/blob/master/CONTRIBUTING.md).
+
+## License
+
+This project is licensed under the Apache License 2.0 - see the [LICENSE.md](https://github.com/apify/crawlee/blob/master/LICENSE.md) file for details.

README.wehub.md

@@ -0,0 +1,7 @@
+# WeHub 来源说明
+
+- 原始项目：`apify/crawlee`
+- 原始仓库：https://github.com/apify/crawlee
+- 导入方式：上游默认分支的最新快照
+- 原作者、版权和许可证信息以原始仓库及本仓库 LICENSE 为准
+- 本文件仅用于记录来源，不代表 WeHub 是原项目作者

RELEASE.md

@@ -0,0 +1,54 @@
+# How to release new versions of Apify SDK
+Release of new versions is managed by GitHub Actions. On pushes to the `master` branch, prerelease versions
+are automatically produced. Latest releases are triggered manually through the GitHub release tool.
+After creating a release there, Actions will automatically produce a latest version of the package.
+
+## TLDR;
+- To **NOT** release anything on a push to `master`, add `[skip ci]` to your commit message.
+- To release `beta`, just push to `master`. If it breaks with a `Version already exists error` increment version
+  in `package.json` and push again.
+- To release `latest`, go to releases on GitHub, draft and publish a release. If you don't know how, read below.
+
+## Prerelease (beta) versions
+On each push to the `master` branch, a new prerelease version is automatically built and published
+by GitHub Actions. To skip the process, add `[skip ci]` to your commit message.
+
+### Release process
+1. Actions build is triggered by a push to `master` (typically a merge of a PR).
+2. Actions lint the source code and run tests in Node.js 10, 12 and 14.
+3. If all is well, a new prerelease version is published to NPM (`${VERSION}-beta.${COUNTER}`),
+   where `VERSION` is the version in package.json and `COUNTER` is a zero based index of existing prereleases.
+   Example: `0.15.1-beta.3`.
+4. The package is tagged with the `beta` NPM tag and a Git tag is associated with the triggering commit.
+5. A build of Apify docker images is triggered that updates the `beta` packages to use the newly published package.
+6. All done and ready to use.
+
+### Updating a release version
+When releasing breaking changes, new features or for any other reason that requires a version bump,
+manually increment the version in the `package.json` file. Such as from `0.14.15` to `0.15.0`.
+This will automatically trigger a prerelease build with the `0.15.0-beta.0` version.
+
+### Existing versions
+Actions will not allow you to publish a prerelease of a version that's already published. For example,
+if version `0.14.15` already exists on NPM, you can no longer release a `0.14.15-beta.0` version.
+
+## Latest release
+To trigger a latest release, go to the GitHub release tool (select `releases` under `<> Code`).
+There, draft a new release, fill the form (see below) and hit `Publish release`.
+Actions will automatically release the latest version of the package.
+
+### How to fill the form
+- The version tag should be in the format `v${VERSION}` where `VERSION` is the version from `package.json`.
+  Such as `v0.15.0` or `v0.16.17`.
+- The target will typically be `master`, but you can also release any previous commit by selecting it or searching
+  for it by ID. This is useful when there have been some changes in master from the latest prerelease and you'd
+  like to release an older prerelease version as latest. You can find the commit ID easily by searching for the
+  prerelease tag.
+- The title should be the same as the version tag.
+- Typically just adding changelog to the release would be fine, but feel free to add extra information.
+
+### Release process
+Similarly to the prerelease, the latest release process:
+1. Triggers build, lints, runs tests.
+2. Publishes new package to NPM with the `latest` tag and the version from package.json. Such as `0.15.1`.
+3. A build and deploy of Apify docker images is triggered with the `latest` tag.

biome.json

@@ -0,0 +1,28 @@
+{
+	"formatter": {
+		"includes": [
+			"**",
+			"!**/website/**",
+			"!**/packages/**/*/dist/**",
+			"!**/package.json",
+			"!**/lerna.json",
+			"!**/scripts/actions/docker-images/state.json"
+		],
+		"formatWithErrors": true
+	},
+	"javascript": {
+		"formatter": {
+			"quoteStyle": "single",
+			"semicolons": "always",
+			"trailingCommas": "all",
+			"lineWidth": 120,
+			"indentStyle": "space",
+			"indentWidth": 4,
+			"quoteProperties": "preserve",
+			"lineEnding": "lf"
+		}
+	},
+	"linter": {
+		"enabled": false
+	}
+}

docs/deployment/apify_platform.mdx

@@ -0,0 +1,305 @@
+---
+id: apify-platform
+title: Apify Platform
+description: Apify platform - large-scale and high-performance web scraping
+---
+
+import ApiLink from '@site/src/components/ApiLink';
+
+import Tabs from '@theme/Tabs';
+import TabItem from '@theme/TabItem';
+import CodeBlock from '@theme/CodeBlock';
+
+import MainSource from '!!raw-loader!./apify_platform_main.ts';
+import InitExitSource from '!!raw-loader!./apify_platform_init_exit.ts';
+
+Apify is a [platform](https://apify.com) built to serve large-scale and high-performance web scraping
+and automation needs. It provides easy access to [compute instances (Actors)](#what-is-an-actor),
+convenient [request](../guides/request-storage) and [result](../guides/result-storage) storages, [proxies](../guides/proxy-management),
+[scheduling](https://docs.apify.com/scheduler), [webhooks](https://docs.apify.com/webhooks)
+and [more](https://docs.apify.com/), accessible through a [web interface](https://console.apify.com)
+or an [API](https://docs.apify.com/api).
+
+While we think that the Apify platform is super cool, and it's definitely worth signing up for a
+[free account](https://console.apify.com/sign-up), **Crawlee is and will always be open source**,
+runnable locally or on any cloud infrastructure.
+
+:::note
+
+We do not test Crawlee in other cloud environments such as Lambda or on specific
+architectures such as Raspberry PI. We strive to make it work, but there are no guarantees.
+
+:::
+
+## Logging into Apify platform from Crawlee
+
+To access your [Apify account](https://console.apify.com/sign-up) from Crawlee, you must provide
+credentials - your [API token](https://console.apify.com/account?tab=integrations). You can do that
+either by utilizing [Apify CLI](https://github.com/apify/apify-cli) or with environment
+variables.
+
+Once you provide credentials to your scraper, you will be able to use all the Apify platform
+features, such as calling actors, saving to cloud storages, using Apify proxies,
+setting up webhooks and so on.
+
+### Log in with CLI
+
+Apify CLI allows you to log in to your Apify account on your computer. If you then run your
+scraper using the CLI, your credentials will automatically be added.
+
+```bash
+npm install -g apify-cli
+apify login -t YOUR_API_TOKEN
+```
+
+### Log in with environment variables
+
+Alternatively, you can always provide credentials to your scraper
+by setting the [`APIFY_TOKEN`](#apify_token) environment
+variable to your API token.
+
+> There's also the [`APIFY_PROXY_PASSWORD`](#apify_proxy_password)
+> environment variable. Actor automatically infers that from your token, but it can be useful
+> when you need to access proxies from a different account than your token represents.
+
+### Log in with Configuration
+
+Another option is to use the [`Configuration`](https://docs.apify.com/sdk/js/reference/class/Configuration) instance and set your api token there.
+
+```javascript
+import { Actor } from 'apify';
+
+const sdk = new Actor({ token: 'your_api_token' });
+```
+
+## What is an actor
+
+When you deploy your script to the Apify platform, it becomes an [actor](https://apify.com/actors).
+An actor is a serverless microservice that accepts an input and produces an output. It can run for
+a few seconds, hours or even infinitely. An actor can perform anything from a simple action such
+as filling out a web form or sending an email, to complex operations such as crawling an entire website
+and removing duplicates from a large dataset.
+
+Actors can be shared in the [Apify Store](https://apify.com/store) so that other people can use them.
+But don't worry, if you share your actor in the store and somebody uses it, it runs under their account,
+not yours.
+
+**Related links**
+
+- [Store of existing actors](https://apify.com/store)
+- [Documentation](https://docs.apify.com/actors)
+- [View actors in Apify Console](https://console.apify.com/actors)
+- [API reference](https://apify.com/docs/api/v2#/reference/actors)
+
+## Running an actor locally
+
+First let's create a boilerplate of the new actor. You could use Apify CLI and just run:
+
+```bash
+apify create my-hello-world
+```
+
+The CLI will prompt you to select a project boilerplate template - let's pick "Hello world". The tool will create a directory called `my-hello-world` with a Node.js project files. You can run the actor as follows:
+
+```bash
+cd my-hello-world
+apify run
+```
+
+## Running Crawlee code as an actor
+
+For running Crawlee code as an actor on [Apify platform](https://apify.com/actors) you should either:
+- use a combination of [`Actor.init()`](https://docs.apify.com/sdk/js/reference/class/Actor#init) and [`Actor.exit()`](https://docs.apify.com/sdk/js/reference/class/Actor#exit) functions;
+- or wrap it into [`Actor.main()`](https://docs.apify.com/sdk/js/reference/class/Actor#main) function.
+
+:::info NOTE
+- Adding [`Actor.init()`](https://docs.apify.com/sdk/js/reference/class/Actor#init) and [`Actor.exit()`](https://docs.apify.com/sdk/js/reference/class/Actor#exit) to your code are the only two important things needed to run it on Apify platform as an actor. `Actor.init()` is needed to initialize your actor (e.g. to set the correct storage implementation), while without `Actor.exit()` the process will simply never stop.
+- [`Actor.main()`](https://docs.apify.com/sdk/js/reference/class/Actor#main) is an alternative to `Actor.init()` and `Actor.exit()` as it calls both behind the scenes.
+:::
+
+Let's look at the `CheerioCrawler` example from the [Quick Start](../quick-start) guide:
+
+<Tabs>
+    <TabItem value="main" label="Using Actor.main()" default>
+        <CodeBlock language="js">
+            {MainSource}
+        </CodeBlock>
+    </TabItem>
+    <TabItem value="init_exit" label="Using Actor.init() and Actor.exit()">
+        <CodeBlock language="js">
+            {InitExitSource}
+        </CodeBlock>
+    </TabItem>
+</Tabs>
+
+Note that you could also run your actor (that is using Crawlee) locally with Apify CLI. You could start it via the following command in your project folder:
+
+```bash
+apify run
+```
+
+## Deploying an actor to Apify platform
+
+Now (assuming you are already logged in to your Apify account) you can easily deploy your code to the Apify platform by running:
+
+```bash
+apify push
+```
+
+Your script will be uploaded to and built on the Apify platform so that it can be run there. For more information, view the
+[Apify Actor](https://docs.apify.com/cli) documentation.
+
+## Usage on Apify platform
+
+You can also develop your actor in an online code editor directly on the platform (you'll need an Apify Account). Let's go to the [Actors](https://console.apify.com/actors) page in the app, click *Create new* and then go to the *Source* tab and start writing the code or paste one of the examples from the [Examples](../examples) section.
+
+## Storages
+
+There are several things worth mentioning here.
+
+### Helper functions for default Key-Value Store and Dataset
+
+To simplify access to the _default_ storages, instead of using the helper functions of respective storage classes, you could use:
+- [`Actor.setValue()`](https://docs.apify.com/sdk/js/reference/class/Actor#setValue), [`Actor.getValue()`](https://docs.apify.com/sdk/js/reference/class/Actor#getValue), [`Actor.getInput()`](https://docs.apify.com/sdk/js/reference/class/Actor#getInput) for `Key-Value Store`
+- [`Actor.pushData()`](https://docs.apify.com/sdk/js/reference/class/Actor#pushData) for `Dataset`
+
+### Using platform storage in a local actor
+
+When you plan to use the platform storage while developing and running your actor locally, you should use [`Actor.openKeyValueStore()`](https://docs.apify.com/sdk/js/reference/class/Actor#openKeyValueStore), [`Actor.openDataset()`](https://docs.apify.com/sdk/js/reference/class/Actor#openDataset) and [`Actor.openRequestQueue()`](https://docs.apify.com/sdk/js/reference/class/Actor#openRequestQueue) to open the respective storage.
+
+Using each of these methods allows to pass the [`OpenStorageOptions`](https://docs.apify.com/sdk/js/reference/interface/OpenStorageOptions) as a second argument, which has only one optional property: [`forceCloud`](https://docs.apify.com/sdk/js/reference/interface/OpenStorageOptions#forceCloud). If set to `true` - cloud storage will be used instead of the folder on the local disk.
+
+:::note
+If you don't plan to force usage of the platform storages when running the actor locally, there is no need to use the [`Actor`](https://docs.apify.com/sdk/js/reference/class/Actor) class for it. The Crawlee variants <ApiLink to="core/class/KeyValueStore#open">`KeyValueStore.open()`</ApiLink>, <ApiLink to="core/class/Dataset#open">`Dataset.open()`</ApiLink> and <ApiLink to="core/class/RequestQueue#open">`RequestQueue.open()`</ApiLink> will work the same.
+:::
+
+### Getting public url of an item in the platform storage
+
+If you need to share a link to some file stored in a Key-Value Store on Apify Platform, you can use [`getPublicUrl()`](https://docs.apify.com/sdk/js/reference/class/KeyValueStore#getPublicUrl) method. It accepts only one parameter: `key` - the key of the item you want to share.
+
+```js
+import { KeyValueStore } from 'apify';
+
+const store = await KeyValueStore.open();
+await store.setValue('your-file', { foo: 'bar' });
+const url = store.getPublicUrl('your-file');
+// https://api.apify.com/v2/key-value-stores/<your-store-id>/records/your-file
+```
+
+### Exporting dataset data
+
+When the <ApiLink to="core/class/Dataset">`Dataset`</ApiLink> is stored on the [Apify platform](https://apify.com/actors), you can export its data to the following formats: HTML, JSON, CSV, Excel, XML and RSS. The datasets are displayed on the actor run details page and in the [Storage](https://console.apify.com/storage) section in the Apify Console. The actual data is exported using the [Get dataset items](https://apify.com/docs/api/v2#/reference/datasets/item-collection/get-items) Apify API endpoint. This way you can easily share the crawling results.
+
+**Related links**
+
+- [Apify platform storage documentation](https://docs.apify.com/storage)
+- [View storage in Apify Console](https://console.apify.com/storage)
+- [Key-value stores API reference](https://apify.com/docs/api/v2#/reference/key-value-stores)
+- [Datasets API reference](https://docs.apify.com/api/v2#/reference/datasets)
+- [Request queues API reference](https://docs.apify.com/api/v2#/reference/request-queues)
+
+## Environment variables
+
+The following are some additional environment variables specific to Apify platform. More Crawlee specific environment variables could be found in the [Environment Variables](../guides/configuration#environment-variables) guide.
+
+:::note
+
+It's important to notice that `CRAWLEE_` environment variables don't need to be replaced with equivalent `APIFY_` ones. Likewise, Crawlee understands `APIFY_` environment variables after calling `Actor.init()` or when using `Actor.main()`.
+
+:::
+
+### `APIFY_TOKEN`
+
+The API token for your Apify account. It is used to access the Apify API, e.g. to access cloud storage
+or to run an actor on the Apify platform. You can find your API token on the
+[Account Settings / Integrations](https://console.apify.com/account?tab=integrations) page.
+
+### Combinations of `APIFY_TOKEN` and `CRAWLEE_STORAGE_DIR`
+
+> `CRAWLEE_STORAGE_DIR` env variable description could be found in [Environment Variables](../guides/configuration#crawlee_storage_dir) guide.
+
+By combining the env vars in various ways, you can greatly influence the actor's behavior.
+
+| Env Vars                                | API | Storages         |
+| --------------------------------------- | --- | ---------------- |
+|  none OR `CRAWLEE_STORAGE_DIR`          | no  | local            |
+| `APIFY_TOKEN`                           | yes | Apify platform   |
+| `APIFY_TOKEN` AND `CRAWLEE_STORAGE_DIR` | yes | local + platform |
+
+When using both `APIFY_TOKEN` and `CRAWLEE_STORAGE_DIR`, you can use all the Apify platform
+features and your data will be stored locally by default. If you want to access platform storages,
+you can use the `{ forceCloud: true }` option in their respective functions.
+
+```js
+import { Actor } from 'apify';
+import { Dataset } from 'crawlee';
+
+// or Dataset.open('my-local-data')
+const localDataset = await Actor.openDataset('my-local-data');
+// but here we need the `Actor` class
+const remoteDataset = await Actor.openDataset('my-dataset', { forceCloud: true });
+```
+
+### `APIFY_PROXY_PASSWORD`
+
+Optional password to [Apify Proxy](https://docs.apify.com/proxy) for IP address rotation.
+Assuming Apify Account was already created, you can find the password on the [Proxy page](https://console.apify.com/proxy)
+in the Apify Console. The password is automatically inferred using the `APIFY_TOKEN` env var,
+so in most cases, you don't need to touch it. You should use it when, for some reason,
+you need access to Apify Proxy, but not access to Apify API, or when you need access to
+proxy from a different account than your token represents.
+
+## Proxy management
+
+In addition to your own proxy servers and proxy servers acquired from
+third-party providers used together with Crawlee, you can also rely on [Apify Proxy](https://apify.com/proxy)
+for your scraping needs.
+
+### Apify Proxy
+
+If you are already subscribed to Apify Proxy, you can start using them immediately in only a few lines of code (for local usage you first should be [logged in](#logging-into-apify-platform-from-crawlee) to your Apify account.
+
+```javascript
+import { Actor } from 'apify';
+
+const proxyConfiguration = await Actor.createProxyConfiguration();
+const proxyUrl = await proxyConfiguration.newUrl();
+```
+
+Note that unlike using your own proxies in Crawlee, you shouldn't use the constructor to create <ApiLink to="core/class/ProxyConfiguration">`ProxyConfiguration`</ApiLink> instance. For using Apify Proxy you should create an instance using the [`Actor.createProxyConfiguration()`](https://docs.apify.com/sdk/js/reference/class/Actor#createProxyConfiguration) function instead.
+
+### Apify Proxy Configuration
+
+With Apify Proxy, you can select specific proxy groups to use, or countries to connect from.
+This allows you to get better proxy performance after some initial research.
+
+```javascript
+import { Actor } from 'apify';
+
+const proxyConfiguration = await Actor.createProxyConfiguration({
+    groups: ['RESIDENTIAL'],
+    countryCode: 'US',
+});
+const proxyUrl = await proxyConfiguration.newUrl();
+```
+
+Now your crawlers will use only Residential proxies from the US. Note that you must first get access
+to a proxy group before you are able to use it. You can check proxy groups available to you
+in the [proxy dashboard](https://console.apify.com/proxy).
+
+### Apify Proxy vs. Own proxies
+
+The `ProxyConfiguration` class covers both Apify Proxy and custom proxy URLs so that
+you can easily switch between proxy providers. However, some features of the class
+are available only to Apify Proxy users, mainly because Apify Proxy is what
+one would call a super-proxy. It's not a single proxy server, but an API endpoint
+that allows connection through millions of different IP addresses. So the class
+essentially has two modes: Apify Proxy or Own (third party) proxy.
+
+The difference is easy to remember.
+- If you're using your own proxies - you should create an instance with the ProxyConfiguration <ApiLink to="core/class/ProxyConfiguration#constructor">`constructor`</ApiLink> function based on the provided <ApiLink to="core/interface/ProxyConfigurationOptions">`ProxyConfigurationOptions`</ApiLink>.
+- If you are planning to use Apify Proxy - you should create an instance using the [`Actor.createProxyConfiguration()`](https://docs.apify.com/sdk/js/reference/class/Actor#createProxyConfiguration) function. <ApiLink to="core/interface/ProxyConfigurationOptions#proxyUrls">`ProxyConfigurationOptions.proxyUrls`</ApiLink> and <ApiLink to="core/interface/ProxyConfigurationOptions#newUrlFunction">`ProxyConfigurationOptions.newUrlFunction`</ApiLink> enable use of your custom proxy URLs, whereas all the other options are there to configure Apify Proxy.
+
+**Related links**
+
+- [Apify Proxy docs](https://docs.apify.com/proxy)

docs/deployment/apify_platform_init_exit.ts

@@ -0,0 +1,27 @@
+import { Actor } from 'apify';
+import { CheerioCrawler } from 'crawlee';
+
+await Actor.init();
+
+const crawler = new CheerioCrawler({
+    async requestHandler({ request, $, enqueueLinks }) {
+        const { url } = request;
+
+        // Extract HTML title of the page.
+        const title = $('title').text();
+        console.log(`Title of ${url}: ${title}`);
+
+        // Add URLs that match the provided pattern.
+        await enqueueLinks({
+            globs: ['https://www.iana.org/*'],
+        });
+
+        // Save extracted data to dataset.
+        await Actor.pushData({ url, title });
+    },
+});
+
+// Enqueue the initial request and run the crawler
+await crawler.run(['https://www.iana.org/']);
+
+await Actor.exit();

docs/deployment/apify_platform_main.ts

@@ -0,0 +1,25 @@
+import { Actor } from 'apify';
+import { CheerioCrawler } from 'crawlee';
+
+await Actor.main(async () => {
+    const crawler = new CheerioCrawler({
+        async requestHandler({ request, $, enqueueLinks }) {
+            const { url } = request;
+
+            // Extract HTML title of the page.
+            const title = $('title').text();
+            console.log(`Title of ${url}: ${title}`);
+
+            // Add URLs that match the provided pattern.
+            await enqueueLinks({
+                globs: ['https://www.iana.org/*'],
+            });
+
+            // Save extracted data to dataset.
+            await Actor.pushData({ url, title });
+        },
+    });
+
+    // Enqueue the initial request and run the crawler
+    await crawler.run(['https://www.iana.org/']);
+});

docs/deployment/aws-browsers.md

@@ -0,0 +1,124 @@
+---
+id: aws-browsers
+title: Browsers on AWS Lambda
+---
+
+Running browser-enabled Crawlee crawlers in AWS Lambda is a bit complicated - but not too much. The main problem is that we have to upload not only our code and the dependencies, but also the **browser binaries**.
+
+## Managing browser binaries
+
+Fortunately, there are already some NPM packages that can help us with managing the browser binaries installation:
+
+- [@sparticuz/chromium](https://www.npmjs.com/package/@sparticuz/chromium) is an NPM package containing brotli-compressed chromium binaries. When run in the Lambda environment, the package unzips the binaries under the `/tmp/` path and returns the path to the executable.
+
+We just add this package to the project dependencies and zip the `node_modules` folder.
+
+```bash
+# Install the package
+npm i -S @sparticuz/chromium
+
+# Zip the dependencies
+zip -r dependencies.zip ./node_modules
+```
+
+We will now upload the `dependencies.zip` as a Lambda Layer to AWS. Unfortunately, we cannot do this directly - there is a 50MB limit on direct uploads (and the compressed Chromium build is around that size itself). Instead, we'll upload it as an object into an S3 storage and provide the link to that object during the layer creation.
+
+## Updating the code
+
+We also have to slightly update the Crawlee code:
+
+- First, we pass a new `Configuration` instance to the Crawler. This way, every crawler instance we create will have its own storage and won’t interfere with other crawler instances running in your Lambda environment.
+
+```javascript title="src/main.js"
+// For more information, see https://crawlee.dev/
+import { Configuration, PlaywrightCrawler } from 'crawlee';
+import { router } from './routes.js';
+
+const startUrls = ['https://crawlee.dev'];
+
+const crawler = new PlaywrightCrawler({
+    requestHandler: router,
+// highlight-start
+}, new Configuration({
+    persistStorage: false,
+}));
+// highlight-end
+
+await crawler.run(startUrls);
+```
+
+- Now, we actually have to supply the code with the Chromium path from the `@sparticuz/chromium` package. AWS Lambda execution also lacks some hardware support for GPU acceleration etc. - you can tell Chrome about this by passing the `aws_chromium.args` to the `args` parameter.
+
+```javascript title="src/main.js"
+// For more information, see https://crawlee.dev/
+import { Configuration, PlaywrightCrawler } from 'crawlee';
+import { router } from './routes.js';
+// highlight-next-line
+import aws_chromium from '@sparticuz/chromium';
+
+const startUrls = ['https://crawlee.dev'];
+
+const crawler = new PlaywrightCrawler({
+    requestHandler: router,
+    // highlight-start
+    launchContext: {
+        launchOptions: {
+             executablePath: await aws_chromium.executablePath(),
+             args: aws_chromium.args,
+             headless: true
+        }
+    }
+    // highlight-end
+}, new Configuration({
+    persistStorage: false,
+}));
+
+```
+
+- Last but not least, we have to wrap the code in the exported `handler` function - this will become the Lambda AWS will be executing.
+
+```javascript title="src/main.js"
+import { Configuration, PlaywrightCrawler } from 'crawlee';
+import { router } from './routes.js';
+import aws_chromium from '@sparticuz/chromium';
+
+const startUrls = ['https://crawlee.dev'];
+
+// highlight-next-line
+export const handler = async (event, context) => {
+    const crawler = new PlaywrightCrawler({
+        requestHandler: router,
+        launchContext: {
+            launchOptions: {
+                executablePath: await aws_chromium.executablePath(),
+                args: aws_chromium.args,
+                headless: true
+            }
+        }
+    }, new Configuration({
+        persistStorage: false,
+    }));
+
+    await crawler.run(startUrls);
+
+    // highlight-start
+    return {
+        statusCode: 200,
+        body: await crawler.getData(),
+    };
+}
+// highlight-end
+
+```
+
+## Deploying the code
+
+Now we can simply pack the code into a zip archive (minus the `node_modules` folder, we have put that in the Lambda Layer, remember?). We upload the code archive to AWS as the Lambda body, set up the Lambda so it uses the dependencies Layer, and test our newly created Lambda.
+
+:::tip Memory settings
+
+Since we’re using full-size browsers here, we have to update the Lambda configurations a bit. Most importantly, make sure to set the memory setting to **1024 MB or more** and update the **Lambda timeout**. 
+
+The target timeout value depends on how long your crawler will be running. Try measuring the execution time when running your crawler locally and set the timeout accordingly.
+
+:::

docs/deployment/aws-cheerio.md

@@ -0,0 +1,126 @@
+---
+id: aws-cheerio
+title: Cheerio on AWS Lambda
+---
+
+Locally, we can conveniently create a Crawlee project with `npx crawlee create`. In order to run this project on AWS Lambda, however, we need to do a few tweaks.
+
+## Updating the code
+
+Whenever we instantiate a new crawler, we have to pass a unique `Configuration` instance to it. By default, all the Crawlee crawler instances share the same storage - this can be convenient, but would also cause “statefulness” of our Lambda, which would lead to hard-to-debug problems.
+
+Also, when creating this Configuration instance, make sure to pass the `persistStorage: false` option. This tells Crawlee to use in-memory storage, as the Lambda filesystem is read-only.
+    
+```javascript title="src/main.js"
+// For more information, see https://crawlee.dev/
+import { CheerioCrawler, Configuration, ProxyConfiguration } from 'crawlee';
+import { router } from './routes.js';
+
+const startUrls = ['https://crawlee.dev'];
+
+const crawler = new CheerioCrawler({
+    requestHandler: router,
+// highlight-start
+}, new Configuration({
+    persistStorage: false,
+}));
+// highlight-end
+
+await crawler.run(startUrls);
+```
+
+Now, we wrap all the logic in a `handler` function. This is the actual “Lambda” that AWS will be executing later on. 
+
+```javascript title="src/main.js"
+// For more information, see https://crawlee.dev/
+import { CheerioCrawler, Configuration } from 'crawlee';
+import { router } from './routes.js';
+
+const startUrls = ['https://crawlee.dev'];
+
+// highlight-next-line
+export const handler = async (event, context) => {
+    const crawler = new CheerioCrawler({
+        requestHandler: router,
+    }, new Configuration({
+        persistStorage: false,
+    }));
+
+    await crawler.run(startUrls);
+// highlight-next-line
+};
+```
+
+:::tip **Important**
+
+Make sure to always instantiate a **new crawler instance for every Lambda**. AWS always keeps the environment running for some time after the first Lambda execution (in order to reduce cold-start times) - so any subsequent Lambda calls will access the already-used crawler instance.
+
+**TLDR: Keep your Lambda stateless.**
+
+:::
+
+
+Last things last, we also want to return the scraped data from the Lambda when the crawler run ends.
+
+In the end, your `main.js` script should look something like this:
+
+```javascript title="src/main.js"
+// For more information, see https://crawlee.dev/
+import { CheerioCrawler, Configuration } from 'crawlee';
+import { router } from './routes.js';
+
+const startUrls = ['https://crawlee.dev'];
+
+export const handler = async (event, context) => {
+    const crawler = new CheerioCrawler({
+        requestHandler: router,
+    }, new Configuration({
+        persistStorage: false,
+    }));
+
+    await crawler.run(startUrls);
+
+    // highlight-start
+    return {
+        statusCode: 200,
+        body: await crawler.getData(),
+    }
+    // highlight-end
+};
+```
+
+## Deploying the project
+
+Now it’s time to deploy our script on AWS!
+
+Let’s create a zip archive from our project (including the `node_modules` folder) by running `zip -r package.zip .` in the project folder.
+
+:::note Large `node_modules` folder?
+
+AWS has a limit of 50MB for direct file upload. Usually, our Crawlee projects won’t be anywhere near this limit, but we can easily exceed this with large dependency trees.
+
+A better way to install your project dependencies is by using Lambda Layers. With Layers, we can also share files between multiple Lambdas - and keep the actual “code” part of the Lambdas as slim as possible.
+
+**To create a Lambda Layer, we need to:** 
+
+- Pack the `node_modules` folder into a separate zip file (the archive should contain one folder named `node_modules`).
+- Create a new Lambda layer from this archive. We’ll probably need to upload this file to AWS S3 storage and create the Lambda Layer like this.
+- After creating it, we simply tell our new Lambda function to use this layer.
+
+:::
+
+To deploy our actual code, we upload the `package.zip` archive as our code source.
+
+In Lambda Runtime Settings, we point the `handler` to the main function that runs the crawler. You can use slashes to describe directory structure and `.` to denote a named export. Our handler function is called `handler` and is exported from the `src/main.js` file, so we’ll use `src/main.handler` as the handler name.
+
+Now we’re all set! By clicking the **Test** button, we can send an example testing event to our new Lambda. The actual contents of the event don’t really matter for now - if you want, further parameterize your crawler run by analyzing the `event` object AWS passes as the first argument to the handler.
+
+:::tip
+
+In the Configuration tab in the AWS Lambda dashboard, you can configure the amount of memory the Lambda is running with or the size of the ephemeral storage. 
+
+The memory size can greatly affect the execution speed of your Lambda. 
+
+See the [official documentation](https://docs.aws.amazon.com/lambda/latest/operatorguide/computing-power.html) to see how the performance and cost scale with more memory.
+
+:::

docs/deployment/gcp-browsers.md

@@ -0,0 +1,91 @@
+---
+id: gcp-browsers
+title: Browsers in GCP Cloud Run
+---
+
+Running full-size browsers on GCP Cloud Functions is actually a bit different from doing so on AWS Lambda - [apparently](https://pptr.dev/troubleshooting#running-puppeteer-on-google-cloud-functions), the latest runtime versions miss dependencies required to run Chromium.
+
+If we want to run browser-enabled Crawlee crawlers on GCP, we’ll need to turn towards **Cloud Run.** Cloud Run is GCP’s platform for running Docker containers - other than that, (almost) everything is the same as with Cloud Functions / AWS Lambdas. 
+
+GCP can spin up your containers on demand, so you’re only billed for the time it takes your container to return an HTTP response to the requesting client. In a way, it also provides a slightly better developer experience (than regular FaaS), as you can debug your Docker containers locally and be sure you’re getting the same setup in the cloud.
+
+## Preparing the project
+
+As always, we first pass a new `Configuration` instance to the crawler constructor:
+
+```javascript  title="src/main.js"
+import { Configuration, PlaywrightCrawler } from 'crawlee';
+import { router } from './routes.js';
+
+const startUrls = ['https://crawlee.dev'];
+
+const crawler = new PlaywrightCrawler({
+    requestHandler: router,
+// highlight-start
+}, new Configuration({
+    persistStorage: false,
+}));
+// highlight-end
+
+await crawler.run(startUrls);
+```
+
+All we now need to do is wrap our crawler with an Express HTTP server handler, so it can communicate with the client via HTTP. Because the Cloud Run platform sees only an opaque Docker container, we have to take care of this bit ourselves. 
+
+:::info
+
+GCP passes you an environment variable called `PORT` - your HTTP server is expected to be listening on this port (GCP exposes this one to the outer world).
+
+:::
+
+The `main.js` script should be looking like this in the end:
+
+```javascript title="src/main.js"
+import { Configuration, PlaywrightCrawler } from 'crawlee';
+import { router } from './routes.js';
+// highlight-start
+import express from 'express';
+const app = express();
+// highlight-end
+
+const startUrls = ['https://crawlee.dev'];
+
+
+// highlight-next-line
+app.get('/', async (req, res) => {
+    const crawler = new PlaywrightCrawler({
+        requestHandler: router,
+    }, new Configuration({
+        persistStorage: false,
+    }));
+    
+    await crawler.run(startUrls);    
+
+    // highlight-next-line
+    return res.send(await crawler.getData());
+// highlight-next-line
+});
+
+// highlight-next-line
+app.listen(parseInt(process.env.PORT) || 3000);
+```
+
+:::tip
+
+Always make sure to keep all the logic in the request handler - as with other FaaS services, your request handlers have to be **stateless.**
+
+:::
+
+## Deploying to GCP
+
+Now, we’re ready to deploy! If you have initialized your project using `npx crawlee create`, the initialization script has prepared a Dockerfile for you. 
+
+All you have to do now is run `gcloud run deploy` in your project folder (the one with your Dockerfile in it). The gcloud CLI application will ask you a few questions, such as what region you want to deploy your application in, or whether you want to make your application public or private.
+
+After answering those questions, you should be able to see your application in the GCP dashboard and run it using the link you find there.
+
+:::tip
+
+In case your first execution of your newly created Cloud Run fails, try editing the Run configuration - mainly setting the available memory to 1GiB or more and updating the request timeout according to the size of the website you are scraping.
+
+:::

docs/deployment/gcp-cheerio.md

@@ -0,0 +1,81 @@
+---
+id: gcp-cheerio
+title: Cheerio on GCP Cloud Functions
+---
+
+Running CheerioCrawler-based project in GCP functions is actually quite easy - you just have to make a few changes to the project code.
+
+## Updating the project
+
+Let’s first create the Crawlee project locally with `npx crawlee create`. Set the `"main"` field in the `package.json` file to `"src/main.js"`.
+
+```json title="package.json"
+{
+    "name": "my-crawlee-project",
+    "version": "1.0.0",
+    // highlight-next-line
+    "main": "src/main.js",
+    ...
+}
+```
+
+Now, let’s update the `main.js` file, namely:
+
+- Pass a separate `Configuration` instance (with the `persistStorage` option set to `false`) to the crawler constructor.
+
+```javascript title="src/main.js"
+import { CheerioCrawler, Configuration } from 'crawlee';
+import { router } from './routes.js';
+
+const startUrls = ['https://crawlee.dev'];
+
+const crawler = new CheerioCrawler({
+    requestHandler: router,
+// highlight-start
+}, new Configuration({
+    persistStorage: false,
+}));
+// highlight-end
+
+await crawler.run(startUrls);
+```
+
+- Wrap the crawler call in a separate handler function. This function:
+    - Can be asynchronous
+    - Takes two positional arguments - `req` (containing details about the user-made request to your cloud function) and `res` (response object you can modify).
+        - Call `res.send(data)` to return any data from the cloud function.
+- Export this function from the `src/main.js` module as a named export.
+
+```javascript title="src/main.js"
+import { CheerioCrawler, Configuration } from 'crawlee';
+import { router } from './routes.js';
+
+const startUrls = ['https://crawlee.dev'];
+
+// highlight-next-line
+export const handler = async (req, res) => {
+    const crawler = new CheerioCrawler({
+        requestHandler: router,
+    }, new Configuration({
+        persistStorage: false,
+    }));
+
+    await crawler.run(startUrls);
+    
+    // highlight-next-line
+    return res.send(await crawler.getData())
+// highlight-next-line
+}
+```
+
+## Deploying to Google Cloud Platform
+
+In the Google Cloud dashboard, create a new function, allocate memory and CPUs to it, set region and function timeout.
+    
+When deploying, pick **ZIP Upload**. You have to create a new GCP storage bucket to store the zip packages in.
+    
+Now, for the package - you should zip all the contents of your project folder **excluding the `node_modules` folder** - GCP doesn’t have Layers like AWS Lambda does, but takes care of the project setup for us based on the `package.json` file).
+    
+Also, make sure to set the **Entry point** to the name of the function you’ve exported from the `src/main.js` file. GCP takes the file from the `package.json`'s `main` field.
+    
+After the Function deploys, you can test it by clicking the “Testing” tab. This tab contains a `curl` script that calls your new Cloud Function. To avoid having to install the `gcloud` CLI application locally, you can also run this script in the Cloud Shell by clicking the link above the code block.

docs/examples/accept_user_input.mdx

@@ -0,0 +1,23 @@
+---
+id: accept-user-input
+title: Accept user input
+---
+
+import CodeBlock from '@theme/CodeBlock';
+import AcceptInputSource from '!!raw-loader!./accept_user_input.ts';
+
+This example accepts and logs user input:
+
+<CodeBlock className="language-js">
+	{AcceptInputSource}
+</CodeBlock>
+
+To provide the actor with input, create a `INPUT.json` file inside the "default" key-value store:
+
+```bash
+{PROJECT_FOLDER}/storage/key_value_stores/default/INPUT.json
+```
+
+Anything in this file will be available to the actor when it runs.
+
+To learn about other ways to provide an actor with input, refer to the [Apify Platform Documentation](https://apify.com/docs/actor#run).

docs/examples/accept_user_input.ts

@@ -0,0 +1,4 @@
+import { KeyValueStore } from 'crawlee';
+
+const input = await KeyValueStore.getInput();
+console.log(input);

docs/examples/add_data_to_dataset.mdx

@@ -0,0 +1,21 @@
+---
+id: add-data-to-dataset
+title: Add data to dataset
+---
+
+import RunnableCodeBlock from '@site/src/components/RunnableCodeBlock';
+import ApiLink from '@site/src/components/ApiLink';
+import AddDataToDatasetSource from '!!raw-loader!roa-loader!./add_data_to_dataset.ts';
+
+This example saves data to the default dataset. If the dataset doesn't exist, it will be created.
+You can save data to custom datasets by using <ApiLink to="core/class/Dataset#open">`Dataset.open()`</ApiLink>
+
+<RunnableCodeBlock className="language-js" type="cheerio">
+	{AddDataToDatasetSource}
+</RunnableCodeBlock>
+
+Each item in this dataset will be saved to its own file in the following directory:
+
+```bash
+{PROJECT_FOLDER}/storage/datasets/default/
+```

docs/examples/add_data_to_dataset.ts

@@ -0,0 +1,21 @@
+import { CheerioCrawler } from 'crawlee';
+
+const crawler = new CheerioCrawler({
+    // Function called for each URL
+    async requestHandler({ pushData, request, body }) {
+        // Save data to default dataset
+        await pushData({
+            url: request.url,
+            html: body,
+        });
+    },
+});
+
+await crawler.addRequests([
+    'http://www.example.com/page-1',
+    'http://www.example.com/page-2',
+    'http://www.example.com/page-3',
+]);
+
+// Run the crawler
+await crawler.run();

docs/examples/basic_crawler.mdx

@@ -0,0 +1,19 @@
+---
+id: basic-crawler
+title: Basic crawler
+---
+
+import RunnableCodeBlock from '@site/src/components/RunnableCodeBlock';
+import ApiLink from '@site/src/components/ApiLink';
+import BasicCrawlerSource from '!!raw-loader!roa-loader!./basic_crawler.ts';
+
+This is the most bare-bones example of using Crawlee, which demonstrates some of its building blocks such as the <ApiLink to="basic-crawler/class/BasicCrawler">`BasicCrawler`</ApiLink>. You probably don't need to go this deep though, and it would be better to start with one of the full-featured crawlers
+like <ApiLink to="cheerio-crawler/class/CheerioCrawler">`CheerioCrawler`</ApiLink> or <ApiLink to="playwright-crawler/class/PlaywrightCrawler">`PlaywrightCrawler`</ApiLink>.
+
+The script simply downloads several web pages with plain HTTP requests using the <ApiLink to="basic-crawler/interface/BasicCrawlingContext#sendRequest">`sendRequest`</ApiLink> utility function (which uses the [`got-scraping`](https://github.com/apify/got-scraping)
+npm module internally) and stores their raw HTML and URL in the default dataset. In local configuration, the data will be stored as JSON files in
+`./storage/datasets/default`.
+
+<RunnableCodeBlock className="language-js" type="cheerio">
+	{BasicCrawlerSource}
+</RunnableCodeBlock>

docs/examples/basic_crawler.ts

@@ -0,0 +1,35 @@
+import { BasicCrawler } from 'crawlee';
+
+// Create a BasicCrawler - the simplest crawler that enables
+// users to implement the crawling logic themselves.
+const crawler = new BasicCrawler({
+    // This function will be called for each URL to crawl.
+    async requestHandler({ pushData, request, sendRequest, log }) {
+        const { url } = request;
+        log.info(`Processing ${url}...`);
+
+        // Fetch the page HTML via the crawlee sendRequest utility method
+        // By default, the method will use the current request that is being handled, so you don't have to
+        // provide it yourself. You can also provide a custom request if you want.
+        const { body } = await sendRequest();
+
+        // Store the HTML and URL to the default dataset.
+        await pushData({
+            url,
+            html: body,
+        });
+    },
+});
+
+// The initial list of URLs to crawl. Here we use just a few hard-coded URLs.
+await crawler.addRequests([
+    'https://www.google.com',
+    'https://www.example.com',
+    'https://www.bing.com',
+    'https://www.wikipedia.com',
+]);
+
+// Run the crawler and wait for it to finish.
+await crawler.run();
+
+console.log('Crawler finished.');

docs/examples/cheerio_crawler.mdx

@@ -0,0 +1,14 @@
+---
+id: cheerio-crawler
+title: Cheerio crawler
+---
+
+import RunnableCodeBlock from '@site/src/components/RunnableCodeBlock';
+import ApiLink from '@site/src/components/ApiLink';
+import CheerioCrawlerSource from '!!raw-loader!roa-loader!./cheerio_crawler.ts';
+
+This example demonstrates how to use <ApiLink to="cheerio-crawler/class/CheerioCrawler">`CheerioCrawler`</ApiLink> to crawl a list of URLs from an external file, load each URL using a plain HTTP request, parse the HTML using the [Cheerio library](https://www.npmjs.com/package/cheerio) and extract some data from it: the page title and all `h1` tags.
+
+<RunnableCodeBlock className="language-js" type="cheerio">
+	{CheerioCrawlerSource}
+</RunnableCodeBlock>

docs/examples/cheerio_crawler.ts

@@ -0,0 +1,62 @@
+import { CheerioCrawler, log, LogLevel } from 'crawlee';
+
+// Crawlers come with various utilities, e.g. for logging.
+// Here we use debug level of logging to improve the debugging experience.
+// This functionality is optional!
+log.setLevel(LogLevel.DEBUG);
+
+// Create an instance of the CheerioCrawler class - a crawler
+// that automatically loads the URLs and parses their HTML using the cheerio library.
+const crawler = new CheerioCrawler({
+    // The crawler downloads and processes the web pages in parallel, with a concurrency
+    // automatically managed based on the available system memory and CPU (see AutoscaledPool class).
+    // Here we define some hard limits for the concurrency.
+    minConcurrency: 10,
+    maxConcurrency: 50,
+
+    // On error, retry each page at most once.
+    maxRequestRetries: 1,
+
+    // Increase the timeout for processing of each page.
+    requestHandlerTimeoutSecs: 30,
+
+    // Limit to 10 requests per one crawl
+    maxRequestsPerCrawl: 10,
+
+    // This function will be called for each URL to crawl.
+    // It accepts a single parameter, which is an object with options as:
+    // https://crawlee.dev/js/api/cheerio-crawler/interface/CheerioCrawlerOptions#requestHandler
+    // We use for demonstration only 2 of them:
+    // - request: an instance of the Request class with information such as the URL that is being crawled and HTTP method
+    // - $: the cheerio object containing parsed HTML
+    async requestHandler({ pushData, request, $ }) {
+        log.debug(`Processing ${request.url}...`);
+
+        // Extract data from the page using cheerio.
+        const title = $('title').text();
+        const h1texts: { text: string }[] = [];
+        $('h1').each((_, el) => {
+            h1texts.push({
+                text: $(el).text(),
+            });
+        });
+
+        // Store the results to the dataset. In local configuration,
+        // the data will be stored as JSON files in ./storage/datasets/default
+        await pushData({
+            url: request.url,
+            title,
+            h1texts,
+        });
+    },
+
+    // This function is called if the page processing failed more than maxRequestRetries + 1 times.
+    failedRequestHandler({ request }) {
+        log.debug(`Request ${request.url} failed twice.`);
+    },
+});
+
+// Run the crawler and wait for it to finish.
+await crawler.run(['https://crawlee.dev']);
+
+log.debug('Crawler finished.');

docs/examples/crawl_all_links.mdx

@@ -0,0 +1,63 @@
+---
+id: crawl-all-links
+title: Crawl all links on a website
+---
+
+import Tabs from '@theme/Tabs';
+import TabItem from '@theme/TabItem';
+import RunnableCodeBlock from '@site/src/components/RunnableCodeBlock';
+import ApiLink from '@site/src/components/ApiLink';
+
+import CheerioSource from '!!raw-loader!roa-loader!./crawl_all_links_cheerio.ts';
+import PuppeteerSource from '!!raw-loader!roa-loader!./crawl_all_links_puppeteer.ts';
+import PlaywrightSource from '!!raw-loader!roa-loader!./crawl_all_links_playwright.ts';
+
+This example uses the `enqueueLinks()` method to add new links to the `RequestQueue`
+as the crawler navigates from page to page. This example can also be used to find all URLs on a domain by removing the `maxRequestsPerCrawl` option.
+
+:::tip
+
+If no options are given, by default the method will only add links that are under the same subdomain. This behavior can be controlled with the <ApiLink to="core/interface/EnqueueLinksOptions#strategy">`strategy`</ApiLink>
+option. You can find more info about this option in the [`Crawl relative links`](./crawl-relative-links) examples.
+
+:::
+
+<Tabs groupId="crawler-type">
+
+<TabItem value="cheerio_crawler" label="Cheerio Crawler" default>
+
+<RunnableCodeBlock className="language-js" type="cheerio">
+	{CheerioSource}
+</RunnableCodeBlock>
+
+</TabItem>
+
+<TabItem value="puppeteer_crawler" label="Puppeteer Crawler">
+
+:::tip
+
+To run this example on the Apify Platform, select the `apify/actor-node-puppeteer-chrome` image for your Dockerfile.
+
+:::
+
+<RunnableCodeBlock className="language-js" type="puppeteer">
+	{PuppeteerSource}
+</RunnableCodeBlock>
+
+</TabItem>
+
+<TabItem value="playwright_crawler" label="Playwright Crawler">
+
+:::tip
+
+To run this example on the Apify Platform, select the `apify/actor-node-playwright-chrome` image for your Dockerfile.
+
+:::
+
+<RunnableCodeBlock className="language-js" type="playwright">
+	{PlaywrightSource}
+</RunnableCodeBlock>
+
+</TabItem>
+
+</Tabs>

docs/examples/crawl_all_links_cheerio.ts

@@ -0,0 +1,13 @@
+import { CheerioCrawler } from 'crawlee';
+
+const crawler = new CheerioCrawler({
+    async requestHandler({ request, enqueueLinks, log }) {
+        log.info(request.url);
+        // Add all links from page to RequestQueue
+        await enqueueLinks();
+    },
+    maxRequestsPerCrawl: 10, // Limitation for only 10 requests (do not use if you want to crawl all links)
+});
+
+// Run the crawler with initial request
+await crawler.run(['https://crawlee.dev']);

docs/examples/crawl_all_links_playwright.ts

@@ -0,0 +1,13 @@
+import { PlaywrightCrawler } from 'crawlee';
+
+const crawler = new PlaywrightCrawler({
+    async requestHandler({ request, enqueueLinks, log }) {
+        log.info(request.url);
+        // Add all links from page to RequestQueue
+        await enqueueLinks();
+    },
+    maxRequestsPerCrawl: 10, // Limitation for only 10 requests (do not use if you want to crawl all links)
+});
+
+// Run the crawler with initial request
+await crawler.run(['https://crawlee.dev']);

docs/examples/crawl_all_links_puppeteer.ts

@@ -0,0 +1,13 @@
+import { PuppeteerCrawler } from 'crawlee';
+
+const crawler = new PuppeteerCrawler({
+    async requestHandler({ request, enqueueLinks, log }) {
+        log.info(request.url);
+        // Add all links from page to RequestQueue
+        await enqueueLinks();
+    },
+    maxRequestsPerCrawl: 10, // Limitation for only 10 requests (do not use if you want to crawl all links)
+});
+
+// Run the crawler with initial request
+await crawler.run(['https://crawlee.dev']);

docs/examples/crawl_multiple_urls.mdx

@@ -0,0 +1,54 @@
+---
+id: crawl-multiple-urls
+title: Crawl multiple URLs
+---
+
+import Tabs from '@theme/Tabs';
+import TabItem from '@theme/TabItem';
+import RunnableCodeBlock from '@site/src/components/RunnableCodeBlock';
+
+import CheerioSource from '!!raw-loader!roa-loader!./crawl_multiple_urls_cheerio.ts';
+import PuppeteerSource from '!!raw-loader!roa-loader!./crawl_multiple_urls_puppeteer.ts';
+import PlaywrightSource from '!!raw-loader!roa-loader!./crawl_multiple_urls_playwright.ts';
+
+This example crawls the specified list of URLs.
+
+<Tabs groupId="crawler-type">
+
+<TabItem value="cheerio_crawler" label="Cheerio Crawler" default>
+
+<RunnableCodeBlock className="language-js" type="cheerio">
+	{CheerioSource}
+</RunnableCodeBlock>
+
+</TabItem>
+
+<TabItem value="puppeteer_crawler" label="Puppeteer Crawler">
+
+:::tip
+
+To run this example on the Apify Platform, select the `apify/actor-node-puppeteer-chrome` image for your Dockerfile.
+
+:::
+
+<RunnableCodeBlock className="language-js" type="puppeteer">
+	{PuppeteerSource}
+</RunnableCodeBlock>
+
+</TabItem>
+
+<TabItem value="playwright_crawler" label="Playwright Crawler">
+
+:::tip
+
+To run this example on the Apify Platform, select the `apify/actor-node-playwright-chrome` image for your Dockerfile.
+
+:::
+
+<RunnableCodeBlock className="language-js" type="playwright">
+	{PlaywrightSource}
+</RunnableCodeBlock>
+
+</TabItem>
+
+</Tabs>

docs/examples/crawl_multiple_urls_cheerio.ts

@@ -0,0 +1,12 @@
+import { CheerioCrawler } from 'crawlee';
+
+const crawler = new CheerioCrawler({
+    // Function called for each URL
+    async requestHandler({ request, $, log }) {
+        const title = $('title').text();
+        log.info(`URL: ${request.url}\nTITLE: ${title}`);
+    },
+});
+
+// Run the crawler with initial request
+await crawler.run(['http://www.example.com/page-1', 'http://www.example.com/page-2', 'http://www.example.com/page-3']);

docs/examples/crawl_multiple_urls_playwright.ts

@@ -0,0 +1,12 @@
+import { PlaywrightCrawler } from 'crawlee';
+
+const crawler = new PlaywrightCrawler({
+    // Function called for each URL
+    async requestHandler({ request, page, log }) {
+        const title = await page.title();
+        log.info(`URL: ${request.url}\nTITLE: ${title}`);
+    },
+});
+
+// Run the crawler with initial request
+await crawler.run(['http://www.example.com/page-1', 'http://www.example.com/page-2', 'http://www.example.com/page-3']);

docs/examples/crawl_multiple_urls_puppeteer.ts

@@ -0,0 +1,12 @@
+import { PuppeteerCrawler } from 'crawlee';
+
+const crawler = new PuppeteerCrawler({
+    // Function called for each URL
+    async requestHandler({ request, page, log }) {
+        const title = await page.title();
+        log.info(`URL: ${request.url}\nTITLE: ${title}`);
+    },
+});
+
+// Run the crawler with initial request
+await crawler.run(['http://www.example.com/page-1', 'http://www.example.com/page-2', 'http://www.example.com/page-3']);

docs/examples/crawl_relative_links.mdx

@@ -0,0 +1,88 @@
+---
+id: crawl-relative-links
+title: Crawl a website with relative links
+---
+
+import Tabs from '@theme/Tabs';
+import TabItem from '@theme/TabItem';
+import RunnableCodeBlock from '@site/src/components/RunnableCodeBlock';
+import ApiLink from '@site/src/components/ApiLink';
+
+import AllLinksSource from '!!raw-loader!roa-loader!./crawl_relative_links_all.ts';
+import SameDomainSource from '!!raw-loader!roa-loader!./crawl_relative_links_same_domain.ts';
+import SameHostnameSource from '!!raw-loader!roa-loader!./crawl_relative_links_same_hostname.ts';
+
+When crawling a website, you may encounter different types of links present that you may want to crawl.
+To facilitate the easy crawling of such links, we provide the `enqueueLinks()` method on the crawler context, which will
+automatically find links and add them to the crawler's <ApiLink to="core/class/RequestQueue">`RequestQueue`</ApiLink>.
+
+We provide 3 different strategies for crawling relative links:
+
+- <ApiLink to="core/enum/EnqueueStrategy#All"><inlineCode>All</inlineCode> (or the string <inlineCode>"all"</inlineCode>)</ApiLink> which will
+enqueue all links found, regardless of the domain they point to.
+- <ApiLink to="core/enum/EnqueueStrategy#SameHostname"><inlineCode>SameHostname</inlineCode> (or the string <inlineCode>"same-hostname"</inlineCode>)</ApiLink> which
+will enqueue all links found for the same hostname. This is the default strategy.
+- <ApiLink to="core/enum/EnqueueStrategy#SameDomain"><inlineCode>SameDomain</inlineCode> (or the string <inlineCode>"same-domain"</inlineCode>)</ApiLink> which
+will enqueue all links found that have the same domain name, including links from any possible subdomain.
+
+:::note
+
+For these examples, we are using the <ApiLink to="cheerio-crawler/class/CheerioCrawler">`CheerioCrawler`</ApiLink>, however
+the same method is available for both the <ApiLink to="puppeteer-crawler/class/PuppeteerCrawler">`PuppeteerCrawler`</ApiLink>
+and <ApiLink to="playwright-crawler/class/PlaywrightCrawler">`PlaywrightCrawler`</ApiLink>, and you use it
+the exact same way.
+
+:::
+
+<Tabs groupId="enqueue_strategy">
+
+<TabItem value="all" label="All Links">
+
+:::note Example domains
+
+Any urls found will be matched by this strategy, even if they go off of the site you are currently crawling.
+
+:::
+
+<RunnableCodeBlock className="language-js" type="cheerio">
+	{AllLinksSource}
+</RunnableCodeBlock>
+
+</TabItem>
+
+<TabItem value="same_hostname" label="Same Hostname">
+
+:::note Example domains
+
+For a url of `https://example.com`, `enqueueLinks()` will match relative urls and urls that point to the same
+hostname.
+
+> This is the default strategy when calling `enqueueLinks()`, so you don't have to specify it.
+
+For instance, hyperlinks like `https://example.com/some/path`, `/absolute/example` or `./relative/example` will all be matched by this strategy. But links to any subdomain like `https://subdomain.example.com/some/path` won't.
+
+:::
+
+<RunnableCodeBlock className="language-js" type="cheerio">
+    {SameHostnameSource}
+</RunnableCodeBlock>
+
+</TabItem>
+
+<TabItem value="same-subdomain" label="Same Subdomain" default>
+
+:::note Example domains
+
+For a url of `https://subdomain.example.com`, `enqueueLinks()` will match relative urls or urls that point to the same domain name, regardless of their subdomain.
+
+For instance, hyperlinks like `https://subdomain.example.com/some/path`, `/absolute/example` or `./relative/example` will all be matched by this strategy, as well as links to other subdomains or to the naked domain, like `https://other-subdomain.example.com` or `https://example.com` will work too.
+
+:::
+
+<RunnableCodeBlock className="language-js" type="cheerio">
+    {SameDomainSource}
+</RunnableCodeBlock>
+
+</TabItem>
+
+</Tabs>

docs/examples/crawl_relative_links_all.ts

@@ -0,0 +1,18 @@
+import { CheerioCrawler, EnqueueStrategy } from 'crawlee';
+
+const crawler = new CheerioCrawler({
+    maxRequestsPerCrawl: 10, // Limitation for only 10 requests (do not use if you want to crawl all links)
+    async requestHandler({ request, enqueueLinks, log }) {
+        log.info(request.url);
+        await enqueueLinks({
+            // Setting the strategy to 'all' will enqueue all links found
+            // highlight-next-line
+            strategy: EnqueueStrategy.All,
+            // Alternatively, you can pass in the string 'all'
+            // strategy: 'all',
+        });
+    },
+});
+
+// Run the crawler with initial request
+await crawler.run(['https://crawlee.dev']);

docs/examples/crawl_relative_links_same_domain.ts

@@ -0,0 +1,19 @@
+import { CheerioCrawler, EnqueueStrategy } from 'crawlee';
+
+const crawler = new CheerioCrawler({
+    maxRequestsPerCrawl: 10, // Limitation for only 10 requests (do not use if you want to crawl all links)
+    async requestHandler({ request, enqueueLinks, log }) {
+        log.info(request.url);
+        await enqueueLinks({
+            // Setting the strategy to 'same-domain' will enqueue all links found that are on the
+            // same hostname as request.loadedUrl or request.url
+            // highlight-next-line
+            strategy: EnqueueStrategy.SameDomain,
+            // Alternatively, you can pass in the string 'same-domain'
+            // strategy: 'same-domain',
+        });
+    },
+});
+
+// Run the crawler with initial request
+await crawler.run(['https://crawlee.dev']);

docs/examples/crawl_relative_links_same_hostname.ts

@@ -0,0 +1,19 @@
+import { CheerioCrawler, EnqueueStrategy } from 'crawlee';
+
+const crawler = new CheerioCrawler({
+    maxRequestsPerCrawl: 10, // Limitation for only 10 requests (do not use if you want to crawl all links)
+    async requestHandler({ request, enqueueLinks, log }) {
+        log.info(request.url);
+        await enqueueLinks({
+            // Setting the strategy to 'same-hostname' will enqueue all links found that are on the
+            // same hostname (including subdomain) as request.loadedUrl or request.url
+            // highlight-next-line
+            strategy: EnqueueStrategy.SameHostname,
+            // Alternatively, you can pass in the string 'same-hostname'
+            // strategy: 'same-hostname',
+        });
+    },
+});
+
+// Run the crawler with initial request
+await crawler.run(['https://crawlee.dev']);

docs/examples/crawl_single_url.mdx

@@ -0,0 +1,17 @@
+---
+id: crawl-single-url
+title: Crawl a single URL
+---
+
+import RunnableCodeBlock from '@site/src/components/RunnableCodeBlock';
+import ApiLink from '@site/src/components/ApiLink';
+import CrawlSource from '!!raw-loader!roa-loader!./crawl_single_url.ts';
+
+This example uses the [`got-scraping`](https://github.com/apify/got-scraping) npm package
+to grab the HTML of a web page.
+
+<RunnableCodeBlock className="language-js" type="cheerio">
+	{CrawlSource}
+</RunnableCodeBlock>
+
+If you don't want to hard-code the URL into the script, refer to the [Accept User Input](./accept-user-input) example.

docs/examples/crawl_single_url.ts

@@ -0,0 +1,5 @@
+import { gotScraping } from 'got-scraping';
+
+// Get the HTML of a web page
+const { body } = await gotScraping({ url: 'https://www.example.com' });
+console.log(body);

docs/examples/crawl_sitemap.mdx

@@ -0,0 +1,55 @@
+---
+id: crawl-sitemap
+title: Crawl a sitemap
+---
+
+import Tabs from '@theme/Tabs';
+import TabItem from '@theme/TabItem';
+import RunnableCodeBlock from '@site/src/components/RunnableCodeBlock';
+import ApiLink from '@site/src/components/ApiLink';
+
+import CheerioSource from '!!raw-loader!roa-loader!./crawl_sitemap_cheerio.ts';
+import PuppeteerSource from '!!raw-loader!roa-loader!./crawl_sitemap_puppeteer.ts';
+import PlaywrightSource from '!!raw-loader!roa-loader!./crawl_sitemap_playwright.ts';
+
+We will crawl sitemap which tells search engines which pages and file are important in the website, it also provides valuable information about these files. This example builds a sitemap crawler which downloads and crawls the URLs from a sitemap, by using the <ApiLink to="utils/class/Sitemap">`Sitemap`</ApiLink> utility class provided by the <ApiLink to="utils">`@crawlee/utils`</ApiLink> module.
+
+<Tabs groupId="crawler-type">
+
+<TabItem value="cheerio_crawler" label="Cheerio Crawler" default>
+
+<RunnableCodeBlock className="language-js" type="cheerio">
+	{CheerioSource}
+</RunnableCodeBlock>
+
+</TabItem>
+
+<TabItem value="puppeteer_crawler" label="Puppeteer Crawler">
+
+:::tip
+
+To run this example on the Apify Platform, select the `apify/actor-node-puppeteer-chrome` image for your Dockerfile.
+
+:::
+
+<RunnableCodeBlock className="language-js" type="puppeteer">
+	{PuppeteerSource}
+</RunnableCodeBlock>
+
+</TabItem>
+
+<TabItem value="playwright_crawler" label="Playwright Crawler">
+
+:::tip
+
+To run this example on the Apify Platform, select the `apify/actor-node-playwright-chrome` image for your Dockerfile.
+
+:::
+
+<RunnableCodeBlock className="language-js" type="playwright">
+	{PlaywrightSource}
+</RunnableCodeBlock>
+
+</TabItem>
+
+</Tabs>

docs/examples/crawl_sitemap_cheerio.ts

@@ -0,0 +1,16 @@
+import { CheerioCrawler, Sitemap } from 'crawlee';
+
+const crawler = new CheerioCrawler({
+    // Function called for each URL
+    async requestHandler({ request, log }) {
+        log.info(request.url);
+    },
+    maxRequestsPerCrawl: 10, // Limitation for only 10 requests (do not use if you want to crawl a sitemap)
+});
+
+const { urls } = await Sitemap.load('https://crawlee.dev/sitemap.xml');
+
+await crawler.addRequests(urls);
+
+// Run the crawler
+await crawler.run();

docs/examples/crawl_sitemap_playwright.ts

@@ -0,0 +1,16 @@
+import { PlaywrightCrawler, Sitemap } from 'crawlee';
+
+const crawler = new PlaywrightCrawler({
+    // Function called for each URL
+    async requestHandler({ request, log }) {
+        log.info(request.url);
+    },
+    maxRequestsPerCrawl: 10, // Limitation for only 10 requests (do not use if you want to crawl a sitemap)
+});
+
+const { urls } = await Sitemap.load('https://crawlee.dev/sitemap.xml');
+
+await crawler.addRequests(urls);
+
+// Run the crawler
+await crawler.run();

docs/examples/crawl_sitemap_puppeteer.ts

@@ -0,0 +1,16 @@
+import { PuppeteerCrawler, Sitemap } from 'crawlee';
+
+const crawler = new PuppeteerCrawler({
+    // Function called for each URL
+    async requestHandler({ request, log }) {
+        log.info(request.url);
+    },
+    maxRequestsPerCrawl: 10, // Limitation for only 10 requests (do not use if you want to crawl a sitemap)
+});
+
+const { urls } = await Sitemap.load('https://crawlee.dev/sitemap.xml');
+
+await crawler.addRequests(urls);
+
+// Run the crawler
+await crawler.run();

docs/examples/crawl_some_links.mdx

@@ -0,0 +1,14 @@
+---
+id: crawl-some-links
+title: Crawl some links on a website
+---
+
+import RunnableCodeBlock from '@site/src/components/RunnableCodeBlock';
+import ApiLink from '@site/src/components/ApiLink';
+import CrawlSource from '!!raw-loader!roa-loader!./crawl_some_links.ts';
+
+This <ApiLink to="cheerio-crawler/class/CheerioCrawler">`CheerioCrawler`</ApiLink> example uses the <ApiLink to="core/interface/EnqueueLinksOptions#globs">`globs`</ApiLink> property in the <ApiLink to="cheerio-crawler/interface/CheerioCrawlingContext#enqueueLinks">`enqueueLinks()`</ApiLink> method to only add links to the <ApiLink to="core/class/RequestQueue">`RequestQueue`</ApiLink> queue if they match the specified pattern.
+
+<RunnableCodeBlock className="language-js" type="cheerio">
+	{CrawlSource}
+</RunnableCodeBlock>

docs/examples/crawl_some_links.ts

@@ -0,0 +1,21 @@
+import { CheerioCrawler } from 'crawlee';
+
+// Create a CheerioCrawler
+const crawler = new CheerioCrawler({
+    // Limits the crawler to only 10 requests (do not use if you want to crawl all links)
+    maxRequestsPerCrawl: 10,
+    // Function called for each URL
+    async requestHandler({ request, enqueueLinks, log }) {
+        log.info(request.url);
+        // Add some links from page to the crawler's RequestQueue
+        await enqueueLinks({
+            globs: ['http?(s)://crawlee.dev/*/*'],
+        });
+    },
+});
+
+// Define the starting URL
+await crawler.addRequests(['https://crawlee.dev']);
+
+// Run the crawler
+await crawler.run();

docs/examples/crawler-plugins/index.mdx

@@ -0,0 +1,78 @@
+---
+id: playwright-extra-puppeteer-stealth
+title: Using Puppeteer Stealth Plugin (puppeteer-extra) and playwright-extra
+---
+
+import RunnableCodeBlock from '@site/src/components/RunnableCodeBlock';
+import Tabs from '@theme/Tabs';
+import TabItem from '@theme/TabItem';
+import ApiLink from '@site/src/components/ApiLink';
+import PuppeteerExtraSource from '!!raw-loader!roa-loader!./puppeteer-extra.ts';
+import PlaywrightExtraSource from '!!raw-loader!roa-loader!./playwright-extra.ts';
+
+[`puppeteer-extra`](https://www.npmjs.com/package/puppeteer-extra) and [`playwright-extra`](https://www.npmjs.com/package/playwright-extra) are community-built
+libraries that bring in a plugin system to enhance the usage of [`puppeteer`](https://www.npmjs.com/package/puppeteer) and
+[`playwright`](https://www.npmjs.com/package/playwright) respectively (bringing in extra functionality, like improving stealth for
+example by using the Puppeteer Stealth plugin [(`puppeteer-extra-plugin-stealth`)](https://www.npmjs.com/package/puppeteer-extra-plugin-stealth)).
+
+:::tip Available plugins
+
+You can see a list of available plugins on the [`puppeteer-extra`s plugin list](https://www.npmjs.com/package/puppeteer-extra#plugins).
+
+For [`playwright`](https://www.npmjs.com/package/playwright), please see [`playwright-extra`s plugin list](https://www.npmjs.com/package/playwright-extra#plugins) instead.
+
+:::
+
+In this example, we'll show you how to use the Puppeteer Stealth [(`puppeteer-extra-plugin-stealth`)](https://www.npmjs.com/package/puppeteer-extra-plugin-stealth) plugin
+to help you avoid bot detections when crawling your target website.
+
+<Tabs>
+<TabItem value="puppeteer" label="Puppeteer & puppeteer-extra" default>
+
+:::info Before you begin
+
+Make sure you've installed the Puppeteer Extra (`puppeteer-extra`) and Puppeteer Stealth plugin(`puppeteer-extra-plugin-stealth`) packages via your preferred package manager
+
+```bash
+npm install puppeteer-extra puppeteer-extra-plugin-stealth
+```
+
+:::
+
+:::tip
+
+To run this example on the Apify Platform, select the `apify/actor-node-puppeteer-chrome` image for your Dockerfile.
+
+:::
+
+<RunnableCodeBlock className="language-js" title="src/crawler.ts" type="puppeteer">
+    {PuppeteerExtraSource}
+</RunnableCodeBlock>
+
+</TabItem>
+
+<TabItem value="playwright" label="Playwright & playwright-extra">
+
+:::info Before you begin
+
+Make sure you've installed the `playwright-extra` and `puppeteer-extra-plugin-stealth` packages via your preferred package manager
+
+```bash
+npm install playwright-extra puppeteer-extra-plugin-stealth
+```
+
+:::
+
+:::tip
+
+To run this example on the Apify Platform, select the `apify/actor-node-puppeteer-chrome` image for your Dockerfile.
+
+:::
+
+<RunnableCodeBlock className="language-js" title="src/crawler.ts" type="playwright">
+    {PlaywrightExtraSource}
+</RunnableCodeBlock>
+
+</TabItem>
+
+</Tabs>

docs/examples/crawler-plugins/playwright-extra.ts

@@ -0,0 +1,74 @@
+import { PlaywrightCrawler } from 'crawlee';
+
+// For playwright-extra you will need to import the browser type itself that you want to use!
+// By default, PlaywrightCrawler uses chromium, but you can also use firefox or webkit.
+import { chromium } from 'playwright-extra';
+import stealthPlugin from 'puppeteer-extra-plugin-stealth';
+
+// First, we tell playwright-extra to use the plugin (or plugins) we want.
+// Certain plugins might have options you can pass in - read up on their documentation!
+chromium.use(stealthPlugin());
+
+// Create an instance of the PlaywrightCrawler class - a crawler
+// that automatically loads the URLs in headless Chrome / Playwright.
+const crawler = new PlaywrightCrawler({
+    launchContext: {
+        // !!! You need to specify this option to tell Crawlee to use playwright-extra as the launcher !!!
+        launcher: chromium,
+        launchOptions: {
+            // Other playwright options work as usual
+            headless: true,
+        },
+    },
+
+    // Stop crawling after several pages
+    maxRequestsPerCrawl: 50,
+
+    // This function will be called for each URL to crawl.
+    // Here you can write the Puppeteer scripts you are familiar with,
+    // with the exception that browsers and pages are automatically managed by Crawlee.
+    // The function accepts a single parameter, which is an object with the following fields:
+    // - request: an instance of the Request class with information such as URL and HTTP method
+    // - page: Puppeteer's Page object (see https://pptr.dev/#show=api-class-page)
+    async requestHandler({ pushData, request, page, enqueueLinks, log }) {
+        log.info(`Processing ${request.url}...`);
+
+        // A function to be evaluated by Puppeteer within the browser context.
+        const data = await page.$$eval('.athing', ($posts) => {
+            const scrapedData: { title?: string; rank?: string; href?: string }[] = [];
+
+            // We're getting the title, rank and URL of each post on Hacker News.
+            $posts.forEach(($post) => {
+                scrapedData.push({
+                    title: $post.querySelector<HTMLElement>('.title a')?.innerText,
+                    rank: $post.querySelector<HTMLElement>('.rank')?.innerText,
+                    href: $post.querySelector<HTMLAnchorElement>('.title a')?.href,
+                });
+            });
+
+            return scrapedData;
+        });
+
+        // Store the results to the default dataset.
+        await pushData(data);
+
+        // Find a link to the next page and enqueue it if it exists.
+        const infos = await enqueueLinks({
+            selector: '.morelink',
+        });
+
+        if (infos.processedRequests.length === 0) log.info(`${request.url} is the last page!`);
+    },
+
+    // This function is called if the page processing failed more than maxRequestRetries+1 times.
+    failedRequestHandler({ request, log }) {
+        log.error(`Request ${request.url} failed too many times.`);
+    },
+});
+
+await crawler.addRequests(['https://news.ycombinator.com/']);
+
+// Run the crawler and wait for it to finish.
+await crawler.run();
+
+console.log('Crawler finished.');

docs/examples/crawler-plugins/puppeteer-extra.ts

@@ -0,0 +1,72 @@
+import { PuppeteerCrawler } from 'crawlee';
+import puppeteerExtra from 'puppeteer-extra';
+import stealthPlugin from 'puppeteer-extra-plugin-stealth';
+
+// First, we tell puppeteer-extra to use the plugin (or plugins) we want.
+// Certain plugins might have options you can pass in - read up on their documentation!
+// @ts-expect-error - The default export types for puppeteer-extra don't properly expose the 'use' method in ESM contexts
+puppeteerExtra.use(stealthPlugin());
+
+// Create an instance of the PuppeteerCrawler class - a crawler
+// that automatically loads the URLs in headless Chrome / Puppeteer.
+const crawler = new PuppeteerCrawler({
+    launchContext: {
+        // !!! You need to specify this option to tell Crawlee to use puppeteer-extra as the launcher !!!
+        launcher: puppeteerExtra,
+        launchOptions: {
+            // Other puppeteer options work as usual
+            headless: true,
+        },
+    },
+
+    // Stop crawling after several pages
+    maxRequestsPerCrawl: 50,
+
+    // This function will be called for each URL to crawl.
+    // Here you can write the Puppeteer scripts you are familiar with,
+    // with the exception that browsers and pages are automatically managed by Crawlee.
+    // The function accepts a single parameter, which is an object with the following fields:
+    // - request: an instance of the Request class with information such as URL and HTTP method
+    // - page: Puppeteer's Page object (see https://pptr.dev/#show=api-class-page)
+    async requestHandler({ pushData, request, page, enqueueLinks, log }) {
+        log.info(`Processing ${request.url}...`);
+
+        // A function to be evaluated by Puppeteer within the browser context.
+        const data = await page.$$eval('.athing', ($posts) => {
+            const scrapedData: { title?: string; rank?: string; href?: string }[] = [];
+
+            // We're getting the title, rank and URL of each post on Hacker News.
+            $posts.forEach(($post) => {
+                scrapedData.push({
+                    title: $post.querySelector<HTMLElement>('.title a')?.innerText,
+                    rank: $post.querySelector<HTMLElement>('.rank')?.innerText,
+                    href: $post.querySelector<HTMLAnchorElement>('.title a')?.href,
+                });
+            });
+
+            return scrapedData;
+        });
+
+        // Store the results to the default dataset.
+        await pushData(data);
+
+        // Find a link to the next page and enqueue it if it exists.
+        const infos = await enqueueLinks({
+            selector: '.morelink',
+        });
+
+        if (infos.processedRequests.length === 0) log.info(`${request.url} is the last page!`);
+    },
+
+    // This function is called if the page processing failed more than maxRequestRetries+1 times.
+    failedRequestHandler({ request, log }) {
+        log.error(`Request ${request.url} failed too many times.`);
+    },
+});
+
+await crawler.addRequests(['https://news.ycombinator.com/']);
+
+// Run the crawler and wait for it to finish.
+await crawler.run();
+
+console.log('Crawler finished.');

docs/examples/export_entire_dataset.mdx

@@ -0,0 +1,14 @@
+---
+id: export-entire-dataset
+title: Export entire dataset to one file
+---
+
+import RunnableCodeBlock from '@site/src/components/RunnableCodeBlock';
+import ApiLink from '@site/src/components/ApiLink';
+import CrawlSource from '!!raw-loader!roa-loader!./export_entire_dataset.ts';
+
+This `Dataset` example uses the `exportToValue` function to export the entire default dataset to a single CSV file into the default key-value store.
+
+<RunnableCodeBlock className="language-js" type="cheerio">
+	{CrawlSource}
+</RunnableCodeBlock>

docs/examples/export_entire_dataset.ts

@@ -0,0 +1,20 @@
+import { Dataset } from 'crawlee';
+
+// Retrieve or generate two items to be pushed
+const data = [
+    {
+        id: 123,
+        name: 'foo',
+    },
+    {
+        id: 456,
+        name: 'bar',
+    },
+];
+
+// Push the two items to the default dataset
+await Dataset.pushData(data);
+
+// Export the entirety of the dataset to a single file in
+// the default key-value store under the key "OUTPUT"
+await Dataset.exportToCSV('OUTPUT');

docs/examples/file_download.mdx

@@ -0,0 +1,17 @@
+---
+id: file-download
+title: Download a file
+---
+
+import RunnableCodeBlock from '@site/src/components/RunnableCodeBlock';
+import ApiLink from '@site/src/components/ApiLink';
+import FileDownloadSource from '!!raw-loader!roa-loader!./file_download.ts';
+
+When webcrawling, you sometimes need to download files such as images, PDFs, or other binary files. This example demonstrates how to download files using Crawlee and save them to the default key-value store.
+
+The script simply downloads several files with plain HTTP requests using the custom <ApiLink to="http-crawler/class/FileDownload">`FileDownload`</ApiLink> crawler class and stores their contents in the default key-value store. 
+In local configuration, the data will be stored as files in `./storage/key_value_stores/default`.
+
+<RunnableCodeBlock className="language-js" type="cheerio">
+	{FileDownloadSource}
+</RunnableCodeBlock>

docs/examples/file_download.ts

@@ -0,0 +1,23 @@
+import { FileDownload } from 'crawlee';
+
+// Create a FileDownload - a custom crawler instance that will download files from URLs.
+const crawler = new FileDownload({
+    async requestHandler({ body, request, contentType, getKeyValueStore }) {
+        const url = new URL(request.url);
+        const kvs = await getKeyValueStore();
+
+        await kvs.setValue(url.pathname.replace(/\//g, '_'), body, { contentType: contentType.type });
+    },
+});
+
+// The initial list of URLs to crawl. Here we use just a few hard-coded URLs.
+await crawler.addRequests([
+    'https://pdfobject.com/pdf/sample.pdf',
+    'https://download.blender.org/peach/bigbuckbunny_movies/BigBuckBunny_320x180.mp4',
+    'https://upload.wikimedia.org/wikipedia/commons/c/c8/Example.ogg',
+]);
+
+// Run the downloader and wait for it to finish.
+await crawler.run();
+
+console.log('Crawler finished.');

docs/examples/file_download_stream.mdx

@@ -0,0 +1,17 @@
+---
+id: file-download-stream
+title: Download a file with Node.js streams
+---
+
+import RunnableCodeBlock from '@site/src/components/RunnableCodeBlock';
+import ApiLink from '@site/src/components/ApiLink';
+import FileDownloadSource from '!!raw-loader!roa-loader!./file_download_stream.ts';
+
+For larger files, it is more efficient to use Node.js streams to download and transfer the files. This example demonstrates how to download files using streams.
+
+The script uses the <ApiLink to="http-crawler/class/FileDownload">`FileDownload`</ApiLink> crawler class to download files with streams, log the progress, and store the data in the key-value store.
+In local configuration, the data will be stored as files in `./storage/key_value_stores/default`.
+
+<RunnableCodeBlock className="language-js" type="cheerio">
+	{FileDownloadSource}
+</RunnableCodeBlock>

docs/examples/file_download_stream.ts

@@ -0,0 +1,62 @@
+import { pipeline, Transform } from 'stream';
+
+import { FileDownload, type Log } from 'crawlee';
+
+// A sample Transform stream logging the download progress.
+function createProgressTracker({ url, log, totalBytes }: { url: URL; log: Log; totalBytes: number }) {
+    let downloadedBytes = 0;
+
+    return new Transform({
+        transform(chunk, _, callback) {
+            if (downloadedBytes % 1e6 > (downloadedBytes + chunk.length) % 1e6) {
+                log.info(
+                    `Downloaded ${downloadedBytes / 1e6} MB (${Math.floor((downloadedBytes / totalBytes) * 100)}%) for ${url}.`,
+                );
+            }
+            downloadedBytes += chunk.length;
+
+            this.push(chunk);
+            callback();
+        },
+    });
+}
+
+// Create a FileDownload - a custom crawler instance that will download files from URLs.
+const crawler = new FileDownload({
+    async streamHandler({ stream, request, log, getKeyValueStore }) {
+        const url = new URL(request.url);
+
+        log.info(`Downloading ${url} to ${url.pathname.replace(/\//g, '_')}...`);
+
+        await new Promise<void>((resolve, reject) => {
+            // With the 'response' event, we have received the headers of the response.
+            stream.on('response', async (response) => {
+                const kvs = await getKeyValueStore();
+                await kvs.setValue(
+                    url.pathname.replace(/\//g, '_'),
+                    pipeline(
+                        stream,
+                        createProgressTracker({ url, log, totalBytes: Number(response.headers['content-length']) }),
+                        (error) => {
+                            if (error) reject(error);
+                        },
+                    ),
+                    { contentType: response.headers['content-type'] },
+                );
+
+                log.info(`Downloaded ${url} to ${url.pathname.replace(/\//g, '_')}.`);
+
+                resolve();
+            });
+        });
+    },
+});
+
+// The initial list of URLs to crawl. Here we use just a few hard-coded URLs.
+await crawler.addRequests([
+    'https://download.blender.org/peach/bigbuckbunny_movies/BigBuckBunny_320x180.mp4',
+    'https://download.blender.org/peach/bigbuckbunny_movies/BigBuckBunny_640x360.m4v',
+]);
+
+// Run the downloader and wait for it to finish.
+await crawler.run();

docs/examples/forms.mdx

@@ -0,0 +1,24 @@
+---
+id: forms
+title: Fill and Submit a Form using Puppeteer
+---
+
+import RunnableCodeBlock from '@site/src/components/RunnableCodeBlock';
+import ApiLink from '@site/src/components/ApiLink';
+import CrawlSource from '!!raw-loader!roa-loader!./forms.ts';
+
+This example demonstrates how to use <ApiLink to="puppeteer-crawler/class/PuppeteerCrawler">`PuppeteerCrawler`</ApiLink> to
+automatically fill and submit a search form to look up repositories on [GitHub](https://github.com) using headless Chrome / Puppeteer.
+The crawler first fills in the search term, repository owner, start date and language of the repository, then submits the form
+and prints out the results. Finally, the results are saved either on the Apify platform to the
+default <ApiLink to="core/class/Dataset">`dataset`</ApiLink> or on the local machine as JSON files in `./storage/datasets/default`.
+
+:::tip
+
+To run this example on the Apify Platform, select the `apify/actor-node-puppeteer-chrome` image for your Dockerfile.
+
+:::
+
+<RunnableCodeBlock className="language-js" type="puppeteer">
+	{CrawlSource}
+</RunnableCodeBlock>

docs/examples/forms.ts

@@ -0,0 +1,39 @@
+import { Dataset, launchPuppeteer } from 'crawlee';
+
+// Launch the web browser.
+const browser = await launchPuppeteer();
+
+// Create and navigate new page
+console.log('Open target page');
+const page = await browser.newPage();
+await page.goto('https://github.com/search/advanced');
+
+// Fill form fields and select desired search options
+console.log('Fill in search form');
+await page.type('#adv_code_search input.js-advanced-search-input', 'apify-js');
+await page.type('#search_from', 'apify');
+await page.type('#search_date', '>2015');
+await page.select('select#search_language', 'JavaScript');
+
+// Submit the form and wait for full load of next page
+console.log('Submit search form');
+await Promise.all([
+    page.waitForNavigation({ waitUntil: 'networkidle2' }),
+    page.click('#adv_code_search button[type="submit"]'),
+]);
+
+// Obtain and print list of search results
+const results = await page.$$eval('[data-testid="results-list"] div.search-title > a', (nodes) =>
+    nodes.map((node) => ({
+        url: node.href,
+        name: node.innerText,
+    })),
+);
+
+console.log('Results:', results);
+
+// Store data in default dataset
+await Dataset.pushData(results);
+
+// Close browser
+await browser.close();

docs/examples/http_crawler.mdx

@@ -0,0 +1,14 @@
+---
+id: http-crawler
+title: HTTP crawler
+---
+
+import RunnableCodeBlock from '@site/src/components/RunnableCodeBlock';
+import ApiLink from '@site/src/components/ApiLink';
+import HttpCrawlerSource from '!!raw-loader!roa-loader!./http_crawler.ts';
+
+This example demonstrates how to use <ApiLink to="http-crawler/class/HttpCrawler">`HttpCrawler`</ApiLink> to build a HTML crawler that crawls a list of URLs from an external file, load each URL using a plain HTTP request, and save HTML.
+
+<RunnableCodeBlock className="language-js" type="cheerio">
+	{HttpCrawlerSource}
+</RunnableCodeBlock>

docs/examples/http_crawler.ts

@@ -0,0 +1,53 @@
+import { HttpCrawler, log, LogLevel } from 'crawlee';
+
+// Crawlers come with various utilities, e.g. for logging.
+// Here we use debug level of logging to improve the debugging experience.
+// This functionality is optional!
+log.setLevel(LogLevel.DEBUG);
+
+// Create an instance of the HttpCrawler class - a crawler
+// that automatically loads the URLs and saves their HTML.
+const crawler = new HttpCrawler({
+    // The crawler downloads and processes the web pages in parallel, with a concurrency
+    // automatically managed based on the available system memory and CPU (see AutoscaledPool class).
+    // Here we define some hard limits for the concurrency.
+    minConcurrency: 10,
+    maxConcurrency: 50,
+
+    // On error, retry each page at most once.
+    maxRequestRetries: 1,
+
+    // Increase the timeout for processing of each page.
+    requestHandlerTimeoutSecs: 30,
+
+    // Limit to 10 requests per one crawl
+    maxRequestsPerCrawl: 10,
+
+    // This function will be called for each URL to crawl.
+    // It accepts a single parameter, which is an object with options as:
+    // https://crawlee.dev/js/api/http-crawler/interface/HttpCrawlerOptions#requestHandler
+    // We use for demonstration only 2 of them:
+    // - request: an instance of the Request class with information such as the URL that is being crawled and HTTP method
+    // - body: the HTML code of the current page
+    async requestHandler({ pushData, request, body }) {
+        log.debug(`Processing ${request.url}...`);
+
+        // Store the results to the dataset. In local configuration,
+        // the data will be stored as JSON files in ./storage/datasets/default
+        await pushData({
+            url: request.url, // URL of the page
+            body, // HTML code of the page
+        });
+    },
+
+    // This function is called if the page processing failed more than maxRequestRetries + 1 times.
+    failedRequestHandler({ request }) {
+        log.debug(`Request ${request.url} failed twice.`);
+    },
+});
+
+// Run the crawler and wait for it to finish.
+// It will crawl a list of URLs from an external file, load each URL using a plain HTTP request, and save HTML
+await crawler.run(['https://crawlee.dev']);
+
+log.debug('Crawler finished.');

docs/examples/jsdom_crawler.mdx

@@ -0,0 +1,22 @@
+---
+id: jsdom-crawler
+title: JSDOM crawler
+---
+
+import RunnableCodeBlock from '@site/src/components/RunnableCodeBlock';
+import ApiLink from '@site/src/components/ApiLink';
+import JSDOMCrawlerSource from '!!raw-loader!roa-loader!./jsdom_crawler.ts';
+import JSDOMCrawlerRunScriptSource from '!!raw-loader!roa-loader!./jsdom_crawler_react.ts';
+
+This example demonstrates how to use <ApiLink to="jsdom-crawler/class/JSDOMCrawler">`JSDOMCrawler`</ApiLink> to interact with a website using [jsdom](https://www.npmjs.com/package/jsdom) DOM implementation.
+Here the script will open a calculator app from the [React examples](https://reactjs.org/community/examples.html), click `1` `+` `1` `=` and extract the result.
+
+<RunnableCodeBlock className="language-ts" type="cheerio">
+	{JSDOMCrawlerRunScriptSource}
+</RunnableCodeBlock>
+
+In the following example, we use <ApiLink to="jsdom-crawler/class/JSDOMCrawler">`JSDOMCrawler`</ApiLink> to crawl a list of URLs from an external file, load each URL using a plain HTTP request, parse the HTML using the [jsdom](https://www.npmjs.com/package/jsdom) DOM implementation and extract some data from it: the page title and all `h1` tags.
+
+<RunnableCodeBlock className="language-ts" type="cheerio">
+	{JSDOMCrawlerSource}
+</RunnableCodeBlock>

docs/examples/jsdom_crawler.ts

@@ -0,0 +1,62 @@
+import { JSDOMCrawler, log, LogLevel } from 'crawlee';
+
+// Crawlers come with various utilities, e.g. for logging.
+// Here we use debug level of logging to improve the debugging experience.
+// This functionality is optional!
+log.setLevel(LogLevel.DEBUG);
+
+// Create an instance of the JSDOMCrawler class - a crawler
+// that automatically loads the URLs and parses their HTML using the jsdom library.
+const crawler = new JSDOMCrawler({
+    // The crawler downloads and processes the web pages in parallel, with a concurrency
+    // automatically managed based on the available system memory and CPU (see AutoscaledPool class).
+    // Here we define some hard limits for the concurrency.
+    minConcurrency: 10,
+    maxConcurrency: 50,
+
+    // On error, retry each page at most once.
+    maxRequestRetries: 1,
+
+    // Increase the timeout for processing of each page.
+    requestHandlerTimeoutSecs: 30,
+
+    // Limit to 10 requests per one crawl
+    maxRequestsPerCrawl: 10,
+
+    // This function will be called for each URL to crawl.
+    // It accepts a single parameter, which is an object with options as:
+    // https://crawlee.dev/js/api/jsdom-crawler/interface/JSDOMCrawlerOptions#requestHandler
+    // We use for demonstration only 2 of them:
+    // - request: an instance of the Request class with information such as the URL that is being crawled and HTTP method
+    // - window: the JSDOM window object
+    async requestHandler({ pushData, request, window }) {
+        log.debug(`Processing ${request.url}...`);
+
+        // Extract data from the page
+        const title = window.document.title;
+        const h1texts: { text: string }[] = [];
+        window.document.querySelectorAll('h1').forEach((element) => {
+            h1texts.push({
+                text: element.textContent!,
+            });
+        });
+
+        // Store the results to the dataset. In local configuration,
+        // the data will be stored as JSON files in ./storage/datasets/default
+        await pushData({
+            url: request.url,
+            title,
+            h1texts,
+        });
+    },
+
+    // This function is called if the page processing failed more than maxRequestRetries + 1 times.
+    failedRequestHandler({ request }) {
+        log.debug(`Request ${request.url} failed twice.`);
+    },
+});
+
+// Run the crawler and wait for it to finish.
+await crawler.run(['https://crawlee.dev']);
+
+log.debug('Crawler finished.');

docs/examples/jsdom_crawler_react.ts

@@ -0,0 +1,30 @@
+import { JSDOMCrawler, log } from 'crawlee';
+
+// Create an instance of the JSDOMCrawler class - crawler that automatically
+// loads the URLs and parses their HTML using the jsdom library.
+const crawler = new JSDOMCrawler({
+    // Setting the `runScripts` option to `true` allows the crawler to execute client-side
+    // JavaScript code on the page. This is required for some websites (such as the React application in this example), but may pose a security risk.
+    runScripts: true,
+    // This function will be called for each crawled URL.
+    // Here we extract the window object from the options and use it to extract data from the page.
+    requestHandler: async ({ window }) => {
+        const { document } = window;
+        // The `document` object is analogous to the `window.document` object you know from your favourite web browsers.
+        // Thanks to this, you can use the regular browser-side APIs here.
+        document.querySelectorAll('button')[12].click(); // 1
+        document.querySelectorAll('button')[15].click(); // +
+        document.querySelectorAll('button')[12].click(); // 1
+        document.querySelectorAll('button')[18].click(); // =
+
+        const result = document.querySelectorAll('.component-display')[0].childNodes[0] as Element;
+        // The result is passed to the console. Unlike with Playwright or Puppeteer crawlers,
+        // this console call goes to the Node.js console, not the browser console. All the code here runs right in Node.js!
+        log.info(result.innerHTML); // 2
+    },
+});
+
+// Run the crawler and wait for it to finish.
+await crawler.run(['https://ahfarmer.github.io/calculator/']);
+
+log.debug('Crawler finished.');

docs/examples/map.ts

@@ -0,0 +1,28 @@
+import { Dataset, KeyValueStore } from 'crawlee';
+
+const dataset = await Dataset.open<{
+    url: string;
+    headingCount: number;
+}>();
+
+// Seeding the dataset with some data
+await dataset.pushData([
+    {
+        url: 'https://crawlee.dev/',
+        headingCount: 11,
+    },
+    {
+        url: 'https://crawlee.dev/storage',
+        headingCount: 8,
+    },
+    {
+        url: 'https://crawlee.dev/proxy',
+        headingCount: 4,
+    },
+]);
+
+// Calling map function and filtering through mapped items...
+const moreThan5headers = (await dataset.map((item) => item.headingCount)).filter((count) => count > 5);
+
+// Saving the result of map to default key-value store...
+await KeyValueStore.setValue('pages_with_more_than_5_headers', moreThan5headers);

docs/examples/map_and_reduce.mdx

@@ -0,0 +1,80 @@
+---
+id: map-and-reduce
+title: Dataset Map and Reduce methods
+---
+
+import RunnableCodeBlock from '@site/src/components/RunnableCodeBlock';
+import ApiLink from '@site/src/components/ApiLink';
+import MapSource from '!!raw-loader!roa-loader!./map.ts';
+import ReduceSource from '!!raw-loader!roa-loader!./reduce.ts';
+
+This example shows an easy use-case of the <ApiLink to="core/class/Dataset">`Dataset`</ApiLink> <ApiLink to="core/class/Dataset#map">`map`</ApiLink>
+and <ApiLink to="core/class/Dataset#reduce">`reduce`</ApiLink> methods. Both methods can be used to simplify
+the dataset results workflow process. Both can be called on the <ApiLink to="core/class/Dataset">dataset</ApiLink> directly.
+
+Important to mention is that both methods return a new result (`map` returns a new array and `reduce` can return any type) - neither method updates
+the dataset in any way.
+
+Examples for both methods are demonstrated on a simple dataset containing the results scraped from a page: the `URL` and a hypothetical number of
+`h1` - `h3` header elements under the `headingCount` key.
+
+This data structure is stored in the default dataset under `{PROJECT_FOLDER}/storage/datasets/default/`. If you want to simulate the
+functionality, you can use the <ApiLink to="core/class/Dataset#pushData">`dataset.pushData()`</ApiLink>
+method to save the example `JSON array` to your dataset.
+
+```json
+[
+    {
+        "url": "https://crawlee.dev/",
+        "headingCount": 11
+    },
+    {
+        "url": "https://crawlee.dev/storage",
+        "headingCount": 8
+    },
+    {
+        "url": "https://crawlee.dev/proxy",
+        "headingCount": 4
+    }
+]
+```
+
+### Map
+
+The dataset `map` method is very similar to standard Array mapping methods. It produces a new array of values by mapping each value in the existing
+array through a transformation function and an options parameter.
+
+The `map` method used to check if are there more than 5 header elements on each page:
+
+<RunnableCodeBlock className="language-js" type="cheerio">
+	{MapSource}
+</RunnableCodeBlock>
+
+The `moreThan5headers` variable is an array of `headingCount` attributes where the number of headers is greater than 5.
+
+The `map` method's result value saved to the <ApiLink to="core/class/KeyValueStore">`key-value store`</ApiLink> should be:
+
+```javascript
+[11, 8]
+```
+
+### Reduce
+
+The dataset `reduce` method does not produce a new array of values - it reduces a list of values down to a single value. The method iterates through
+the items in the dataset using the <ApiLink to="core/class/Dataset#reduce">`memo` argument</ApiLink>. After performing the necessary
+calculation, the `memo` is sent to the next iteration, while the item just processed is reduced (removed).
+
+Using the `reduce` method to get the total number of headers scraped (all items in the dataset):
+
+<RunnableCodeBlock className="language-js" type="cheerio">
+	{ReduceSource}
+</RunnableCodeBlock>
+
+The original dataset will be reduced to a single value, `pagesHeadingCount`, which contains the count of all headers for all scraped pages (all
+dataset items).
+
+The `reduce` method's result value saved to the <ApiLink to="core/class/KeyValueStore">`key-value store`</ApiLink> should be:
+
+```javascript
+23
+```

docs/examples/playwright_crawler.mdx

@@ -0,0 +1,22 @@
+---
+id: playwright-crawler
+title: Playwright crawler
+---
+
+import RunnableCodeBlock from '@site/src/components/RunnableCodeBlock';
+import ApiLink from '@site/src/components/ApiLink';
+import CrawlSource from '!!raw-loader!roa-loader!./playwright_crawler.ts';
+
+This example demonstrates how to use <ApiLink to="playwright-crawler/class/PlaywrightCrawler">`PlaywrightCrawler`</ApiLink> in combination with <ApiLink to="core/class/RequestQueue">`RequestQueue`</ApiLink> to recursively scrape the [Hacker News website](https://news.ycombinator.com) using headless Chrome / Playwright.
+
+The crawler starts with a single URL, finds links to next pages, enqueues them and continues until no more desired links are available. The results are stored to the default dataset. In local configuration, the results are stored as JSON files in `./storage/datasets/default`.
+
+:::tip
+
+To run this example on the Apify Platform, select the `apify/actor-node-playwright-chrome` image for your Dockerfile.
+
+:::
+
+<RunnableCodeBlock className="language-js">
+	{CrawlSource}
+</RunnableCodeBlock>

docs/examples/playwright_crawler.ts

@@ -0,0 +1,64 @@
+import { PlaywrightCrawler } from 'crawlee';
+
+// Create an instance of the PlaywrightCrawler class - a crawler
+// that automatically loads the URLs in headless Chrome / Playwright.
+const crawler = new PlaywrightCrawler({
+    launchContext: {
+        // Here you can set options that are passed to the playwright .launch() function.
+        launchOptions: {
+            headless: true,
+        },
+    },
+
+    // Stop crawling after several pages
+    maxRequestsPerCrawl: 50,
+
+    // This function will be called for each URL to crawl.
+    // Here you can write the Playwright scripts you are familiar with,
+    // with the exception that browsers and pages are automatically managed by Crawlee.
+    // The function accepts a single parameter, which is an object with a lot of properties,
+    // the most important being:
+    // - request: an instance of the Request class with information such as URL and HTTP method
+    // - page: Playwright's Page object (see https://playwright.dev/docs/api/class-page)
+    async requestHandler({ pushData, request, page, enqueueLinks, log }) {
+        log.info(`Processing ${request.url}...`);
+
+        // A function to be evaluated by Playwright within the browser context.
+        const data = await page.$$eval('.athing', ($posts) => {
+            const scrapedData: { title?: string; rank?: string; href?: string }[] = [];
+
+            // We're getting the title, rank and URL of each post on Hacker News.
+            $posts.forEach(($post) => {
+                scrapedData.push({
+                    title: $post.querySelector<HTMLElement>('.title a')?.innerText,
+                    rank: $post.querySelector<HTMLElement>('.rank')?.innerText,
+                    href: $post.querySelector<HTMLAnchorElement>('.title a')?.href,
+                });
+            });
+
+            return scrapedData;
+        });
+
+        // Store the results to the default dataset.
+        await pushData(data);
+
+        // Find a link to the next page and enqueue it if it exists.
+        const infos = await enqueueLinks({
+            selector: '.morelink',
+        });
+
+        if (infos.processedRequests.length === 0) log.info(`${request.url} is the last page!`);
+    },
+
+    // This function is called if the page processing failed more than maxRequestRetries+1 times.
+    failedRequestHandler({ request, log }) {
+        log.info(`Request ${request.url} failed too many times.`);
+    },
+});
+
+await crawler.addRequests(['https://news.ycombinator.com/']);
+
+// Run the crawler and wait for it to finish.
+await crawler.run();
+
+console.log('Crawler finished.');

docs/examples/playwright_crawler_firefox.mdx

@@ -0,0 +1,22 @@
+---
+id: playwright-crawler-firefox
+title: Using Firefox browser with Playwright crawler
+---
+
+import RunnableCodeBlock from '@site/src/components/RunnableCodeBlock';
+import ApiLink from '@site/src/components/ApiLink';
+import PlaywrightFirefoxSource from '!!raw-loader!roa-loader!./playwright_crawler_firefox.ts';
+
+This example demonstrates how to use <ApiLink to="playwright-crawler/class/PlaywrightCrawler">`PlaywrightCrawler`</ApiLink> with headless Firefox browser.
+
+:::tip
+
+To run this example on the Apify Platform, select the `apify/actor-node-playwright-firefox` image for your Dockerfile.
+
+:::
+
+<RunnableCodeBlock className="language-js">
+    {PlaywrightFirefoxSource}
+</RunnableCodeBlock>
+
+To see a real-world example of how to use <ApiLink to="playwright-crawler/class/PlaywrightCrawler">`PlaywrightCrawler`</ApiLink> in combination with <ApiLink to="core/class/RequestQueue">`RequestQueue`</ApiLink> to recursively scrape the [Hacker News website](https://news.ycombinator.com) check out the [`Playwright crawler example`](./playwright-crawler).

docs/examples/playwright_crawler_firefox.ts

@@ -0,0 +1,22 @@
+import { PlaywrightCrawler } from 'crawlee';
+import { firefox } from 'playwright';
+
+// Create an instance of the PlaywrightCrawler class.
+const crawler = new PlaywrightCrawler({
+    launchContext: {
+        // Set the Firefox browser to be used by the crawler.
+        // If launcher option is not specified here,
+        // default Chromium browser will be used.
+        launcher: firefox,
+    },
+    async requestHandler({ request, page, log }) {
+        const pageTitle = await page.title();
+
+        log.info(`URL: ${request.loadedUrl} | Page title: ${pageTitle}`);
+    },
+});
+
+await crawler.addRequests(['https://example.com']);
+
+// Run the crawler and wait for it to finish.
+await crawler.run();

docs/examples/puppeteer_capture_screenshot.mdx

@@ -0,0 +1,77 @@
+---
+id: capture-screenshot
+title: Capture a screenshot using Puppeteer
+---
+
+import Tabs from '@theme/Tabs';
+import TabItem from '@theme/TabItem';
+import RunnableCodeBlock from '@site/src/components/RunnableCodeBlock';
+import ApiLink from '@site/src/components/ApiLink';
+
+import PuppeteerPageScreenshotSource from '!!raw-loader!roa-loader!./puppeteer_page_screenshot.ts';
+import PuppeteerCrawlerUtilsSnapshotSource from '!!raw-loader!roa-loader!./puppeteer_crawler_utils_snapshot.ts';
+import PuppeteerCrawlerPageScreenshotSource from '!!raw-loader!roa-loader!./puppeteer_crawler_page_screenshot.ts';
+import PuppeteerCrawlerCrawlerUtilsSnapshotSource from '!!raw-loader!roa-loader!./puppeteer_crawler_crawler_utils_snapshot.ts';
+
+## Using Puppeteer directly
+
+:::tip
+
+To run this example on the Apify Platform, select the `apify/actor-node-puppeteer-chrome` image for your Dockerfile.
+
+:::
+
+This example captures a screenshot of a web page using `Puppeteer`. It would look almost exactly the same with `Playwright`.
+
+<Tabs groupId="puppeteer-capture-screenshot">
+<TabItem value="pagescreenshot" label="Page Screenshot">
+
+Using `page.screenshot()`:
+
+<RunnableCodeBlock className="language-js" type="puppeteer">
+	{PuppeteerPageScreenshotSource}
+</RunnableCodeBlock>
+
+</TabItem>
+
+<TabItem value="crawlerutilsscreenshot" label="Crawler Utils Screenshot" default>
+
+Using `utils.puppeteer.saveSnapshot()`:
+
+<RunnableCodeBlock className="language-js" type="puppeteer">
+	{PuppeteerCrawlerUtilsSnapshotSource}
+</RunnableCodeBlock>
+
+</TabItem>
+</Tabs>
+
+## Using `PuppeteerCrawler`
+
+This example captures a screenshot of multiple web pages when using `PuppeteerCrawler`:
+
+<Tabs groupId="puppeteer-capture-screenshot">
+<TabItem value="pagescreenshot" label="Page Screenshot">
+
+Using `page.screenshot()`:
+
+<RunnableCodeBlock className="language-js" type="puppeteer">
+	{PuppeteerCrawlerPageScreenshotSource}
+</RunnableCodeBlock>
+
+</TabItem>
+
+<TabItem value="crawlerutilsscreenshot" label="Crawler Utils Screenshot" default>
+
+Using the context-aware <ApiLink to="puppeteer-crawler/namespace/puppeteerUtils#saveSnapshot">`saveSnapshot()`</ApiLink> utility:
+
+<RunnableCodeBlock className="language-js" type="puppeteer">
+	{PuppeteerCrawlerCrawlerUtilsSnapshotSource}
+</RunnableCodeBlock>
+
+</TabItem>
+</Tabs>
+
+To take full page screenshot using puppeteer we need to pass parameter `fullPage` as `true`in the `screenshot()`: `page.screenshot(fullPage: true)`
+
+In both examples using `page.screenshot()`, a `key` variable is created based on the URL of the web page. This variable is used as the key when saving
+each screenshot into a key-value store.

docs/examples/puppeteer_crawler.mdx

@@ -0,0 +1,25 @@
+---
+id: puppeteer-crawler
+title: Puppeteer crawler
+---
+
+import RunnableCodeBlock from '@site/src/components/RunnableCodeBlock';
+import ApiLink from '@site/src/components/ApiLink';
+import CrawlSource from '!!raw-loader!roa-loader!./puppeteer_crawler.ts';
+
+This example demonstrates how to use <ApiLink to="puppeteer-crawler/class/PuppeteerCrawler">`PuppeteerCrawler`</ApiLink> in combination
+with <ApiLink to="core/class/RequestQueue">`RequestQueue`</ApiLink>
+to recursively scrape the [Hacker News website](https://news.ycombinator.com) using headless Chrome / Puppeteer.
+
+The crawler starts with a single URL, finds links to next pages, enqueues them and continues until no more desired links are available. The results
+are stored to the default dataset. In local configuration, the results are stored as JSON files in `./storage/datasets/default`
+
+:::tip
+
+To run this example on the Apify Platform, select the `apify/actor-node-puppeteer-chrome` image for your Dockerfile.
+
+:::
+
+<RunnableCodeBlock className="language-js" type="puppeteer">
+	{CrawlSource}
+</RunnableCodeBlock>

docs/examples/puppeteer_crawler.ts

@@ -0,0 +1,64 @@
+import { PuppeteerCrawler } from 'crawlee';
+
+// Create an instance of the PuppeteerCrawler class - a crawler
+// that automatically loads the URLs in headless Chrome / Puppeteer.
+const crawler = new PuppeteerCrawler({
+    // Here you can set options that are passed to the launchPuppeteer() function.
+    launchContext: {
+        launchOptions: {
+            headless: true,
+            // Other Puppeteer options
+        },
+    },
+
+    // Stop crawling after several pages
+    maxRequestsPerCrawl: 50,
+
+    // This function will be called for each URL to crawl.
+    // Here you can write the Puppeteer scripts you are familiar with,
+    // with the exception that browsers and pages are automatically managed by Crawlee.
+    // The function accepts a single parameter, which is an object with the following fields:
+    // - request: an instance of the Request class with information such as URL and HTTP method
+    // - page: Puppeteer's Page object (see https://pptr.dev/#show=api-class-page)
+    async requestHandler({ pushData, request, page, enqueueLinks, log }) {
+        log.info(`Processing ${request.url}...`);
+
+        // A function to be evaluated by Puppeteer within the browser context.
+        const data = await page.$$eval('.athing', ($posts) => {
+            const scrapedData: { title?: string; rank?: string; href?: string }[] = [];
+
+            // We're getting the title, rank and URL of each post on Hacker News.
+            $posts.forEach(($post) => {
+                scrapedData.push({
+                    title: $post.querySelector<HTMLElement>('.title a')?.innerText,
+                    rank: $post.querySelector<HTMLElement>('.rank')?.innerText,
+                    href: $post.querySelector<HTMLAnchorElement>('.title a')?.href,
+                });
+            });
+
+            return scrapedData;
+        });
+
+        // Store the results to the default dataset.
+        await pushData(data);
+
+        // Find a link to the next page and enqueue it if it exists.
+        const infos = await enqueueLinks({
+            selector: '.morelink',
+        });
+
+        if (infos.processedRequests.length === 0) log.info(`${request.url} is the last page!`);
+    },
+
+    // This function is called if the page processing failed more than maxRequestRetries+1 times.
+    failedRequestHandler({ request, log }) {
+        log.error(`Request ${request.url} failed too many times.`);
+    },
+});
+
+await crawler.addRequests(['https://news.ycombinator.com/']);
+
+// Run the crawler and wait for it to finish.
+await crawler.run();
+
+console.log('Crawler finished.');

docs/examples/puppeteer_crawler_crawler_utils_snapshot.ts

@@ -0,0 +1,20 @@
+import { PuppeteerCrawler } from 'crawlee';
+
+// Create a PuppeteerCrawler
+const crawler = new PuppeteerCrawler({
+    async requestHandler({ request, saveSnapshot }) {
+        // Convert the URL into a valid key
+        const key = request.url.replace(/[:/]/g, '_');
+        // Capture the screenshot
+        await saveSnapshot({ key, saveHtml: false });
+    },
+});
+
+await crawler.addRequests([
+    { url: 'http://www.example.com/page-1' },
+    { url: 'http://www.example.com/page-2' },
+    { url: 'http://www.example.com/page-3' },
+]);
+
+// Run the crawler
+await crawler.run();

docs/examples/puppeteer_crawler_page_screenshot.ts

@@ -0,0 +1,22 @@
+import { PuppeteerCrawler, KeyValueStore } from 'crawlee';
+
+// Create a PuppeteerCrawler
+const crawler = new PuppeteerCrawler({
+    async requestHandler({ request, page }) {
+        // Capture the screenshot with Puppeteer
+        const screenshot = await page.screenshot();
+        // Convert the URL into a valid key
+        const key = request.url.replace(/[:/]/g, '_');
+        // Save the screenshot to the default key-value store
+        await KeyValueStore.setValue(key, screenshot, { contentType: 'image/png' });
+    },
+});
+
+await crawler.addRequests([
+    { url: 'http://www.example.com/page-1' },
+    { url: 'http://www.example.com/page-2' },
+    { url: 'http://www.example.com/page-3' },
+]);
+
+// Run the crawler
+await crawler.run();

docs/examples/puppeteer_crawler_utils_snapshot.ts

@@ -0,0 +1,17 @@
+import { launchPuppeteer, utils } from 'crawlee';
+
+const url = 'http://www.example.com/';
+// Start a browser
+const browser = await launchPuppeteer();
+
+// Open new tab in the browser
+const page = await browser.newPage();
+
+// Navigate to the URL
+await page.goto(url);
+
+// Capture the screenshot
+await utils.puppeteer.saveSnapshot(page, { key: 'my-key', saveHtml: false });
+
+// Close Puppeteer
+await browser.close();

docs/examples/puppeteer_page_screenshot.ts

@@ -0,0 +1,22 @@
+import { KeyValueStore, launchPuppeteer } from 'crawlee';
+
+const keyValueStore = await KeyValueStore.open();
+
+const url = 'https://crawlee.dev';
+// Start a browser
+const browser = await launchPuppeteer();
+
+// Open new tab in the browser
+const page = await browser.newPage();
+
+// Navigate to the URL
+await page.goto(url);
+
+// Capture the screenshot
+const screenshot = await page.screenshot();
+
+// Save the screenshot to the default key-value store
+await keyValueStore.setValue('my-key', screenshot, { contentType: 'image/png' });
+
+// Close Puppeteer
+await browser.close();

docs/examples/puppeteer_recursive_crawl.mdx

@@ -0,0 +1,20 @@
+---
+id: puppeteer-recursive-crawl
+title: Puppeteer recursive crawl
+---
+
+import RunnableCodeBlock from '@site/src/components/RunnableCodeBlock';
+import ApiLink from '@site/src/components/ApiLink';
+import CrawlSource from '!!raw-loader!roa-loader!./puppeteer_recursive_crawl.ts';
+
+Run the following example to perform a recursive crawl of a website using <ApiLink to="puppeteer-crawler/class/PuppeteerCrawler">`PuppeteerCrawler`</ApiLink>.
+
+:::tip
+
+To run this example on the Apify Platform, select the `apify/actor-node-puppeteer-chrome` image for your Dockerfile.
+
+:::
+
+<RunnableCodeBlock className="language-js" type="puppeteer">
+	{CrawlSource}
+</RunnableCodeBlock>

docs/examples/puppeteer_recursive_crawl.ts

@@ -0,0 +1,17 @@
+import { PuppeteerCrawler } from 'crawlee';
+
+const crawler = new PuppeteerCrawler({
+    async requestHandler({ request, page, enqueueLinks, log }) {
+        const title = await page.title();
+        log.info(`Title of ${request.url}: ${title}`);
+
+        await enqueueLinks({
+            globs: ['http?(s)://www.iana.org/**'],
+        });
+    },
+    maxRequestsPerCrawl: 10,
+});
+
+await crawler.addRequests(['https://www.iana.org/']);
+
+await crawler.run();

docs/examples/reduce.ts

@@ -0,0 +1,30 @@
+import { Dataset, KeyValueStore } from 'crawlee';
+
+const dataset = await Dataset.open<{
+    url: string;
+    headingCount: number;
+}>();
+
+// Seeding the dataset with some data
+await dataset.pushData([
+    {
+        url: 'https://crawlee.dev/',
+        headingCount: 11,
+    },
+    {
+        url: 'https://crawlee.dev/storage',
+        headingCount: 8,
+    },
+    {
+        url: 'https://crawlee.dev/proxy',
+        headingCount: 4,
+    },
+]);
+
+// calling reduce function and using memo to calculate number of headers
+const pagesHeadingCount = await dataset.reduce((memo, value) => {
+    return memo + value.headingCount;
+}, 0);
+
+// saving result of map to default Key-value store
+await KeyValueStore.setValue('pages_heading_count', pagesHeadingCount);

docs/examples/skip-navigation.mdx

@@ -0,0 +1,23 @@
+---
+id: skip-navigation
+title: Skipping navigations for certain requests
+---
+
+import RunnableCodeBlock from '@site/src/components/RunnableCodeBlock';
+import ApiLink from '@site/src/components/ApiLink';
+import SkipNavigationSource from '!!raw-loader!roa-loader!./skip-navigation.ts';
+
+While crawling a website, you may encounter certain resources you'd like to save, but don't need the full power of a crawler to do so (like images delivered through a CDN).
+
+By combining the <ApiLink to="core/class/Request#skipNavigation">`Request#skipNavigation`</ApiLink> option with <ApiLink to="basic-crawler/interface/BasicCrawlingContext#sendRequest">`sendRequest`</ApiLink>, we can fetch the image from the CDN, and save it to our key-value store without needing to use the full crawler.
+
+
+:::info
+
+For this example, we are using the <ApiLink to="playwright-crawler/class/PlaywrightCrawler">`PlaywrightCrawler`</ApiLink> to showcase this, but this is available on all the crawlers we provide.
+
+:::
+
+<RunnableCodeBlock className="language-js">
+	{SkipNavigationSource}
+</RunnableCodeBlock>

docs/examples/skip-navigation.ts

@@ -0,0 +1,31 @@
+import { PlaywrightCrawler, KeyValueStore } from 'crawlee';
+
+// Create a key value store for all images we find
+const imageStore = await KeyValueStore.open('images');
+
+const crawler = new PlaywrightCrawler({
+    async requestHandler({ request, page, sendRequest }) {
+        // The request should have the navigation skipped
+        if (request.skipNavigation) {
+            // Request the image and get its buffer back
+            const imageResponse = await sendRequest({ responseType: 'buffer' });
+
+            // Save the image in the key-value store
+            await imageStore.setValue(`${request.userData.key}.png`, imageResponse.body);
+
+            // Prevent executing the rest of the code as we do not need it
+            return;
+        }
+
+        // Get all the image sources in the current page
+        const images = await page.$$eval('img', (imgs) => imgs.map((img) => img.src));
+
+        // Add all the urls as requests for the crawler, giving each image a key
+        await crawler.addRequests(images.map((url, i) => ({ url, skipNavigation: true, userData: { key: i } })));
+    },
+});
+
+await crawler.addRequests(['https://crawlee.dev']);
+
+// Run the crawler
+await crawler.run();

docs/experiments/request_locking.mdx

@@ -0,0 +1,146 @@
+---
+id: experiments-request-locking
+title: Request Locking
+description: Parallelize crawlers with ease using request locking
+---
+
+import ApiLink from '@site/src/components/ApiLink';
+
+:::tip Release announcement
+
+As of **May 2024** (`crawlee` version `3.10.0`), this experiment is now enabled by default! With that said, if you encounter issues you can:
+
+- set `requestLocking` to `false` in the `experiments` object of your crawler options
+- update all imports of `RequestQueue` to `RequestQueueV1`
+- open an issue on our [GitHub repository](https://github.com/apify/crawlee)
+
+The content below is kept for documentation purposes.
+If you're interested in the changes, you can read the [blog post about the new Request Queue storage system on the Apify blog](https://blog.apify.com/new-apify-request-queue/).
+
+:::
+
+---
+
+:::caution
+
+This is an experimental feature. While we welcome testers, keep in mind that it is currently not recommended to use this in production.
+
+The API is subject to change, and we might introduce breaking changes in the future.
+
+Should you be using this, feel free to open issues on our [GitHub repository](https://github.com/apify/crawlee), and we'll take a look.
+
+:::
+
+Starting with `crawlee` version `3.5.5`, we have introduced a new crawler option that lets you enable using a new request locking
+API. With this API, you will be able to pass a `RequestQueue` to multiple crawlers to parallelize the crawling process.
+
+:::info Keep in mind
+
+The request queue that supports request locking is currently exported via the `RequestQueueV2` class. Once the experiment is over, this class will replace
+the current `RequestQueue` class
+
+:::
+
+## How to enable the experiment
+
+### In crawlers
+
+:::note
+
+This example shows how to enable the experiment in the <ApiLink to="cheerio-crawler/class/CheerioCrawler">`CheerioCrawler`</ApiLink>,
+but you can apply this to any crawler type.
+
+:::
+
+```ts
+import { CheerioCrawler } from 'crawlee';
+
+const crawler = new CheerioCrawler({
+    // highlight-next-line
+    experiments: {
+        // highlight-next-line
+        requestLocking: true,
+        // highlight-next-line
+    },
+    async requestHandler({ $, request }) {
+        const title = $('title').text();
+        console.log(`The title of "${request.url}" is: ${title}.`);
+    },
+});
+
+await crawler.run(['https://crawlee.dev']);
+```
+
+### Outside crawlers (to setup your own request queue that supports locking)
+
+Previously, you would import `RequestQueue` from `crawlee`. To switch to the queue that supports locking, you need to import `RequestQueueV2` instead.
+
+```ts
+// highlight-next-line
+import { RequestQueueV2 } from 'crawlee';
+
+const queue = await RequestQueueV2.open('my-locking-queue');
+await queue.addRequests([
+    { url: 'https://crawlee.dev' },
+    { url: 'https://crawlee.dev/js/docs' },
+    { url: 'https://crawlee.dev/js/api' },
+]);
+```
+
+### Using the new request queue in crawlers
+
+If you make your own request queue that supports locking, you will also need to enable the experiment in your crawlers.
+
+:::danger
+
+If you do not enable the experiment, you will receive a runtime error and the crawler will not start.
+
+:::
+
+```ts
+import { CheerioCrawler, RequestQueueV2 } from 'crawlee';
+
+// highlight-next-line
+const queue = await RequestQueueV2.open('my-locking-queue');
+
+const crawler = new CheerioCrawler({
+    // highlight-next-line
+    experiments: {
+        // highlight-next-line
+        requestLocking: true,
+        // highlight-next-line
+    },
+    // highlight-next-line
+    requestQueue: queue,
+    async requestHandler({ $, request }) {
+        const title = $('title').text();
+        console.log(`The title of "${request.url}" is: ${title}.`);
+    },
+});
+
+await crawler.run();
+```
+
+## Other changes
+
+:::info
+
+This section is only useful if you're a tinkerer and want to see what's going on under the hood.
+
+:::
+
+In order to facilitate the new request locking API, as well as keep both the current request queue logic and the new, locking based request queue
+logic, we have implemented a common starting point called `RequestProvider`.
+
+This class implements almost all functions by default, but expects you, the developer, to implement the following methods:
+`fetchNextRequest` and `ensureHeadIsNotEmpty`. These methods are responsible for loading and returning requests to process,
+and tell crawlers if there are more requests to process.
+
+You can use this base class to implement your own request providers if you need to fetch requests from a different source.
+
+:::tip
+
+We recommend you use TypeScript when implementing your own request provider, as it comes with suggestions for the abstract methods, as well as
+giving you the exact types you need to return.
+
+:::

docs/experiments/systemInfoV2.mdx

@@ -0,0 +1,95 @@
+---
+id: experiments-system-information-v2
+title: System Information V2
+description: Improved autoscaling through cgroup aware metric collection.
+---
+
+import ApiLink from '@site/src/components/ApiLink';
+
+:::caution
+
+This is an experimental feature. While we welcome testers, keep in mind that it is currently not recommended to use this in production.
+
+The API is subject to change, and we might introduce breaking changes in the future.
+
+Should you be using this, feel free to open issues on our [GitHub repository](https://github.com/apify/crawlee), and we'll take a look.
+
+:::
+
+Starting with the newest `crawlee` beta, we have introduced a new crawler option that enables an improved metric collection system.
+This new system should collect cpu and memory metrics more accurately in containerised environments by checking for cgroup enforce limits. 
+
+## How to enable the experiment
+
+:::note
+
+This example shows how to enable the experiment in the <ApiLink to="cheerio-crawler/class/CheerioCrawler">`CheerioCrawler`</ApiLink>,
+but you can apply this to any crawler type.
+
+:::
+
+```ts
+import { CheerioCrawler, Configuration } from 'crawlee';
+
+Configuration.set('systemInfoV2', true);
+
+const crawler = new CheerioCrawler({
+    async requestHandler({ $, request }) {
+        const title = $('title').text();
+        console.log(`The title of "${request.url}" is: ${title}.`);
+    },
+});
+
+await crawler.run(['https://crawlee.dev']);
+```
+
+## Other changes
+
+:::info
+
+This section is only useful if you're a tinkerer and want to see what's going on under the hood.
+
+:::
+
+The existing solution checked the bare metal metrics for how much cpu and memory was being used and how much headroom was available. 
+This is an intuitive solution but unfortunately doesnt account for when there is an external limit on the amount of resources a process can consume.
+This is often the case in containerized environments where each container will have a quota for its cpu and memory usage.
+
+This experiment attempts to address this issue by introducing a new `isContainerized()` utility function and changing the way resources are collected 
+when a container is detected.
+
+:::note
+
+This `isContainerized()` function is very similar to the existing `isDocker()` function however for now they both work side by side.
+If this experiment is successful, `isDocker()` may eventually be deprecated in favour of `isContainerized()`.
+
+:::
+
+### Cgroup detection
+
+On linux, to detect if cgroup is available, we check if there is a directory at `/sys/fs/cgroup`.
+If the directory exists, a version of cgroup is installed.
+Next we check the version of cgroup installed by checking for a directory at `/sys/fs/cgroup/memory/`.
+If it exists, cgroup V1 is installed. If it is missing, it is assumed cgroup V2 is installed.
+
+### CPU metric collection
+
+The existing solution worked by checking the fraction of cpu idle ticks to the total number of cpu ticks since the last profile.
+If 100000 ticks elapse and 5000 were idle, the cpu is at 95% utilisation.
+
+In this experiment, the method of cpu load calculation depends on the result of `isContainerized()` or if set, the `CRAWLEE_CONTAINERIZED` environment variable.
+If `isContainerized()` returns true, the new cgroup aware metric collection will be used over the "bare metal" numbers.
+This works by inspecting the `/sys/fs/cgroup/cpuacct/cpuacct.usage`, `/sys/fs/cgroup/cpu/cpu.cfs_quota_us` and `/sys/fs/cgroup/cpu/cpu.cfs_period_us`
+files for cgroup V1 and the `/sys/fs/cgroup/cpu.stat` and `/sys/fs/cgroup/cpu.max` files for cgroup V2.
+The actual cpu usage figure is calculated in the same manner as the "bare metal" figure by comparing the total number of ticks elapsed to the number
+of idle ticks between profiles but by using the figures from the cgroup files. 
+If no cgroup quota is enforced, the "bare metal" numbers will be used.
+
+### Memory metric collection
+
+The existing solution was already cgroup aware however an improvement has been made to memory metric collection when running on windows.
+The existing solution used an external package `apify/ps-tree` to find the amount of memory crawlee and any child processes were using. 
+On Windows, this package used the depreciated "WMIC" command line utility to determine memory usage.
+
+In this experiment, `apify/ps-tree` has been removed and replaced by the `packages/utils/src/internals/ps-tree.ts` file. This works in much the 
+same manner however, instead of using "WMIC", it uses "powershell" to collect the same data.

docs/guides/avoid_blocking.mdx

@@ -0,0 +1,71 @@
+---
+id: avoid-blocking
+title: Avoid getting blocked
+description: How to avoid getting blocked when scraping
+---
+
+import Tabs from '@theme/Tabs';
+import TabItem from '@theme/TabItem';
+import CodeBlock from '@theme/CodeBlock';
+
+import PlaywrightSource from '!!raw-loader!./avoid_blocking_playwright.ts';
+import PuppeteerSource from '!!raw-loader!./avoid_blocking_puppeteer';
+import PlaywrightFingerprintsOffSource from '!!raw-loader!./avoid_blocking_playwright_fingerprints_off.ts';
+import PuppeteerFingerprintsOffSource from '!!raw-loader!./avoid_blocking_puppeteer_fingerprints_off.ts';
+import PlaywrightCamoufox from '!!raw-loader!./avoid_blocking_camoufox.ts';
+
+A scraper might get blocked for numerous reasons. Let's narrow it down to the two main ones. The first is a bad or blocked IP address. You can learn about this topic in the [proxy management guide](./proxy-management). The second reason is [browser fingerprints](https://pixelprivacy.com/resources/browser-fingerprinting/) (or signatures), which we will explore more in this guide. Check the [Apify Academy anti-scraping course](https://docs.apify.com/academy/anti-scraping) to gain a deeper theoretical understanding of blocking and learn a few tips and tricks.
+
+Browser fingerprint is a collection of browser attributes and significant features that can show if our browser is a bot or a real user. Moreover, most browsers have these unique features that allow the website to track the browser even within different IP addresses. This is the main reason why scrapers should change browser fingerprints while doing browser-based scraping. In return, it should significantly reduce the blocking.
+
+## Using browser fingerprints
+
+Changing browser fingerprints can be a tedious job. Luckily, Crawlee provides this feature with zero configuration necessary - the usage of fingerprints is enabled by default and available in `PlaywrightCrawler` and `PuppeteerCrawler`. So whenever we build a scraper that is using one of these crawlers - the fingerprints are going to be generated for the default browser and the operating system out of the box.
+
+## Customizing browser fingerprints
+
+In certain cases we want to narrow down the fingerprints used - e.g. specify a certain operating system, locale or browser. This is also possible with Crawlee - the crawler can have the generation algorithm customized to reflect the particular browser version and many more. Let's take a look at the examples bellow:
+
+<Tabs groupId="avoid_blocking">
+    <TabItem value="playwright" label="PlaywrightCrawler" default>
+        <CodeBlock language="js">
+            {PlaywrightSource}
+        </CodeBlock>
+    </TabItem>
+    <TabItem value="puppeteer" label="PuppeteerCrawler">
+        <CodeBlock language="js">
+            {PuppeteerSource}
+        </CodeBlock>
+    </TabItem>
+</Tabs>
+
+## Disabling browser fingerprints
+
+On the contrary, sometimes we want to entirely disable the usage of browser fingerprints. This is easy to do with Crawlee too. All we have to do is set the `useFingerprints` option of the `browserPoolOptions` to `false`:
+
+<Tabs groupId="avoid_blocking">
+    <TabItem value="playwright" label="PlaywrightCrawler" default>
+        <CodeBlock language="js">
+            {PlaywrightFingerprintsOffSource}
+        </CodeBlock>
+    </TabItem>
+    <TabItem value="puppeteer" label="PuppeteerCrawler">
+        <CodeBlock language="js">
+            {PuppeteerFingerprintsOffSource}
+        </CodeBlock>
+    </TabItem>
+</Tabs>
+
+## Camoufox
+
+For some protections, using our integrated solutions is not enough, one example could be the Cloudflare challenge. For such pages, you can try [Camoufox](https://camoufox.com/), a custom stealthy build of Firefox for web scraping. It might not get you through the challenge automatically, but with our `handleCloudflareChallenge` helper, it should be able to successfully mimic the required user action and get you through it.
+
+<CodeBlock language="ts">
+    {PlaywrightCamoufox}
+</CodeBlock>
+
+**Related links**
+
+- [Fingerprint Suite Docs](https://github.com/apify/fingerprint-suite)
+- [Apify Academy anti-scraping course](https://docs.apify.com/academy/anti-scraping)
+- [Camoufox JS wrapper](https://github.com/apify/camoufox-js)

docs/guides/avoid_blocking_camoufox.ts

@@ -0,0 +1,22 @@
+import { PlaywrightCrawler } from 'crawlee';
+import { launchOptions } from 'camoufox-js';
+import { firefox } from 'playwright';
+
+const crawler = new PlaywrightCrawler({
+    postNavigationHooks: [
+        async ({ handleCloudflareChallenge }) => {
+            await handleCloudflareChallenge();
+        },
+    ],
+    browserPoolOptions: {
+        // Disable the default fingerprint spoofing to avoid conflicts with Camoufox.
+        useFingerprints: false,
+    },
+    launchContext: {
+        launcher: firefox,
+        launchOptions: await launchOptions({
+            headless: true,
+        }),
+    },
+    // ...
+});

docs/guides/avoid_blocking_playwright.ts

@@ -0,0 +1,21 @@
+import { PlaywrightCrawler } from 'crawlee';
+import { BrowserName, DeviceCategory, OperatingSystemsName } from '@crawlee/browser-pool';
+
+const crawler = new PlaywrightCrawler({
+    browserPoolOptions: {
+        useFingerprints: true, // this is the default
+        fingerprintOptions: {
+            fingerprintGeneratorOptions: {
+                browsers: [
+                    {
+                        name: BrowserName.edge,
+                        minVersion: 96,
+                    },
+                ],
+                devices: [DeviceCategory.desktop],
+                operatingSystems: [OperatingSystemsName.windows],
+            },
+        },
+    },
+    // ...
+});

docs/guides/avoid_blocking_playwright_fingerprints_off.ts

@@ -0,0 +1,8 @@
+import { PlaywrightCrawler } from 'crawlee';
+
+const crawler = new PlaywrightCrawler({
+    browserPoolOptions: {
+        useFingerprints: false,
+    },
+    // ...
+});

docs/guides/avoid_blocking_puppeteer.ts

@@ -0,0 +1,16 @@
+import { PuppeteerCrawler } from 'crawlee';
+import { BrowserName, DeviceCategory } from '@crawlee/browser-pool';
+
+const crawler = new PuppeteerCrawler({
+    browserPoolOptions: {
+        useFingerprints: true, // this is the default
+        fingerprintOptions: {
+            fingerprintGeneratorOptions: {
+                browsers: [BrowserName.chrome, BrowserName.firefox],
+                devices: [DeviceCategory.mobile],
+                locales: ['en-US'],
+            },
+        },
+    },
+    // ...
+});