From f1ba9c6c366e07d7ddb78d305988fde20bf3979b Mon Sep 17 00:00:00 2001 From: wehub-resource-sync Date: Mon, 13 Jul 2026 12:23:39 +0800 Subject: [PATCH] chore: import upstream snapshot with attribution --- .c8rc.json | 15 + .editorconfig | 17 + .github/workflows/.keep | 0 .github/workflows/_test.yml | 41 + .github/workflows/ci.yml | 24 + .github/workflows/oss-image.yml | 87 + .gitignore | 81 + .gitmodules | 0 .nvmrc | 1 + .vscode/exensions.json | 10 + .vscode/launch.json | 144 + .vscode/settings.json | 32 + .vscode/tasks.json | 64 + CLAUDE.md | 126 + CONTRIBUTING.md | 132 + Dockerfile | 39 + LICENSE | 193 + README.md | 288 + README.wehub.md | 7 + architecture.md | 83 + bin/compile-secret.js | 7 + bin/extract-secret.js | 7 + cookbooks.md | 215 + docker-compose.yml | 31 + download-external-assets.sh | 50 + fixtures/sample.html | 96 + integrity-check.cjs | 11 + package-lock.json | 7996 ++++++++++++++ package.json | 81 + public/favicon.ico | Bin 0 -> 14671 bytes public/robots.txt | 14 + src/3rd-party/anthropic.ts | 177 + src/3rd-party/cloud-flare.ts | 22 + src/3rd-party/common-serp.ts | 179 + src/3rd-party/google-gemini.ts | 774 ++ src/3rd-party/internal-cloudrun.ts | 35 + src/3rd-party/jina-embeddings.ts | 102 + src/3rd-party/open-router.ts | 122 + src/3rd-party/openai-compat.ts | 119 + src/3rd-party/openai.ts | 54 + src/3rd-party/replicate.ts | 226 + src/3rd-party/serper-search.ts | 194 + src/api/crawler.ts | 1808 ++++ src/api/searcher.ts | 950 ++ src/api/serp.ts | 451 + src/config.ts | 13 + src/db/bucket-storage.ts | 315 + src/db/models.ts | 293 + src/db/noop-storage.ts | 169 + src/dto/base-auth.ts | 138 + src/dto/crawler-options.ts | 1027 ++ src/dto/turndown-tweakable-options.ts | 61 + src/fetch.d.ts | 13 + src/lib/filtered-stream.ts | 59 + src/lib/json-parse-stream.ts | 1535 +++ src/lib/koa-compress.ts | 243 + src/lib/pseudo-boolean.ts | 49 + src/lib/transform-server-event-stream.ts | 170 + src/scripts/smoke-test-llm.ts | 55 + src/services/alt-text.ts | 190 + src/services/async-context.ts | 10 + src/services/binary-extractor.ts | 375 + src/services/blackhole-detector.ts | 82 + src/services/bogo-sites.ts | 105 + src/services/canvas.ts | 225 + src/services/cf-browser-rendering.ts | 54 + src/services/common-iminterrogate/base.ts | 241 + src/services/common-iminterrogate/index.ts | 5 + .../common-iminterrogate/instruct-blip.ts | 129 + src/services/common-iminterrogate/llms.ts | 85 + src/services/common-iminterrogate/registry.ts | 136 + src/services/common-llm/base.ts | 1098 ++ src/services/common-llm/google-gemini.ts | 1022 ++ src/services/common-llm/gpt-1106.ts | 239 + src/services/common-llm/gpt-35.ts | 946 ++ src/services/common-llm/gpt-4.ts | 61 + src/services/common-llm/index.ts | 6 + src/services/common-llm/jina-vlm.ts | 40 + src/services/common-llm/misc.ts | 315 + src/services/common-llm/open-router.ts | 919 ++ src/services/common-llm/prompt-profile.ts | 77 + src/services/common-llm/reader-lm.ts | 41 + src/services/common-llm/registry.ts | 547 + src/services/common-llm/vertex-gemini.ts | 216 + src/services/curl.ts | 643 ++ src/services/default-bucket.ts | 63 + src/services/envconfig.ts | 67 + src/services/errors.ts | 87 + src/services/finalizer.ts | 41 + src/services/geoip.ts | 134 + src/services/ipasn.ts | 79 + src/services/jsdom.ts | 613 ++ src/services/lm.ts | 152 + src/services/logger.ts | 57 + src/services/markify.ts | 854 ++ src/services/minimal-stealth.js | 603 ++ src/services/misc.ts | 126 + src/services/pdf-extract.ts | 396 + src/services/proxy-provider/brightdata.ts | 63 + src/services/proxy-provider/index.ts | 123 + src/services/proxy-provider/thordata.ts | 98 + src/services/pseudo-transfer.ts | 65 + src/services/puppeteer.ts | 1671 +++ src/services/registry.ts | 122 + src/services/robots-text.ts | 127 + src/services/serp/bing.ts | 292 + src/services/serp/common-serp.ts | 142 + src/services/serp/compat.ts | 12 + src/services/serp/google.ts | 709 ++ src/services/serp/puppeteer.ts | 672 ++ src/services/serp/serper.ts | 165 + src/services/serper-search.ts | 217 + src/services/snapshot-formatter.ts | 1153 ++ src/services/soffice.ts | 316 + src/services/temp-file.ts | 23 + src/services/threaded.ts | 89 + src/stand-alone/crawl.ts | 171 + src/stand-alone/search.ts | 178 + src/stand-alone/serp.ts | 178 + src/types.d.ts | 18 + src/utils/encoding.ts | 55 + src/utils/ip.ts | 229 + src/utils/languages.ts | 277 + src/utils/markdown.ts | 39 + src/utils/misc.ts | 121 + src/utils/openai.ts | 96 + src/utils/search-query.ts | 95 + src/utils/tailwind-classes.ts | 9625 +++++++++++++++++ tests/e2e/assert-status-code.test.ts | 155 + tests/e2e/base-url-resolution.test.ts | 85 + tests/e2e/browser-options.test.ts | 91 + tests/e2e/cache-control.test.ts | 57 + tests/e2e/compression.test.ts | 231 + tests/e2e/content-fidelity.test.ts | 108 + tests/e2e/cookbook-recipes.test.ts | 405 + tests/e2e/cookbook-uploads.test.ts | 150 + tests/e2e/crawler-options-dto.test.ts | 212 + tests/e2e/crawler-options-misc.test.ts | 171 + tests/e2e/data-url-images.test.ts | 72 + tests/e2e/detach-invisibles.test.ts | 144 + tests/e2e/dom-options.test.ts | 77 + tests/e2e/error-handling.test.ts | 61 + tests/e2e/gfm-tables.test.ts | 77 + tests/e2e/html-input.test.ts | 88 + tests/e2e/markdown-chunking.test.ts | 77 + tests/e2e/markdown-options-extended.test.ts | 70 + tests/e2e/markdown-options.test.ts | 133 + tests/e2e/option-interactions.test.ts | 101 + tests/e2e/request-options.test.ts | 109 + tests/e2e/respond-with.test.ts | 89 + tests/e2e/retain-images.test.ts | 123 + tests/e2e/retain-links.test.ts | 104 + tests/e2e/retain-media.test.ts | 531 + tests/e2e/selectors.test.ts | 127 + tests/e2e/stream-output.test.ts | 53 + tests/e2e/summaries.test.ts | 152 + tests/e2e/token-budget.test.ts | 57 + tests/global-setup.ts | 12 + tests/helpers/client.ts | 67 + tests/helpers/fixture-server.ts | 113 + tests/helpers/pdf-fixture.ts | 76 + tests/run-unit.ts | 27 + tests/run.ts | 46 + tests/tsconfig.json | 16 + tests/unit/crawler-options-dto.test.ts | 561 + tests/unit/filtered-stream.test.ts | 153 + tests/unit/frontmatter-representation.test.ts | 208 + tests/unit/ip-utils.test.ts | 256 + tests/unit/json-parse-stream.test.ts | 275 + tests/unit/languages.test.ts | 77 + tests/unit/markdown-utils.test.ts | 140 + tests/unit/markify.test.ts | 603 ++ tests/unit/misc-utils.test.ts | 135 + tests/unit/proxy-provider.test.ts | 58 + tests/unit/pseudo-boolean.test.ts | 214 + tests/unit/sse-stream.test.ts | 242 + tsconfig.json | 24 + 177 files changed, 54678 insertions(+) create mode 100644 .c8rc.json create mode 100644 .editorconfig create mode 100644 .github/workflows/.keep create mode 100644 .github/workflows/_test.yml create mode 100644 .github/workflows/ci.yml create mode 100644 .github/workflows/oss-image.yml create mode 100644 .gitignore create mode 100644 .gitmodules create mode 100644 .nvmrc create mode 100644 .vscode/exensions.json create mode 100644 .vscode/launch.json create mode 100644 .vscode/settings.json create mode 100644 .vscode/tasks.json create mode 100644 CLAUDE.md create mode 100644 CONTRIBUTING.md create mode 100644 Dockerfile create mode 100644 LICENSE create mode 100644 README.md create mode 100644 README.wehub.md create mode 100644 architecture.md create mode 100755 bin/compile-secret.js create mode 100755 bin/extract-secret.js create mode 100644 cookbooks.md create mode 100644 docker-compose.yml create mode 100755 download-external-assets.sh create mode 100644 fixtures/sample.html create mode 100755 integrity-check.cjs create mode 100644 package-lock.json create mode 100644 package.json create mode 100644 public/favicon.ico create mode 100644 public/robots.txt create mode 100644 src/3rd-party/anthropic.ts create mode 100644 src/3rd-party/cloud-flare.ts create mode 100644 src/3rd-party/common-serp.ts create mode 100644 src/3rd-party/google-gemini.ts create mode 100644 src/3rd-party/internal-cloudrun.ts create mode 100644 src/3rd-party/jina-embeddings.ts create mode 100644 src/3rd-party/open-router.ts create mode 100644 src/3rd-party/openai-compat.ts create mode 100644 src/3rd-party/openai.ts create mode 100644 src/3rd-party/replicate.ts create mode 100644 src/3rd-party/serper-search.ts create mode 100644 src/api/crawler.ts create mode 100644 src/api/searcher.ts create mode 100644 src/api/serp.ts create mode 100644 src/config.ts create mode 100644 src/db/bucket-storage.ts create mode 100644 src/db/models.ts create mode 100644 src/db/noop-storage.ts create mode 100644 src/dto/base-auth.ts create mode 100644 src/dto/crawler-options.ts create mode 100644 src/dto/turndown-tweakable-options.ts create mode 100644 src/fetch.d.ts create mode 100644 src/lib/filtered-stream.ts create mode 100644 src/lib/json-parse-stream.ts create mode 100644 src/lib/koa-compress.ts create mode 100644 src/lib/pseudo-boolean.ts create mode 100644 src/lib/transform-server-event-stream.ts create mode 100644 src/scripts/smoke-test-llm.ts create mode 100644 src/services/alt-text.ts create mode 100644 src/services/async-context.ts create mode 100644 src/services/binary-extractor.ts create mode 100644 src/services/blackhole-detector.ts create mode 100644 src/services/bogo-sites.ts create mode 100644 src/services/canvas.ts create mode 100644 src/services/cf-browser-rendering.ts create mode 100644 src/services/common-iminterrogate/base.ts create mode 100644 src/services/common-iminterrogate/index.ts create mode 100644 src/services/common-iminterrogate/instruct-blip.ts create mode 100644 src/services/common-iminterrogate/llms.ts create mode 100644 src/services/common-iminterrogate/registry.ts create mode 100644 src/services/common-llm/base.ts create mode 100644 src/services/common-llm/google-gemini.ts create mode 100644 src/services/common-llm/gpt-1106.ts create mode 100644 src/services/common-llm/gpt-35.ts create mode 100644 src/services/common-llm/gpt-4.ts create mode 100644 src/services/common-llm/index.ts create mode 100644 src/services/common-llm/jina-vlm.ts create mode 100644 src/services/common-llm/misc.ts create mode 100644 src/services/common-llm/open-router.ts create mode 100644 src/services/common-llm/prompt-profile.ts create mode 100644 src/services/common-llm/reader-lm.ts create mode 100644 src/services/common-llm/registry.ts create mode 100644 src/services/common-llm/vertex-gemini.ts create mode 100644 src/services/curl.ts create mode 100644 src/services/default-bucket.ts create mode 100644 src/services/envconfig.ts create mode 100644 src/services/errors.ts create mode 100644 src/services/finalizer.ts create mode 100644 src/services/geoip.ts create mode 100644 src/services/ipasn.ts create mode 100644 src/services/jsdom.ts create mode 100644 src/services/lm.ts create mode 100644 src/services/logger.ts create mode 100644 src/services/markify.ts create mode 100644 src/services/minimal-stealth.js create mode 100644 src/services/misc.ts create mode 100644 src/services/pdf-extract.ts create mode 100644 src/services/proxy-provider/brightdata.ts create mode 100644 src/services/proxy-provider/index.ts create mode 100644 src/services/proxy-provider/thordata.ts create mode 100644 src/services/pseudo-transfer.ts create mode 100644 src/services/puppeteer.ts create mode 100644 src/services/registry.ts create mode 100644 src/services/robots-text.ts create mode 100644 src/services/serp/bing.ts create mode 100644 src/services/serp/common-serp.ts create mode 100644 src/services/serp/compat.ts create mode 100644 src/services/serp/google.ts create mode 100644 src/services/serp/puppeteer.ts create mode 100644 src/services/serp/serper.ts create mode 100644 src/services/serper-search.ts create mode 100644 src/services/snapshot-formatter.ts create mode 100644 src/services/soffice.ts create mode 100644 src/services/temp-file.ts create mode 100644 src/services/threaded.ts create mode 100644 src/stand-alone/crawl.ts create mode 100644 src/stand-alone/search.ts create mode 100644 src/stand-alone/serp.ts create mode 100644 src/types.d.ts create mode 100644 src/utils/encoding.ts create mode 100644 src/utils/ip.ts create mode 100644 src/utils/languages.ts create mode 100644 src/utils/markdown.ts create mode 100644 src/utils/misc.ts create mode 100644 src/utils/openai.ts create mode 100644 src/utils/search-query.ts create mode 100644 src/utils/tailwind-classes.ts create mode 100644 tests/e2e/assert-status-code.test.ts create mode 100644 tests/e2e/base-url-resolution.test.ts create mode 100644 tests/e2e/browser-options.test.ts create mode 100644 tests/e2e/cache-control.test.ts create mode 100644 tests/e2e/compression.test.ts create mode 100644 tests/e2e/content-fidelity.test.ts create mode 100644 tests/e2e/cookbook-recipes.test.ts create mode 100644 tests/e2e/cookbook-uploads.test.ts create mode 100644 tests/e2e/crawler-options-dto.test.ts create mode 100644 tests/e2e/crawler-options-misc.test.ts create mode 100644 tests/e2e/data-url-images.test.ts create mode 100644 tests/e2e/detach-invisibles.test.ts create mode 100644 tests/e2e/dom-options.test.ts create mode 100644 tests/e2e/error-handling.test.ts create mode 100644 tests/e2e/gfm-tables.test.ts create mode 100644 tests/e2e/html-input.test.ts create mode 100644 tests/e2e/markdown-chunking.test.ts create mode 100644 tests/e2e/markdown-options-extended.test.ts create mode 100644 tests/e2e/markdown-options.test.ts create mode 100644 tests/e2e/option-interactions.test.ts create mode 100644 tests/e2e/request-options.test.ts create mode 100644 tests/e2e/respond-with.test.ts create mode 100644 tests/e2e/retain-images.test.ts create mode 100644 tests/e2e/retain-links.test.ts create mode 100644 tests/e2e/retain-media.test.ts create mode 100644 tests/e2e/selectors.test.ts create mode 100644 tests/e2e/stream-output.test.ts create mode 100644 tests/e2e/summaries.test.ts create mode 100644 tests/e2e/token-budget.test.ts create mode 100644 tests/global-setup.ts create mode 100644 tests/helpers/client.ts create mode 100644 tests/helpers/fixture-server.ts create mode 100644 tests/helpers/pdf-fixture.ts create mode 100644 tests/run-unit.ts create mode 100644 tests/run.ts create mode 100644 tests/tsconfig.json create mode 100644 tests/unit/crawler-options-dto.test.ts create mode 100644 tests/unit/filtered-stream.test.ts create mode 100644 tests/unit/frontmatter-representation.test.ts create mode 100644 tests/unit/ip-utils.test.ts create mode 100644 tests/unit/json-parse-stream.test.ts create mode 100644 tests/unit/languages.test.ts create mode 100644 tests/unit/markdown-utils.test.ts create mode 100644 tests/unit/markify.test.ts create mode 100644 tests/unit/misc-utils.test.ts create mode 100644 tests/unit/proxy-provider.test.ts create mode 100644 tests/unit/pseudo-boolean.test.ts create mode 100644 tests/unit/sse-stream.test.ts create mode 100644 tsconfig.json diff --git a/.c8rc.json b/.c8rc.json new file mode 100644 index 0000000..436d167 --- /dev/null +++ b/.c8rc.json @@ -0,0 +1,15 @@ +{ + "all": true, + "src": ["src"], + "include": ["src/**/*.ts"], + "exclude": [ + "src/**/*.d.ts", + "tests/**", + "tests-build/**", + "src/scripts/**" + ], + "reporter": ["text", "html", "lcov"], + "reports-dir": "coverage", + "excludeAfterRemap": true, + "skip-full": false +} diff --git a/.editorconfig b/.editorconfig new file mode 100644 index 0000000..4bb6796 --- /dev/null +++ b/.editorconfig @@ -0,0 +1,17 @@ +# EditorConfig is awesome: https://EditorConfig.org + +# top-most EditorConfig file +root = true + +# Unix-style newlines with a newline ending every file +[*] +end_of_line = lf +insert_final_newline = true +charset = utf-8 +indent_style = space +indent_size = 4 + +[*.html] +[*.htm] +indent_style = space +indent_size = 2 diff --git a/.github/workflows/.keep b/.github/workflows/.keep new file mode 100644 index 0000000..e69de29 diff --git a/.github/workflows/_test.yml b/.github/workflows/_test.yml new file mode 100644 index 0000000..8b63510 --- /dev/null +++ b/.github/workflows/_test.yml @@ -0,0 +1,41 @@ +run-name: CI test (reusable) +on: + workflow_call: + inputs: + node-version: + type: string + default: '24.14.1' + description: 'Node.js version to use.' + +jobs: + test: + runs-on: ubuntu-latest + permissions: + contents: read + steps: + - uses: actions/checkout@v4 + with: + lfs: true + - name: Set up Node.js + uses: actions/setup-node@v4 + with: + node-version: ${{ inputs.node-version }} + cache: npm + + - name: npm install + run: npm ci + - name: get non-redistributable assets + run: npm run assets:download + + - name: build application + run: npm run build + + - name: tests with coverage + run: npm run test:coverage + + - name: upload coverage to Codecov + uses: codecov/codecov-action@v6 + with: + files: coverage/lcov.info + token: ${{ secrets.CODECOV_TOKEN }} + fail_ci_if_error: true diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..f63cf6d --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,24 @@ +run-name: CI (tests) +on: + pull_request: + paths-ignore: + - '**.md' + push: + branches-ignore: + - main + - ops + - ci-debug + - dev + paths-ignore: + - '**.md' + +concurrency: + group: ci-${{ github.ref }} + cancel-in-progress: true + +jobs: + test: + uses: ./.github/workflows/_test.yml + permissions: + contents: read + secrets: inherit diff --git a/.github/workflows/oss-image.yml b/.github/workflows/oss-image.yml new file mode 100644 index 0000000..0ddf889 --- /dev/null +++ b/.github/workflows/oss-image.yml @@ -0,0 +1,87 @@ +run-name: Build push container image to GH +on: + push: + branches: + - main + tags: + - '*' + +jobs: + test: + uses: ./.github/workflows/_test.yml + permissions: + contents: read + secrets: inherit + + build-and-push-to-ghcr: + needs: test + runs-on: ubuntu-latest + concurrency: + group: ${{ github.ref_type == 'branch' && github.ref }} + cancel-in-progress: true + permissions: + contents: read + packages: write + steps: + - uses: actions/checkout@v5 + with: + lfs: true + submodules: true + - name: Login to GitHub Container Registry + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + - name: Set controller release version + run: echo "RELEASE_VERSION=${GITHUB_REF#refs/*/}" >> $GITHUB_ENV + - name: Set up Node.js + uses: actions/setup-node@v5 + with: + node-version: 24.14.1 + cache: npm + + - name: npm install + run: npm ci + - name: get maxmind mmdb + run: mkdir -p licensed && curl -o licensed/GeoLite2-City.mmdb https://raw.githubusercontent.com/P3TERX/GeoLite.mmdb/download/GeoLite2-City.mmdb + - name: get asn mmdb + run: curl -o licensed/geolite2-asn.mmdb https://cdn.jsdelivr.net/npm/@ip-location-db/geolite2-asn-mmdb/geolite2-asn.mmdb + - name: get source han sans font + run: curl -o licensed/SourceHanSansSC-Regular.otf https://raw.githubusercontent.com/adobe-fonts/source-han-sans/refs/heads/release/OTF/SimplifiedChinese/SourceHanSansSC-Regular.otf + - name: get gsa user agent list + run: curl -o licensed/gsa_useragents.txt https://raw.githubusercontent.com/searxng/searxng/refs/heads/master/searx/data/gsa_useragents.txt + - name: build application + run: npm run build + - name: Set package version + run: npm version --no-git-tag-version ${{ env.RELEASE_VERSION }} + if: github.ref_type == 'tag' + - name: Docker meta + id: meta + uses: docker/metadata-action@v5 + with: + images: | + ghcr.io/jina-ai/reader + labels: | + org.opencontainers.image.created={{timestamp}} + org.opencontainers.image.source="https://github.com/jina-ai/reader" + tags: | + type=schedule + type=ref,event=branch + type=ref,event=tag + type=ref,event=pr + type=raw,value=oss,enable=${{ github.ref == format('refs/heads/{0}', 'main') }} + flavor: | + latest=true + - name: Set up QEMU + uses: docker/setup-qemu-action@v4 + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v4 + - name: Build and push + uses: docker/build-push-action@v7 + with: + context: . + push: true + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + platforms: linux/amd64,linux/arm64 diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..9d6811a --- /dev/null +++ b/.gitignore @@ -0,0 +1,81 @@ +# Logs +logs +*.log +npm-debug.log* +yarn-debug.log* +yarn-error.log* +firebase-debug.log* +firebase-debug.*.log* + +# Firebase cache +.firebase/ + +# Firebase config + +# Uncomment this if you'd like others to create their own Firebase project. +# For a team working on the same Firebase project(s), it is recommended to leave +# it commented so all members can deploy to the same project(s) in .firebaserc. +# .firebaserc + +# Runtime data +pids +*.pid +*.seed +*.pid.lock + +# Directory for instrumented libs generated by jscoverage/JSCover +lib-cov + +# Coverage directory used by tools like istanbul +coverage + +# nyc test coverage +.nyc_output + +# Grunt intermediate storage (http://gruntjs.com/creating-plugins#storing-task-files) +.grunt + +# Bower dependency directory (https://bower.io/) +bower_components + +# node-waf configuration +.lock-wscript + +# Compiled binary addons (http://nodejs.org/api/addons.html) +build/Release + +# Dependency directories +node_modules/ + +# Optional npm cache directory +.npm + +# Optional eslint cache +.eslintcache + +# Optional REPL history +.node_repl_history + +# Output of 'npm pack' +*.tgz + +# Yarn Integrity file +.yarn-integrity + +# dotenv environment variables file +.env +.secret.local + +toy*.ts + +.DS_Store +build/ +tests-build/ +.firebase-emu/ +*.log +.DS_Store + +*.local +.secret.* +licensed/ +.claude/ diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 0000000..e69de29 diff --git a/.nvmrc b/.nvmrc new file mode 100644 index 0000000..cabf43b --- /dev/null +++ b/.nvmrc @@ -0,0 +1 @@ +24 \ No newline at end of file diff --git a/.vscode/exensions.json b/.vscode/exensions.json new file mode 100644 index 0000000..37bdb37 --- /dev/null +++ b/.vscode/exensions.json @@ -0,0 +1,10 @@ +{ + "recommendations": [ + "editorconfig.editorconfig", + "octref.vetur", + "redhat.vscode-yaml", + "dbaeumer.vscode-eslint", + "esbenp.prettier-vscode", + "streetsidesoftware.code-spell-checker" + ] +} \ No newline at end of file diff --git a/.vscode/launch.json b/.vscode/launch.json new file mode 100644 index 0000000..1ae7453 --- /dev/null +++ b/.vscode/launch.json @@ -0,0 +1,144 @@ +{ + "version": "0.2.0", + "configurations": [ + { + "name": "Debug Stand Alone Crawl", + "request": "launch", + "runtimeVersion": "24", + "runtimeArgs": [ + ], + "env": { + }, + "cwd": "${workspaceFolder}", + "program": "build/stand-alone/crawl.js", + "skipFiles": [ + "/**" + ], + "type": "node", + "outputCapture": "std", + "preLaunchTask": "Backend:prepare", + "killBehavior": "forceful" + }, + { + "name": "Debug Stand Alone Crawl + Browser", + "request": "launch", + "runtimeVersion": "24", + "runtimeArgs": [ + ], + "env": { + "DEBUG_BROWSER": "true" + }, + "cwd": "${workspaceFolder}", + "program": "build/stand-alone/crawl.js", + "skipFiles": [ + "/**" + ], + "type": "node", + "outputCapture": "std", + "preLaunchTask": "Backend:prepare", + "killBehavior": "forceful" + }, + { + "name": "Debug Stand Alone Crawl - EU", + "request": "launch", + "runtimeVersion": "24", + "runtimeArgs": [ + ], + "env": { + "GCLOUD_PROJECT": "reader-6b7dc", + "FIRESTORE_DATABASE": "reader-eu", + "GCP_STORAGE_BUCKET": "reader-eu" + }, + "cwd": "${workspaceFolder}", + "program": "build/stand-alone/crawl.js", + "skipFiles": [ + "/**" + ], + "type": "node", + "outputCapture": "std", + "preLaunchTask": "Backend:prepare", + "killBehavior": "forceful" + }, + { + "name": "Debug Stand Alone Search", + "request": "launch", + "runtimeVersion": "24", + "runtimeArgs": [ + ], + "env": { + // "GCLOUD_PROJECT": "reader-6b7dc", + // "PREFERRED_PROXY_COUNTRY": "us", + // "JINA_CRAWLER_OFFLOAD_ORIGIN": "https://r.jina.ai" + }, + "cwd": "${workspaceFolder}", + "program": "build/stand-alone/search.js", + "skipFiles": [ + "/**" + ], + "type": "node", + "outputCapture": "std", + "preLaunchTask": "Backend:prepare", + "killBehavior": "forceful" + }, + { + "name": "Debug Stand Alone Search + Offload", + "request": "launch", + "runtimeVersion": "24", + "runtimeArgs": [ + ], + "env": { + // "GCLOUD_PROJECT": "reader-6b7dc", + // "PREFERRED_PROXY_COUNTRY": "us", + // "JINA_CRAWLER_OFFLOAD_ORIGIN": "https://r.jina.ai" + }, + "cwd": "${workspaceFolder}", + "program": "build/stand-alone/search.js", + "skipFiles": [ + "/**" + ], + "type": "node", + "outputCapture": "std", + "preLaunchTask": "Backend:prepare", + "killBehavior": "forceful" + }, + { + "name": "Debug Stand Alone SERP", + "request": "launch", + "runtimeVersion": "24", + "runtimeArgs": [ + ], + "env": { + "GCLOUD_PROJECT": "reader-6b7dc", + // "OVERRIDE_GOOGLE_DOMAIN": "www.google.com.hk", + // "PREFERRED_PROXY_COUNTRY": "us" + }, + "cwd": "${workspaceFolder}", + "program": "build/stand-alone/serp.js", + "skipFiles": [ + "/**" + ], + "type": "node", + "outputCapture": "std", + "preLaunchTask": "Backend:prepare", + "killBehavior": "forceful" + }, + { + "name": "Attach", + "port": 9229, + "request": "attach", + "skipFiles": [ + "/**" + ], + "type": "node" + }, + { + "name": "Attach by Process ID", + "processId": "${command:PickProcess}", + "request": "attach", + "skipFiles": [ + "/**" + ], + "type": "node" + }, + ] +} diff --git a/.vscode/settings.json b/.vscode/settings.json new file mode 100644 index 0000000..87ecca7 --- /dev/null +++ b/.vscode/settings.json @@ -0,0 +1,32 @@ +{ + "editor.wordWrap": "on", + "editor.wordWrapColumn": 120, + "files.trimTrailingWhitespace": true, + "files.trimFinalNewlines": true, + "[javascript]": { + "editor.defaultFormatter": "vscode.typescript-language-features" + }, + "[jsonc]": { + "editor.defaultFormatter": "vscode.json-language-features" + }, + "[typescript]": { + "editor.defaultFormatter": "vscode.typescript-language-features" + }, + "[json]": { + "editor.defaultFormatter": "vscode.json-language-features" + }, + "[yaml]": { + "editor.defaultFormatter": "redhat.vscode-yaml" + }, + "[markdown]": { + "files.trimTrailingWhitespace": false + }, + "typescript.tsdk": "node_modules/typescript/lib", + "typescript.preferences.quoteStyle": "single", + "typescript.format.semicolons": "insert", + "typescript.preferences.importModuleSpecifier": "project-relative", + "typescript.locale": "en", + "cSpell.enabled": true, + "cSpell.words": [ + ], +} \ No newline at end of file diff --git a/.vscode/tasks.json b/.vscode/tasks.json new file mode 100644 index 0000000..70271c7 --- /dev/null +++ b/.vscode/tasks.json @@ -0,0 +1,64 @@ +{ + "version": "2.0.0", + "tasks": [ + { + "type": "npm", + "script": "build", + "group": "build", + "options": { + "cwd": "${workspaceFolder}" + }, + "problemMatcher": [], + "label": "Backend:rebuild", + "detail": "Backend:rebuild" + }, + { + "type": "typescript", + "options": { + "cwd": "${workspaceFolder}" + }, + "tsconfig": "tsconfig.json", + "option": "watch", + "isBackground": true, + "problemMatcher": [ + "$tsc-watch" + ], + "group": "build", + "label": "Backend:build:watch" + }, + { + "label": "Backend:docker-compose:up", + "type": "shell", + "command": "docker", + "args": [ + "compose", + "up" + ], + "group": "build", + "isBackground": true, + "presentation": { + "echo": true, + "reveal": "always", + "focus": false, + "panel": "shared" + }, + "problemMatcher": { + "pattern": { + "regexp": ".*?" + }, + "background": { + "activeOnStart": true, + "beginsPattern": "OKOKOK", + "endsPattern": ".*" + } + } + }, + { + "label": "Backend:prepare", + "dependsOn": [ + "Backend:docker-compose:up", + "Backend:build:watch" + ], + } + ] +} \ No newline at end of file diff --git a/CLAUDE.md b/CLAUDE.md new file mode 100644 index 0000000..31703bb --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1,126 @@ +# CLAUDE.md + +This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository. + +## Repository identity + +This is the `oss` branch of `reader-saas`, the codebase behind `https://r.jina.ai` (URL → markdown) and `https://s.jina.ai` (search → markdown). It is published to https://github.com/jina-ai/reader. The MongoDB-backed SaaS storage layer is **not** in this branch — only the stateless and bucket-cached storage modes ship here. + +Companion docs in this repo: +- `architecture.md` — engines, formatting profiles, abuse mitigation, deployment topology. +- `CONTRIBUTING.md` — full env var table, dev workflow, test policy. +- `cookbooks.md` — header recipes for common downstream pipelines (RAG, embedding, deep research, uploads, etc.). + +## Common commands + +Build / run: +- `npm run build` — runs `integrity-check.cjs` (requires `licensed/GeoLite2-City.mmdb` to exist) then `tsc -p .`. The integrity check is **not** optional — bare `tsc` will skip it. +- `npm run build:watch` — incremental build for the F5 debug flow. +- `npm start` — runs the compiled crawler entrypoint (`build/stand-alone/crawl.js`). +- `npm run dry-run` — `NODE_ENV=dry-run` boots `search.js`, resolves the DI graph, then exits via `finalizer.terminate()`. Used in the Dockerfile to warm `NODE_COMPILE_CACHE`. + +Local services (`docker compose up -d`): +- `minio` on `:9000` (API) / `:9001` (console) — also aliased as `minio.dev.jina.ai`. Only needed when running with `BucketStorageLayer`. + +VSCode F5 launches `Debug Stand Alone Crawl` (or `Search` / `SERP`), which runs `Backend:prepare` (docker compose + tsc watch) first. `--env-file=.secret.local` is loaded by Node directly — that file is gitignored and must be created locally. + +Three independent stand-alone servers: +- `build/stand-alone/crawl.js` — mounts `CrawlerHost` (the `r.jina.ai` surface). +- `build/stand-alone/search.js` — mounts `SearcherHost`. At startup it deletes any registry entries tagged `'crawl'`, so search and crawl are mutually exclusive on a single process. +- `build/stand-alone/serp.js` — SERP-only. + +Linting: `npm run lint` (eslint over `.js`/`.ts`). + +## Tests + +The repo uses the **Node.js built-in test runner**. Do not introduce Jest, Vitest, or similar. + +- `npm run test:unit` — pure TS unit tests under `tests/unit/`. Compiled into `tests-build/` before running. No Docker required. +- `npm run test:e2e` — boots the real `CrawlStandAloneServer` via `serviceReady()` and hits `tests/e2e/*.test.ts` against it. Needs Docker services up and `.secret.local` configured. +- `npm test` — runs unit then e2e. +- `npm run test:coverage` — c8 across both suites; merges coverage from unit and e2e runs (`--no-clean` between them). + +Single test: +```bash +tsc -p tests/tsconfig.json +node --test tests-build/unit/.test.js +node --test --test-name-pattern '' tests-build/unit/.test.js +``` +For e2e, the test runner expects the crawl server already initialized — easier to run the full file via `node tests-build/run.js` after editing `run.ts` to filter, or just run the whole suite. + +The e2e harness shuts down via `finalizer.teardown()` once the enqueued count matches the completed count; if you add async tests that don't go through `node:test`'s lifecycle, the process won't exit cleanly. + +## Architecture + +### DI: tsyringe + civkit + +Every service is a `@singleton()` registered against the global `tsyringe` container. The graph is wired by side-effect: importing a module is what registers it. The conventional bootstrap is: + +1. `import 'reflect-metadata';` +2. `import '../config';` — sets `AUTH_DTO_CLS` and `STORAGE_CLS` based on env vars (see below). +3. `container.resolve(...)` to get a fully-injected instance. + +`AsyncService` (from `civkit`) is the base. Services emit `'ready'` after `dependencyReady()` resolves. `serviceReady()` waits until the entire graph is initialized. Always wait on `serviceReady()` before listening / running tests. + +Don't construct services with `new` — go through `container.resolve` (or constructor injection). Adding a new service means: `@singleton()` + add it to a constructor that's already in the graph (or call `container.resolve` once). + +### `src/config.ts` — runtime mode selection + +The same code runs in two storage modes. `config.ts` swaps the implementation classes at import time: + +- `AUTH_DTO_CLS`: `BaseAuthDTO` (the oss branch always uses the base DTO — no SaaS auth). +- `STORAGE_CLS`: defaults to `StorageLayer` (`db/noop-storage.ts` — every method returns nothing). With `GCP_STORAGE_*` → `BucketStorageLayer` (Stage 1: bucket-only cache). + +This mirrors the "Progressive Clustering" stages in `architecture.md`. The noop layer means the same code paths exist in stateless mode — handlers always call `storage.findPageCache` etc., and the noop returns `undefined` so the request just falls through to a live fetch. Don't add `if (storage)` guards; rely on the noop. + +When you change a method on the storage layer, you must update **both** implementations: `db/noop-storage.ts` and `db/bucket-storage.ts`. + +### RPC routing: civkit + Koa + +`src/services/registry.ts` is the routing core. It re-exports `Method`, `RPCMethod`, `Param`, `Ctx`, `RPCReflect` decorators and the `Context` type. Don't import these from `koa` or `civkit/civ-rpc` directly — go through `services/registry.ts` so you get the configured `RPCRegistry` and `ReaderEnvelope`. + +`ReaderEnvelope` content-negotiates errors: JSON, text/markdown, and SSE branches all live there. If you add a new content type, add an error path here too — otherwise errors will fall back to the JSON envelope. + +Endpoints are `RPCHost` subclasses in `src/api/`: +- `crawler.ts` — `CrawlerHost`, the `r.jina.ai` surface. +- `searcher.ts` — `SearcherHost`, the `s.jina.ai` surface (calls back into crawler). +- `serp.ts` — pure SERP. + +### Worker threads + +`ThreadedServiceRegistry` (`src/services/threaded.ts`) extends `civkit`'s `AbstractThreadedServiceRegistry`. CPU-heavy work (DOM manipulation, PDF parsing, markify) runs in worker threads. The registry sniffs hyperthreading and sets `maxWorkers` to `cpus.length` or `cpus.length / 2` accordingly. Workers share state via `PseudoTransfer`; do not use `postMessage` directly. + +To make a method run in a worker, decorate it with `@Threaded()` — exported by `services/threaded.ts` alongside the RPC decorators. See `services/jsdom.ts` for the canonical pattern (its narrowing/parsing methods are all threaded). + +### Engines and formatting profiles + +The crawler is multi-engine and multi-profile: +- Engines (URL→HTML): `puppeteer.ts` (browser), `curl.ts` (curl-impersonate), `cf-browser-rendering.ts`. `auto` is the default and is implemented in `crawler.ts`. +- Formatting profiles (HTML→Markdown): `@mozilla/readability`, the `markify` rule-based engine (`services/markify.ts`), and `lm.ts` (ReaderLM v2). +- Header `x-respond-with` selects the profile; `x-engine` selects the engine. + +The full surface — including `x-respond-timing`, `x-retain-images`, `x-retain-links`, `x-markdown-chunking`, and the body-field equivalents — lives in `src/dto/crawler-options.ts`. When users ask "what does header X do," that file is authoritative. `cookbooks.md` shows curated combinations. + +When adding a new engine or profile, wire it into `CrawlerHost.crawl()` (the dispatch lives there) and register a singleton service for the implementation. + +### Auth and rate limiting + +Every public method takes an `auth: AUTH_DTO_CLS`. In this branch the base DTO accepts any request — there is no upstream auth check. Rate limiting goes through `storage.rateLimit(ctx, rpcReflect, auth)`, which the noop layer answers with an empty policy. The SaaS-mode enforcement lives outside this branch. + +## Environment variables + +The full table lives in `CONTRIBUTING.md`. Two shapes you'll see in practice: + +- `.secret.local` — gitignored, loaded via Node's `--env-file` flag from `.vscode/launch.json`. Compiled / extracted by `bin/compile-secret.js` and `bin/extract-secret.js`. +- `SECRETS_COMBINED` — base64 JSON blob, the production shape. Decoded and merged by `services/envconfig.ts`. + +Mode-selecting vars worth remembering when reading code: `GCP_STORAGE_ENDPOINT` + `GCP_STORAGE_BUCKET` (Stage 1 bucket cache), `NODE_ENV=dry-run` (DI-graph warmup, used in the Dockerfile), `DEBUG_BROWSER=true` (non-headless puppeteer), `JINA_CRAWLER_OFFLOAD_ORIGIN` (search → peer crawler cluster). + +## Conventions specific to this repo + +- Decorators (`@singleton`, `@Method`, `@Param`, `@Threaded`, etc.) require `experimentalDecorators` and `emitDecoratorMetadata` — both already on. Do not migrate to TC39 decorators. +- The `licensed/` folder holds external (non-redistributable) artifacts (GeoLite mmdbs, Source Han Sans font, gsa user-agent list). It's gitignored. CI fetches them inline; locally, run `npm run assets:download` (or `bash ./download-external-assets.sh` directly) — idempotent, skips files already present. The build's integrity check requires `licensed/GeoLite2-City.mmdb` to exist; if `npm run build` errors on it, you forgot to run the download. The folder is still named `licensed/` for source-code/Dockerfile compatibility — only the script and env vars use the "external" naming. +- Build output is `build/`, test build output is `tests-build/`. Both are gitignored. The Dockerfile copies pre-built `build/` rather than building inside the image. +- HTTP/2 cleartext (h2c) is used in production; `crawl.ts` / `search.ts` install an `http2.createServer` on top of the Koa callback. Each request gets its own `traceId` (random UUID) because h2c connections multiplex — don't rely on `x-cloud-trace-context` for per-request identity. +- Public assets in `public/` are served directly by the stand-alone server's `walkForAssets()` — drop a file in there, rebuild, it's served at `/`. +- The Reader API surface is entirely header- and body-driven; there are no path-based routes for options. When changing behavior, the right place is almost always `src/dto/crawler-options.ts` (parsing) plus `src/api/crawler.ts` (dispatch). diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000..582971a --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,132 @@ +# Contributing to Reader + +Thanks for your interest in contributing. This is the open source branch of the codebase that runs at `https://r.jina.ai` and `https://s.jina.ai`. The MongoDB-backed SaaS storage layer is not part of this branch — local development uses the stateless / bucket-cached modes only. + +If you're not sure where to start, take a look at [architecture.md](./architecture.md) first. + +## Local development + +### Requirements + +- **Node.js 22+** — earlier versions will not build. +- **Docker** *(optional)* — only needed if you want to run the bucket-cached storage mode against a local MinIO. Pure stateless mode needs nothing extra. +- **LibreOffice** *(optional)* — only needed if you want to test MS Office document handling locally. + +### First-time setup + +```bash +git clone git@github.com:jina-ai/reader.git +cd reader +npm install +# Optional: only if you want the local bucket cache +docker compose up -d +``` + +`docker compose up -d` starts: + +| Service | Port(s) | Purpose | +| ------- | -------------- | ------------------------------------------------------------------ | +| `minio` | `9000`, `9001` | S3-compatible object storage for cached pages. Console on `:9001`. | + +### Running the server + +In VSCode, press `F5` to launch the debugger. + +Or, after exporting the environment variables (see below): + +```bash +docker compose up -d +npm run dev +``` + +### Useful scripts + +- `npm run build` — TypeScript compile (also runs an integrity check). +- `npm run build:watch` — incremental build. +- `npm run start` — run the compiled `crawl` entrypoint. +- `npm run dry-run` — run `search.js` with `NODE_ENV=dry-run` to resolve the DI graph and exit. Used to warm `NODE_COMPILE_CACHE` in the Dockerfile. +- `npm run lint` — ESLint over `.js` / `.ts`. + +## Environment variables + +Reader picks up configuration from environment variables. The most relevant ones for local development are: + +### Storage & data + +| Variable | Notes | +| -------------------------- | ------------------------------------------------------------------------------------- | +| `GCP_STORAGE_ENDPOINT` | Object storage endpoint (use the local MinIO endpoint for dev). Enables Stage 1 bucket-cached mode. | +| `GCP_STORAGE_BUCKET` | Bucket name for cached objects. | +| `GCP_STORAGE_ACCESS_KEY` | MinIO root user locally. | +| `GCP_STORAGE_SECRET_KEY` | MinIO root password locally. | +| `GCP_STORAGE_REGION` | Optional; for parity with GCS. | +| `GCLOUD_PROJECT` | Alternative trigger for the bucket layer when combined with `GCP_STORAGE_ENDPOINT`. | +| `CACHE_LOCAL_STORAGE_ROOT` | Filesystem root for local cache (alternative to object storage in stateless modes). | + +### Vendors & integrations + +| Variable | Purpose | +| ------------------------------------------------- | -------------------------------------------------------------------- | +| `JINA_SERP_API_KEY` / `JINA_SERP_API_ORIGIN` | Jina SERP backend. | +| `JINA_SERP_API_POLICY` | SERP routing policy. | +| `SERPER_SEARCH_API_KEY` | serper.dev search backend. | +| `THORDATA_PROXY_URL` / `THORDATA_PROXY_URL_ALT` | Thordata residential proxy. | +| `THORDATA_SERP_API_KEY` | Thordata SERP API. | +| `BRIGHTDATA_PROXY_URL` / `BRIGHTDATA_ISP_PROXY_URL` / `BRIGHTDATA_SERP_API_KEY` | BrightData proxy + SERP. | +| `CLOUD_FLARE_API_KEY` | Required for the `cf-browser-rendering` engine. | +| `STRIPE_SECRET_KEY` / `STRIPE_WEBHOOK_KEY` | Billing integration. | +| `OPENAI_API_KEY` / `ANTHROPIC_API_KEY` / `OPENROUTER_API_KEY` / `GOOGLE_AI_STUDIO_API_KEY` / `REPLICATE_API_KEY` | LLM/VLM access. | + +### Overrides & toggles + +| Variable | Purpose | +| --------------------------------- | ------------------------------------------------------------------ | +| `PORT` | HTTP port. | +| `NODE_ENV` | `dry-run` is recognized for offline `search` runs. | +| `DEBUG_BROWSER` | Run headless Chrome with non-headless / debug behavior. | +| `OVERRIDE_CHROME_EXECUTABLE_PATH` | Use a specific Chrome binary instead of the bundled Puppeteer one. | +| `OVERRIDE_JINA_VLM_URL` | Point at a different VLM endpoint. | +| `OVERRIDE_READERLM_V` | Switch between ReaderLM versions. | +| `OVERRIDE_GOOGLE_DOMAIN` / `OVERRIDE_BING_DOMAIN` | Use a regional search domain. | +| `OVERRIDE_MANAGE_SERVER_URL` | Redirect calls to the management server. | +| `JINA_BOGO_SITES_RESORT_ORIGIN` | Origin for the bogo-sites resort list. | +| `JINA_CRAWLER_OFFLOAD_ORIGIN` | Offload crawler traffic to a peer cluster. | +| `PREFERRED_PROXY_COUNTRY` | Hint for proxy country selection. | +| `SLACK_REPORT_WEBHOOK_URL` | Slack channel for runtime reports. | + +### `SECRETS_COMBINED` + +You can pass a base64-encoded JSON object via `SECRETS_COMBINED` to bundle multiple variables into one. See `src/services/envconfig.ts`. + +## Tests + +The repo uses the Node.js built-in test runner (no Jest, no Vitest). + +```bash +npm run test:unit # unit tests +npm run test:e2e # end-to-end tests (slower, hits docker services) +npm test # both + +npm run test:unit:coverage +npm run test:e2e:coverage +npm run test:coverage # combined coverage report (c8) +``` + +Tests are written in TypeScript under `tests/` and compiled into `tests-build/` before running. The test entrypoints are `tests-build/run-unit.js` and `tests-build/run.js`. + +## Submitting changes + +1. Open an issue first if the change is non-trivial — it saves churn for both sides. +2. Keep PRs focused. A bug fix and a refactor in the same PR are harder to review and revert. +3. Run `npm run lint` and `npm test` before pushing. +4. Reference the issue in the PR description if one exists. + +## Reporting issues + +Bug reports are most useful when they include: + +- The exact URL (for `r.jina.ai`) or query (for `s.jina.ai`). +- The request headers in use, especially any `x-*` overrides. +- The expected vs actual output. + +Open an issue on GitHub and we'll take a look. diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..1412261 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,39 @@ +# syntax=docker/dockerfile:1 +FROM node:24 AS base + +FROM base AS build-amd64 +RUN apt-get update \ + && apt-get install -y wget gnupg \ + && wget -q -O - https://dl-ssl.google.com/linux/linux_signing_key.pub | apt-key add - \ + && sh -c 'echo "deb [arch=amd64] http://dl.google.com/linux/chrome/deb/ stable main" >> /etc/apt/sources.list.d/google.list' \ + && apt-get update \ + && apt-get install -y libreoffice google-chrome-stable fonts-noto-cjk-extra fonts-noto-color-emoji fonts-liberation fonts-ipafont-gothic fonts-wqy-zenhei fonts-thai-tlwg fonts-kacst fonts-freefont-ttf libxss1 zstd libc++-dev \ + --no-install-recommends \ + && rm -rf /var/lib/apt/lists/* +ENV OVERRIDE_CHROME_EXECUTABLE_PATH=/usr/bin/google-chrome-stable + +FROM base AS build-arm64 +RUN apt-get update \ + && apt-get install -y libreoffice chromium fonts-noto-cjk-extra fonts-noto-color-emoji fonts-liberation fonts-ipafont-gothic fonts-wqy-zenhei fonts-thai-tlwg fonts-kacst fonts-freefont-ttf libxss1 zstd libc++-dev \ + --no-install-recommends \ + && rm -rf /var/lib/apt/lists/* +ENV OVERRIDE_CHROME_EXECUTABLE_PATH=/usr/bin/chromium + +FROM build-${TARGETARCH} AS final +RUN groupadd -r jina +RUN useradd -g jina -G audio,video -m jina +USER jina +WORKDIR /app +COPY package.json package-lock.json ./ +RUN npm ci +COPY build ./build +COPY public ./public +COPY licensed ./licensed +RUN rm -rf ~/.config/chromium && mkdir -p ~/.config/chromium +RUN NODE_COMPILE_CACHE=node_modules npm run dry-run +ENV NODE_COMPILE_CACHE=node_modules +ENV PORT=8080 + +EXPOSE 8080 8081 +ENTRYPOINT ["node"] +CMD [ "build/stand-alone/crawl.js" ] diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..4875d6a --- /dev/null +++ b/LICENSE @@ -0,0 +1,193 @@ +Copyright 2020-2024 Jina AI Limited. All rights reserved. + + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + Copyright 2020-2021 Jina AI Limited + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/README.md b/README.md new file mode 100644 index 0000000..3688a3e --- /dev/null +++ b/README.md @@ -0,0 +1,288 @@ +# Reader + +[![codecov](https://codecov.io/gh/jina-ai/reader/branch/main/graph/badge.svg)](https://codecov.io/gh/jina-ai/reader) +[![Ask DeepWiki](https://deepwiki.com/badge.svg)](https://deepwiki.com/jina-ai/reader) + +Your LLMs deserve better input. + +Reader does two things: +- **Read**: It converts any URL to an **LLM-friendly** input with `https://r.jina.ai/https://your.url`. Get improved output for your agent and RAG systems at no cost. +- **Search**: It searches the web for a given query with `https://s.jina.ai/your+query`. This allows your LLMs to access the latest world knowledge from the web. + +Check out [the live demo](https://jina.ai/reader#demo) + +Or just visit these URLs (**Read**) https://r.jina.ai/https://github.com/jina-ai/reader, (**Search**) https://s.jina.ai/Who%20will%20win%202024%20US%20presidential%20election%3F and see yourself. + +> Feel free to use Reader API in production. It is free, stable and scalable. We are maintaining it actively as one of the core products of Jina AI. [Check out rate limit](https://jina.ai/reader#pricing) + +image +image + +> This repository is the open source branch of the codebase behind `https://r.jina.ai` and `https://s.jina.ai`. It runs in stateless or bucket-cached mode; the MongoDB-backed SaaS storage layer is not included here. + +## Updates + +- **2026-04** — Re-synchronized the open source branch with the SaaS code. The MongoDB-backed storage layer is stripped; the oss branch runs in stateless mode out of the box, with optional MinIO/S3-compatible bucket caching via `docker compose`. See [Local development](#local-development). +- **2025-12** — Storage layer decoupled and binary file uploads landed. PDFs and MS Office documents (Word, Excel, PowerPoint) can now be POSTed directly via the `file` body field — no need to host them first. See [cookbooks.md](./cookbooks.md#pdf-ms-office-and-raw-html-uploads). +- **2025-03** — Major refactor: Reader is no longer a Firebase application. The SaaS migrated off Firestore + Cloud Functions to a Cloud Run image with MongoDB Atlas, removing the platform-coupled bits and unblocking the local-Docker path above. +- **2024-05** — `s.jina.ai` launched, extending Reader from URL→markdown to search→markdown. PDFs added the same month — any URL ending in `.pdf` is parsed with PDF.js and returned as markdown. +- **2024-04** — Reader released and `r.jina.ai` went live as Jina AI's first SaaS API for converting URLs to LLM-friendly input. + +## What Reader can read + +- **Web pages** — rendered with headless Chrome, or fetched lightweight via `curl-impersonate`. Reader picks intelligently between the two. +- **PDFs** — any URL, parsed with PDF.js. [See this NASA PDF result](https://r.jina.ai/https://www.nasa.gov/wp-content/uploads/2023/01/55583main_vision_space_exploration2.pdf) vs [the original](https://www.nasa.gov/wp-content/uploads/2023/01/55583main_vision_space_exploration2.pdf). +- **MS Office documents** — Word, Excel, PowerPoint, converted via LibreOffice and then processed as HTML/PDF. +- **Images** — captioned by a vision-language model, so your downstream text-only LLM gets *just enough* hints to reason about them. + +## Usage + +### Using `r.jina.ai` for single URL fetching +Simply prepend `https://r.jina.ai/` to any URL. For example, to convert the URL `https://en.wikipedia.org/wiki/Artificial_intelligence` to an LLM-friendly input, use the following URL: + +[https://r.jina.ai/https://en.wikipedia.org/wiki/Artificial_intelligence](https://r.jina.ai/https://en.wikipedia.org/wiki/Artificial_intelligence) + +### [Using `r.jina.ai` for a full website fetching (Google Colab)](https://colab.research.google.com/drive/1uoBy6_7BhxqpFQ45vuhgDDDGwstaCt4P#scrollTo=5LQjzJiT9ewT) + +### Using `s.jina.ai` for web search +Simply prepend `https://s.jina.ai/` to your search query. Note that if you are using this in the code, make sure to encode your search query first, e.g. if your query is `Who will win 2024 US presidential election?` then your url should look like: + +[https://s.jina.ai/Who%20will%20win%202024%20US%20presidential%20election%3F](https://s.jina.ai/Who%20will%20win%202024%20US%20presidential%20election%3F) + +Behind the scenes, Reader searches the web, fetches the top 5 results, visits each URL, and applies `r.jina.ai` to it. This is different from many `web search function-calling` in agent/RAG frameworks, which often return only the title, URL, and description provided by the search engine API. If you want to read one result more deeply, you have to fetch the content yourself from that URL. With Reader, `http://s.jina.ai` automatically fetches the content from the top 5 search result URLs for you (reusing the tech stack behind `http://r.jina.ai`). This means you don't have to handle browser rendering, blocking, or any issues related to JavaScript and CSS yourself. + +### Using `s.jina.ai` for in-site search +Simply specify `site` in the query parameters such as: + +```bash +curl 'https://s.jina.ai/When%20was%20Jina%20AI%20founded%3F?site=jina.ai&site=github.com' +``` + +### [Interactive Code Snippet Builder](https://jina.ai/reader#apiform) + +We highly recommend using the code builder to explore different parameter combinations of the Reader API. + +image + +### Using request headers + +You can control the behavior of the Reader API using request headers. The list below covers the most useful ones — for the full surface with up-to-date defaults and validation rules, see the live API docs at [https://r.jina.ai/docs](https://r.jina.ai/docs), or the source of truth in [`src/dto/crawler-options.ts`](./src/dto/crawler-options.ts). + +- `x-respond-with` — select the output format. + - `markdown` returns markdown *without* going through `readability` + - `html` returns `documentElement.outerHTML` + - `text` returns `document.body.innerText` + - `screenshot` returns the URL of the webpage's screenshot + - `pageshot` similar to `screenshot` but tries to capture the whole page instead of just the viewport + - `frontmatter` returns **Markdown with a YAML frontmatter block**. The default plain-text response uses a custom `Title: …` / `URL Source: …` header format; `frontmatter` replaces that with a front matter block. Example: + + ```bash + curl -H 'X-Respond-With: frontmatter' 'https://r.jina.ai/https://example.com' + ``` + + ```markdown + --- + title: "Example Domain" + description: "This domain is for use in illustrative examples." + url: "https://example.com/" + --- + + ## Example Domain + + This domain is for use in illustrative examples in documents. ... + ``` + + - `markdown+frontmatter` — like `frontmatter` but covers the full page without readability filtering. +- `x-engine` — enforces a fetching engine: `browser` (headless Chrome), `curl` (lightweight, no JS), or `auto` (the default — Combined use of both browser and curl). +- `x-proxy-url` — route the traffic through your designated proxy. +- `x-cache-tolerance` — integer seconds; how stale a cached page is acceptable. +- `x-no-cache: true` — bypass the cached page (lifetime 3600s). Equivalent to `x-cache-tolerance: 0`. +- `x-target-selector` — a CSS selector. Reader returns content within the matched element instead of the full page. Useful when automatic content extraction misses what you want. +- `x-wait-for-selector` — a CSS selector. Reader waits until the matched element is rendered before returning. If `x-target-selector` is set, this can be omitted to wait for the same element. +- `x-timeout` — integer seconds (max 180). When set, Reader will not return early; it waits for network idle or until the timeout is reached. +- `x-max-tokens` — integer (≥500). Trim the response so it never exceeds this many tokens. Useful as a per-request guardrail when feeding a fixed-size context window — Reader truncates rather than rejects. +- `x-token-budget` — integer. Reject the request if the resulting content would exceed this many tokens. Use this when *over*-budget output is worse than no output (e.g. cost control). Ignored on the search endpoint. +- `x-respond-timing` — explicit control over *when* Reader is willing to return. Trade off latency against completeness: + - `html` — return as soon as the raw HTML lands. No JS execution, no waiting. + - `visible-content` — return the moment readable content is parseable. Lowest latency that still produces text. + - `mutation-idle` — wait for DOM mutations to settle for ≥0.2s. Good default for SPAs that lazy-render above the fold. + - `resource-idle` — wait for content-affecting resources to finish loading (≥0.5s quiet). The default heuristic for content-shaped requests. + - `media-idle` — wait for media (images, video, fonts) to also finish. Use with `screenshot` / `pageshot` / `vlm`. + - `network-idle` — full `networkidle0`. Slowest, most complete. Implied when `x-timeout` ≥ 20. + + When omitted, Reader picks one based on `x-respond-with`, `x-timeout`, and `x-with-iframe`. See `presumedRespondTiming` in [src/dto/crawler-options.ts](./src/dto/crawler-options.ts) for the exact rules. +- `x-with-generated-alt: true` — caption images on the page with a VLM. +- `x-retain-images` — control how images survive into the output: + - `all` (default) — keep `![alt](url)` markdown for every image. + - `none` — drop images entirely. + - `alt` — keep alt text only, no URLs. Cheap on tokens; useful when the downstream LLM has no use for the image link. +- `x-retain-links` — control how links survive into the output: + - `all` (default) — keep `[text](url)` markdown. + - `none` — drop links entirely. + - `text` — keep link anchor text only, drop URLs. Best for embedding / semantic-index pipelines where URLs are noise. + - `gpt-oss` — emit citations in gpt-oss's `【{id}†...】` format and append a numbered URL footer (also auto-enables `x-with-links-summary`). +- `x-retain-media` — control how `