map $http_accept $serve_markdown { default 0; ~*text/plain 1; ~*text/markdown 1; } map $request_uri $has_no_extension { ~^[^.]*$ 1; default 0; } # Nginx reverse proxy configuration for crawlee.dev # Routes to GitHub Pages and handles legacy URL redirects server { listen 0.0.0.0:8080; server_name 'crawlee.dev'; # comment out the resolver and use localhost:3000 for local development set $backend "https://apify.github.io/crawlee"; resolver 1.1.1.1 8.8.8.8 valid=30s ipv6=off; # Health check endpoint location /health { access_log off; return 200 '{"status":"UP"}'; add_header Content-Type application/json; } location = / { if ($serve_markdown) { rewrite ^ /llms.txt last; } proxy_pass $backend/; } location ~ ^/(llms|llms-full)\.txt$ { proxy_hide_header Content-Type; add_header Content-Type 'text/markdown; charset=utf-8' always; proxy_pass $backend$uri; } # remove trailing slashes from all URLs (except root /) # exact match locations (e.g., location = /python/) take priority over this regex # Only match URIs composed of safe characters (letters, digits, dots, hyphens, # underscores, forward slashes). This prevents open redirect via %5C (backslash): # nginx decodes %5C to \ in $uri, and \ in the Location header gets normalized # to / by browsers, turning /\evil.com into //evil.com (protocol-relative URL). location ~ ^(/[a-zA-Z0-9][a-zA-Z0-9_./-]*)/$ { rewrite ^(.+)/$ $1$is_args$args? redirect; } location / { set $rewrite_condition "$serve_markdown$has_no_extension"; set $proxy_path $request_uri; if ($rewrite_condition = "11") { set $proxy_path "${request_uri}.md"; } proxy_pass $backend$proxy_path; } ### Repository path: "/python" location = /python { if ($serve_markdown) { rewrite ^ /python/llms.txt last; } proxy_pass https://apify.github.io/crawlee-python/; } location = /python/ { if ($serve_markdown) { rewrite ^ /python/llms.txt last; } proxy_pass https://apify.github.io/crawlee-python/; } location ~ ^/python/(llms|llms-full)\.txt$ { proxy_hide_header Content-Type; add_header Content-Type 'text/markdown; charset=utf-8' always; proxy_pass https://apify.github.io/crawlee-python/$1.txt; } location ~ ^/python/(.*)$ { set $path_suffix $1; set $proxy_path "/$path_suffix"; set $rewrite_condition "$serve_markdown$has_no_extension"; if ($rewrite_condition = "11") { set $proxy_path "${proxy_path}.md"; } proxy_pass https://apify.github.io/crawlee-python$proxy_path; } # So that we can have both GH pages and crawlee.dev/python working and loading assets from the same path location /crawlee-python { proxy_pass https://apify.github.io/crawlee-python/; } # Redirect rule for old JS docs to go under /js prefix rewrite ^/docs(.*)$ /js/docs$1 permanent; rewrite ^/api(.*)$ /js/api$1 permanent; # Remove version numbers from /js/api/3.[0-9]/* and /js/docs/3.[0-9]/* rewrite ^/js/api/3\.\d(/.*)?$ /js/api$1 permanent; rewrite ^/js/docs/3\.\d(/.*)?$ /js/docs$1 permanent; # Redirect rule for "upgrading-to-v03" to "upgrading-to-v0x" rewrite ^/python/docs/upgrading/upgrading-to-v03$ /python/docs/upgrading/upgrading-to-v0x permanent; # Redirect rule so that /python/docs actually leads somewhere rewrite ^/python/docs/?$ /python/docs/quick-start; rewrite ^/versions/?$ /js/api/core/changelog permanent; }