wehub-resource-sync
115 lines
3.7 KiB
Nginx Configuration File
115 lines
3.7 KiB
Nginx Configuration File
map $http_accept $serve_markdown {
|
|
default 0;
|
|
~*text/plain 1;
|
|
~*text/markdown 1;
|
|
}
|
|
|
|
map $request_uri $has_no_extension {
|
|
~^[^.]*$ 1;
|
|
default 0;
|
|
}
|
|
|
|
# Nginx reverse proxy configuration for crawlee.dev
|
|
# Routes to GitHub Pages and handles legacy URL redirects
|
|
server {
|
|
listen 0.0.0.0:8080;
|
|
server_name 'crawlee.dev';
|
|
|
|
# comment out the resolver and use localhost:3000 for local development
|
|
set $backend "https://apify.github.io/crawlee";
|
|
resolver 1.1.1.1 8.8.8.8 valid=30s ipv6=off;
|
|
|
|
# Health check endpoint
|
|
location /health {
|
|
access_log off;
|
|
return 200 '{"status":"UP"}';
|
|
add_header Content-Type application/json;
|
|
}
|
|
|
|
location = / {
|
|
if ($serve_markdown) {
|
|
rewrite ^ /llms.txt last;
|
|
}
|
|
proxy_pass $backend/;
|
|
}
|
|
|
|
location ~ ^/(llms|llms-full)\.txt$ {
|
|
proxy_hide_header Content-Type;
|
|
add_header Content-Type 'text/markdown; charset=utf-8' always;
|
|
proxy_pass $backend$uri;
|
|
}
|
|
|
|
# remove trailing slashes from all URLs (except root /)
|
|
# exact match locations (e.g., location = /python/) take priority over this regex
|
|
# Only match URIs composed of safe characters (letters, digits, dots, hyphens,
|
|
# underscores, forward slashes). This prevents open redirect via %5C (backslash):
|
|
# nginx decodes %5C to \ in $uri, and \ in the Location header gets normalized
|
|
# to / by browsers, turning /\evil.com into //evil.com (protocol-relative URL).
|
|
location ~ ^(/[a-zA-Z0-9][a-zA-Z0-9_./-]*)/$ {
|
|
rewrite ^(.+)/$ $1$is_args$args? redirect;
|
|
}
|
|
|
|
location / {
|
|
set $rewrite_condition "$serve_markdown$has_no_extension";
|
|
set $proxy_path $request_uri;
|
|
|
|
if ($rewrite_condition = "11") {
|
|
set $proxy_path "${request_uri}.md";
|
|
}
|
|
proxy_pass $backend$proxy_path;
|
|
}
|
|
|
|
### Repository path: "/python"
|
|
|
|
location = /python {
|
|
if ($serve_markdown) {
|
|
rewrite ^ /python/llms.txt last;
|
|
}
|
|
proxy_pass https://apify.github.io/crawlee-python/;
|
|
}
|
|
|
|
location = /python/ {
|
|
if ($serve_markdown) {
|
|
rewrite ^ /python/llms.txt last;
|
|
}
|
|
proxy_pass https://apify.github.io/crawlee-python/;
|
|
}
|
|
|
|
location ~ ^/python/(llms|llms-full)\.txt$ {
|
|
proxy_hide_header Content-Type;
|
|
add_header Content-Type 'text/markdown; charset=utf-8' always;
|
|
proxy_pass https://apify.github.io/crawlee-python/$1.txt;
|
|
}
|
|
|
|
location ~ ^/python/(.*)$ {
|
|
set $path_suffix $1;
|
|
set $proxy_path "/$path_suffix";
|
|
set $rewrite_condition "$serve_markdown$has_no_extension";
|
|
if ($rewrite_condition = "11") {
|
|
set $proxy_path "${proxy_path}.md";
|
|
}
|
|
proxy_pass https://apify.github.io/crawlee-python$proxy_path;
|
|
}
|
|
|
|
# So that we can have both GH pages and crawlee.dev/python working and loading assets from the same path
|
|
location /crawlee-python {
|
|
proxy_pass https://apify.github.io/crawlee-python/;
|
|
}
|
|
|
|
# Redirect rule for old JS docs to go under /js prefix
|
|
rewrite ^/docs(.*)$ /js/docs$1 permanent;
|
|
rewrite ^/api(.*)$ /js/api$1 permanent;
|
|
|
|
# Remove version numbers from /js/api/3.[0-9]/* and /js/docs/3.[0-9]/*
|
|
rewrite ^/js/api/3\.\d(/.*)?$ /js/api$1 permanent;
|
|
rewrite ^/js/docs/3\.\d(/.*)?$ /js/docs$1 permanent;
|
|
|
|
# Redirect rule for "upgrading-to-v03" to "upgrading-to-v0x"
|
|
rewrite ^/python/docs/upgrading/upgrading-to-v03$ /python/docs/upgrading/upgrading-to-v0x permanent;
|
|
|
|
# Redirect rule so that /python/docs actually leads somewhere
|
|
rewrite ^/python/docs/?$ /python/docs/quick-start;
|
|
|
|
rewrite ^/versions/?$ /js/api/core/changelog permanent;
|
|
}
|