chore: import upstream snapshot with attribution
This commit is contained in:
wehub-resource-sync
@@ -0,0 +1,12 @@
|
||||
name: Check PR title
|
||||
|
||||
on:
|
||||
pull_request_target:
|
||||
types: [ opened, edited, synchronize ]
|
||||
|
||||
jobs:
|
||||
check_pr_title:
|
||||
name: 'Check PR title'
|
||||
runs-on: ubuntu-22.04
|
||||
steps:
|
||||
- uses: apify/actions/pr-title-check@v1.4.0
|
||||
@@ -0,0 +1,24 @@
|
||||
# Triggers deployment of the Nginx reverse proxy configuration for crawlee.dev
|
||||
# when the configuration file is updated in this repository.
|
||||
name: Deploy Nginx Configuration
|
||||
|
||||
on:
|
||||
push:
|
||||
branches:
|
||||
- master
|
||||
paths:
|
||||
- 'website/nginx.conf'
|
||||
|
||||
jobs:
|
||||
trigger-deployment:
|
||||
name: Trigger Nginx deployment
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Trigger deployment workflow in apify-docs-private
|
||||
run: |
|
||||
gh workflow run deploy-nginx.yml \
|
||||
--repo apify/apify-docs-private \
|
||||
--field deployment=crawlee-web
|
||||
echo "✅ Deployment workflow triggered successfully"
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ secrets.APIFY_SERVICE_ACCOUNT_GITHUB_TOKEN }}
|
||||
@@ -0,0 +1,65 @@
|
||||
name: docs
|
||||
|
||||
on:
|
||||
push:
|
||||
branches:
|
||||
- master
|
||||
workflow_dispatch:
|
||||
|
||||
jobs:
|
||||
build:
|
||||
environment:
|
||||
name: github-pages
|
||||
permissions:
|
||||
contents: write
|
||||
pages: write
|
||||
id-token: write
|
||||
runs-on: ubuntu-22.04
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v7
|
||||
|
||||
- name: Use Node.js 24
|
||||
uses: actions/setup-node@v6
|
||||
with:
|
||||
node-version: 24
|
||||
package-manager-cache: false
|
||||
|
||||
- name: Enable corepack
|
||||
run: |
|
||||
corepack enable
|
||||
corepack prepare yarn@stable --activate
|
||||
|
||||
- name: Build & deploy docs
|
||||
run: |
|
||||
# install project deps
|
||||
yarn
|
||||
# go to website dir
|
||||
cd website
|
||||
# install website deps
|
||||
yarn
|
||||
# build the docs
|
||||
yarn build
|
||||
env:
|
||||
APIFY_SIGNING_TOKEN: ${{ secrets.APIFY_SIGNING_TOKEN }}
|
||||
SEGMENT_TOKEN: ${{ secrets.SEGMENT_TOKEN }}
|
||||
|
||||
- name: Set up GitHub Pages
|
||||
uses: actions/configure-pages@v6
|
||||
|
||||
- name: Upload GitHub Pages artifact
|
||||
uses: actions/upload-pages-artifact@v5
|
||||
with:
|
||||
path: ./website/build
|
||||
|
||||
- name: Deploy artifact to GitHub Pages
|
||||
uses: actions/deploy-pages@v5
|
||||
|
||||
- name: Invalidate CloudFront cache
|
||||
run: |
|
||||
gh workflow run invalidate-cloudfront.yml \
|
||||
--repo apify/apify-docs-private \
|
||||
--field deployment=crawlee-web
|
||||
echo "✅ CloudFront cache invalidation workflow triggered successfully"
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ secrets.APIFY_SERVICE_ACCOUNT_GITHUB_TOKEN }}
|
||||
@@ -0,0 +1,112 @@
|
||||
name: Publish to NPM
|
||||
|
||||
on:
|
||||
workflow_call:
|
||||
inputs:
|
||||
ref:
|
||||
description: Git ref to publish (branch, tag, or commit SHA)
|
||||
required: true
|
||||
type: string
|
||||
dist-tag:
|
||||
description: NPM dist-tag (prod or next)
|
||||
required: false
|
||||
type: string
|
||||
default: 'prod'
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
ref:
|
||||
description: Git ref to publish (branch, tag, or commit SHA)
|
||||
required: true
|
||||
type: string
|
||||
dist-tag:
|
||||
description: NPM dist-tag (prod or next)
|
||||
required: false
|
||||
type: string
|
||||
default: 'prod'
|
||||
|
||||
concurrency:
|
||||
group: publish-to-npm
|
||||
cancel-in-progress: false
|
||||
|
||||
permissions:
|
||||
id-token: write # Required for OIDC
|
||||
contents: write # Required to push updates
|
||||
|
||||
jobs:
|
||||
publish_to_npm:
|
||||
name: Publish to NPM (${{ inputs.dist-tag }})
|
||||
runs-on: ubuntu-22.04
|
||||
steps:
|
||||
- uses: actions/checkout@v7
|
||||
with:
|
||||
ref: ${{ inputs.ref }}
|
||||
token: ${{ secrets.APIFY_SERVICE_ACCOUNT_GITHUB_TOKEN }}
|
||||
fetch-depth: 0
|
||||
|
||||
- name: Use Node.js 24
|
||||
uses: actions/setup-node@v6
|
||||
with:
|
||||
node-version: 24
|
||||
package-manager-cache: false
|
||||
|
||||
- name: Enable corepack
|
||||
run: |
|
||||
corepack enable
|
||||
corepack prepare yarn@stable --activate
|
||||
|
||||
- name: Activate cache for Node.js 24
|
||||
uses: actions/setup-node@v6
|
||||
with:
|
||||
cache: 'yarn'
|
||||
|
||||
- name: Turbo cache
|
||||
id: turbo-cache
|
||||
uses: actions/cache@v6
|
||||
with:
|
||||
path: .turbo
|
||||
key: turbo-${{ github.job }}-${{ github.ref_name }}-${{ github.sha }}
|
||||
restore-keys: |
|
||||
turbo-${{ github.job }}-${{ github.ref_name }}-
|
||||
|
||||
- name: Install dependencies
|
||||
run: yarn
|
||||
|
||||
- name: Build packages
|
||||
run: yarn ci:build
|
||||
|
||||
- name: Bump canary versions
|
||||
if: inputs.dist-tag == 'next'
|
||||
run: |
|
||||
yarn turbo copy --force -- --canary --preid=beta
|
||||
|
||||
- name: Commit changes
|
||||
if: inputs.dist-tag == 'next'
|
||||
uses: EndBug/add-and-commit@v10
|
||||
id: commit
|
||||
with:
|
||||
author_name: Apify Release Bot
|
||||
author_email: noreply@apify.com
|
||||
message: 'chore: bump canary versions [skip ci]'
|
||||
push: false
|
||||
|
||||
- name: Publish to NPM (@latest)
|
||||
if: inputs.dist-tag == 'prod'
|
||||
uses: nick-fields/retry@v4
|
||||
with:
|
||||
timeout_minutes: 10
|
||||
max_attempts: 5
|
||||
retry_wait_seconds: 30
|
||||
command: git checkout . && yarn publish:prod --yes
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.APIFY_SERVICE_ACCOUNT_GITHUB_TOKEN }}
|
||||
|
||||
- name: Publish to NPM (@next)
|
||||
if: inputs.dist-tag == 'next'
|
||||
uses: nick-fields/retry@v4
|
||||
with:
|
||||
timeout_minutes: 10
|
||||
max_attempts: 5
|
||||
retry_wait_seconds: 30
|
||||
command: git checkout . && yarn publish:next --yes
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.APIFY_SERVICE_ACCOUNT_GITHUB_TOKEN }}
|
||||
@@ -0,0 +1,248 @@
|
||||
name: Release @latest
|
||||
|
||||
env:
|
||||
YARN_IGNORE_NODE: 1
|
||||
RETRY_TESTS: 1
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
version:
|
||||
description: The version to bump (if you choose custom, please include it under custom version)
|
||||
required: true
|
||||
default: 'patch'
|
||||
type: choice
|
||||
options:
|
||||
- 'patch'
|
||||
- 'minor'
|
||||
- 'major'
|
||||
- 'custom'
|
||||
custom_version:
|
||||
description: The custom version to bump to (only for "custom" type)
|
||||
required: false
|
||||
type: string
|
||||
default: ''
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.ref }}
|
||||
cancel-in-progress: true
|
||||
|
||||
jobs:
|
||||
# Test before releasing to ensure we don't release a broken version
|
||||
build_and_test:
|
||||
name: Build & Test
|
||||
if: (!contains(github.event.head_commit.message, '[skip ci]') && !contains(github.event.head_commit.message, 'docs:'))
|
||||
strategy:
|
||||
fail-fast: true
|
||||
matrix:
|
||||
# We don't test on Windows as the tests are flaky
|
||||
os: [ ubuntu-22.04 ]
|
||||
node-version: [ 20, 22, 24 ]
|
||||
|
||||
runs-on: ${{ matrix.os }}
|
||||
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@v7
|
||||
|
||||
- name: Use Node.js ${{ matrix.node-version }}
|
||||
uses: actions/setup-node@v6
|
||||
with:
|
||||
node-version: ${{ matrix.node-version }}
|
||||
package-manager-cache: false
|
||||
|
||||
- name: Enable corepack
|
||||
run: |
|
||||
corepack enable
|
||||
corepack prepare yarn@stable --activate
|
||||
|
||||
- name: Activate cache for Node.js ${{ matrix.node-version }}
|
||||
uses: actions/setup-node@v6
|
||||
with:
|
||||
cache: 'yarn'
|
||||
|
||||
- name: Turbo cache
|
||||
id: turbo-cache
|
||||
uses: actions/cache@v6
|
||||
with:
|
||||
path: .turbo
|
||||
key: turbo-${{ github.job }}-${{ github.ref_name }}-${{ github.sha }}
|
||||
restore-keys: |
|
||||
turbo-${{ github.job }}-${{ github.ref_name }}-
|
||||
|
||||
- name: Install Dependencies
|
||||
run: yarn
|
||||
|
||||
- name: Cache Playwright browsers
|
||||
uses: actions/cache@v6
|
||||
with:
|
||||
path: ~/.cache/ms-playwright
|
||||
key: playwright-${{ runner.os }}-${{ hashFiles('**/yarn.lock') }}
|
||||
restore-keys: |
|
||||
playwright-${{ runner.os }}-
|
||||
|
||||
- name: Install Playwright browsers
|
||||
timeout-minutes: 10
|
||||
run: yarn playwright install --with-deps
|
||||
|
||||
- name: Build
|
||||
run: yarn ci:build
|
||||
|
||||
- name: Tests
|
||||
run: yarn test
|
||||
|
||||
release:
|
||||
name: "Bump Crawlee: ${{ inputs.version }} version (${{ inputs.custom_version || 'n/a' }} custom version)"
|
||||
if: (!contains(github.event.head_commit.message, '[skip ci]') && !contains(github.event.head_commit.message, 'docs:'))
|
||||
needs: build_and_test
|
||||
runs-on: ubuntu-22.04
|
||||
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@v7
|
||||
with:
|
||||
token: ${{ secrets.APIFY_SERVICE_ACCOUNT_GITHUB_TOKEN }}
|
||||
fetch-depth: 0
|
||||
|
||||
- name: Use Node.js 24
|
||||
uses: actions/setup-node@v6
|
||||
with:
|
||||
node-version: 24
|
||||
package-manager-cache: false
|
||||
|
||||
- name: Enable corepack
|
||||
run: |
|
||||
corepack enable
|
||||
corepack prepare yarn@stable --activate
|
||||
|
||||
- name: Activate cache for Node.js 24
|
||||
uses: actions/setup-node@v6
|
||||
with:
|
||||
cache: 'yarn'
|
||||
|
||||
- name: Turbo cache
|
||||
id: turbo-cache
|
||||
uses: actions/cache@v6
|
||||
with:
|
||||
path: .turbo
|
||||
key: turbo-${{ github.job }}-${{ github.ref_name }}-${{ github.sha }}
|
||||
restore-keys: |
|
||||
turbo-${{ github.job }}-${{ github.ref_name }}-
|
||||
|
||||
- name: Install Dependencies
|
||||
run: yarn
|
||||
|
||||
- name: Build
|
||||
run: yarn ci:build
|
||||
|
||||
- name: Bump version to custom version
|
||||
if: ${{ github.event.inputs.version == 'custom' && github.event.inputs.custom_version != '' }}
|
||||
run: yarn lerna version ${{ github.event.inputs.custom_version }} --force-publish --yes
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.APIFY_SERVICE_ACCOUNT_GITHUB_TOKEN }}
|
||||
GIT_AUTHOR_NAME: Apify Release Bot
|
||||
GIT_COMMITTER_NAME: Apify Release Bot
|
||||
GIT_AUTHOR_EMAIL: noreply@apify.com
|
||||
GIT_COMMITTER_EMAIL: noreply@apify.com
|
||||
|
||||
- name: Bump version to ${{ github.event.inputs.version }} version
|
||||
if: ${{ github.event.inputs.version != 'custom' }}
|
||||
run: yarn lerna version ${{ github.event.inputs.version }} --force-publish --yes
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.APIFY_SERVICE_ACCOUNT_GITHUB_TOKEN }}
|
||||
GIT_AUTHOR_NAME: Apify Release Bot
|
||||
GIT_COMMITTER_NAME: Apify Release Bot
|
||||
GIT_AUTHOR_EMAIL: noreply@apify.com
|
||||
GIT_COMMITTER_EMAIL: noreply@apify.com
|
||||
|
||||
- name: Pin versions in internal dependencies and update lockfile
|
||||
run: |
|
||||
yarn release:pin-versions
|
||||
yarn install --no-immutable
|
||||
|
||||
- name: Commit changes
|
||||
id: commit
|
||||
uses: EndBug/add-and-commit@v10
|
||||
with:
|
||||
author_name: Apify Release Bot
|
||||
author_email: noreply@apify.com
|
||||
message: 'chore(release): update internal dependencies [skip ci]'
|
||||
pull: '--rebase --autostash'
|
||||
|
||||
- name: Publish packages
|
||||
uses: apify/actions/execute-workflow@v1.4.0
|
||||
with:
|
||||
workflow: publish-to-npm.yml
|
||||
inputs: >
|
||||
{
|
||||
"ref": "${{ steps.commit.outputs.commit_long_sha || github.sha }}",
|
||||
"dist-tag": "prod"
|
||||
}
|
||||
|
||||
- name: Trigger Docker image builds
|
||||
uses: peter-evans/repository-dispatch@v4
|
||||
with:
|
||||
token: ${{ secrets.APIFY_SERVICE_ACCOUNT_GITHUB_TOKEN }}
|
||||
repository: apify/apify-actor-docker
|
||||
event-type: build-node-images
|
||||
|
||||
version-docs:
|
||||
needs: release
|
||||
runs-on: ubuntu-22.04
|
||||
if: (github.event.inputs.version == 'minor' || github.event.inputs.version == 'major')
|
||||
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@v7
|
||||
with:
|
||||
ref: ${{ github.event.repository.default_branch }}
|
||||
token: ${{ secrets.APIFY_SERVICE_ACCOUNT_GITHUB_TOKEN }}
|
||||
fetch-depth: 0
|
||||
|
||||
- name: Use Node.js 24
|
||||
uses: actions/setup-node@v6
|
||||
with:
|
||||
node-version: 24
|
||||
package-manager-cache: false
|
||||
|
||||
- name: Install jq
|
||||
run: sudo apt-get install jq
|
||||
|
||||
- name: Enable corepack
|
||||
run: |
|
||||
corepack enable
|
||||
corepack prepare yarn@stable --activate
|
||||
|
||||
- name: Activate cache for Node.js 24
|
||||
uses: actions/setup-node@v6
|
||||
with:
|
||||
cache: 'yarn'
|
||||
|
||||
- name: Install dependencies
|
||||
run: |
|
||||
# install project deps
|
||||
yarn
|
||||
# install website deps
|
||||
cd website
|
||||
yarn
|
||||
|
||||
- name: Snapshot the current version
|
||||
run: |
|
||||
cd website
|
||||
VERSION=$(jq -r '.version' ../lerna.json)
|
||||
if [ -z "$VERSION" ]; then
|
||||
echo "Error: VERSION is empty. Ensure lerna.json exists and contains a valid .version field."
|
||||
exit 1
|
||||
fi
|
||||
echo "VERSION=$VERSION" >> "$GITHUB_ENV"
|
||||
MAJOR_MINOR=$(echo $VERSION | cut -d. -f1,2)
|
||||
yarn docusaurus docs:version $MAJOR_MINOR
|
||||
yarn docusaurus api:version $MAJOR_MINOR
|
||||
|
||||
- name: Commit and push the version snapshot
|
||||
uses: EndBug/add-and-commit@v10
|
||||
with:
|
||||
author_name: Apify Release Bot
|
||||
author_email: noreply@apify.com
|
||||
message: 'docs: update docs for ${{ env.VERSION }} version'
|
||||
pull: '--rebase --autostash'
|
||||
@@ -0,0 +1,369 @@
|
||||
name: Check
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [ master, renovate/** ]
|
||||
pull_request:
|
||||
branches: [ master ]
|
||||
|
||||
env:
|
||||
YARN_IGNORE_NODE: 1
|
||||
RETRY_TESTS: 1
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.ref }}
|
||||
cancel-in-progress: true
|
||||
|
||||
jobs:
|
||||
# `yarn install` is done in a separate job and cached to speed up the following jobs.
|
||||
build_and_test:
|
||||
name: Build & Test
|
||||
if: (!contains(github.event.head_commit.message, '[skip ci]') && !contains(github.event.head_commit.message, 'docs:'))
|
||||
runs-on: ${{ matrix.os }}
|
||||
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
# tests on windows are extremely unstable
|
||||
# os: [ ubuntu-22.04, windows-2019 ]
|
||||
os: [ ubuntu-22.04 ]
|
||||
node-version: [ 20, 22, 24 ]
|
||||
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@v7
|
||||
|
||||
- name: Use Node.js ${{ matrix.node-version }}
|
||||
uses: actions/setup-node@v6
|
||||
with:
|
||||
node-version: ${{ matrix.node-version }}
|
||||
package-manager-cache: false
|
||||
|
||||
- name: Enable corepack
|
||||
run: |
|
||||
corepack enable
|
||||
corepack prepare yarn@stable --activate
|
||||
|
||||
- name: Activate cache for Node.js ${{ matrix.node-version }}
|
||||
uses: actions/setup-node@v6
|
||||
with:
|
||||
cache: 'yarn'
|
||||
|
||||
- name: Turbo cache
|
||||
id: turbo-cache
|
||||
uses: actions/cache@v6
|
||||
with:
|
||||
path: .turbo
|
||||
key: turbo-${{ github.job }}-${{ matrix.node-version }}-${{ github.ref_name }}-${{ github.sha }}
|
||||
restore-keys: |
|
||||
turbo-${{ github.job }}-${{ matrix.node-version }}-${{ github.ref_name }}-
|
||||
|
||||
- name: Install Dependencies
|
||||
run: yarn
|
||||
env:
|
||||
YARN_IGNORE_NODE: 1
|
||||
|
||||
- name: Cache Playwright browsers
|
||||
uses: actions/cache@v6
|
||||
with:
|
||||
path: ~/.cache/ms-playwright
|
||||
key: playwright-${{ runner.os }}-${{ hashFiles('**/yarn.lock') }}
|
||||
restore-keys: |
|
||||
playwright-${{ runner.os }}-
|
||||
|
||||
- name: Install Playwright browsers
|
||||
timeout-minutes: 10
|
||||
run: yarn playwright install --with-deps
|
||||
env:
|
||||
YARN_IGNORE_NODE: 1
|
||||
|
||||
- name: Build
|
||||
run: yarn ci:build
|
||||
env:
|
||||
YARN_IGNORE_NODE: 1
|
||||
|
||||
- name: Test TS
|
||||
run: yarn tsc-check-tests
|
||||
env:
|
||||
YARN_IGNORE_NODE: 1
|
||||
|
||||
- name: Typecheck documentation examples
|
||||
working-directory: ./docs
|
||||
run: |
|
||||
yarn
|
||||
yarn typecheck
|
||||
env:
|
||||
YARN_IGNORE_NODE: 1
|
||||
|
||||
- name: Tests
|
||||
run: yarn test
|
||||
env:
|
||||
YARN_IGNORE_NODE: 1
|
||||
|
||||
docs:
|
||||
name: Docs build
|
||||
if: (!contains(github.event.head_commit.message, '[skip ci]') && github.ref != 'refs/heads/master')
|
||||
runs-on: ubuntu-22.04
|
||||
steps:
|
||||
- name: Checkout Source code
|
||||
uses: actions/checkout@v7
|
||||
|
||||
- name: Use Node.js 24
|
||||
uses: actions/setup-node@v6
|
||||
with:
|
||||
node-version: 24
|
||||
package-manager-cache: false
|
||||
|
||||
- name: Enable corepack
|
||||
run: |
|
||||
corepack enable
|
||||
corepack prepare yarn@stable --activate
|
||||
|
||||
- name: Activate cache for Node.js 24
|
||||
uses: actions/setup-node@v6
|
||||
with:
|
||||
cache: 'yarn'
|
||||
|
||||
- name: Turbo cache
|
||||
id: turbo-cache
|
||||
uses: actions/cache@v6
|
||||
with:
|
||||
path: .turbo
|
||||
key: turbo-${{ github.job }}-${{ github.ref_name }}-${{ github.sha }}
|
||||
restore-keys: |
|
||||
turbo-${{ github.job }}-${{ github.ref_name }}-
|
||||
|
||||
- name: Install Dependencies
|
||||
run: yarn
|
||||
|
||||
- name: Build & deploy docs
|
||||
run: |
|
||||
cd website
|
||||
yarn
|
||||
yarn build
|
||||
env:
|
||||
APIFY_SIGNING_TOKEN: ${{ secrets.APIFY_SIGNING_TOKEN }}
|
||||
SEGMENT_TOKEN: ${{ secrets.SEGMENT_TOKEN }}
|
||||
|
||||
- name: Install Nginx
|
||||
run: |
|
||||
sudo apt-get update
|
||||
sudo apt-get install -y nginx
|
||||
|
||||
- name: Start Docusaurus server
|
||||
run: |
|
||||
cd website
|
||||
nohup yarn docusaurus serve --port 3000 --no-open &
|
||||
sleep 5
|
||||
curl -f http://localhost:3000 > /dev/null
|
||||
|
||||
- name: Start Nginx with project config
|
||||
run: |
|
||||
PWD_PATH="$(pwd)"
|
||||
cat > default.conf <<EOF
|
||||
worker_processes auto;
|
||||
error_log ${PWD_PATH}/logs/error.log;
|
||||
pid ${PWD_PATH}/logs/nginx.pid;
|
||||
events {}
|
||||
http {
|
||||
access_log ${PWD_PATH}/logs/access.log;
|
||||
include ${PWD_PATH}/website/nginx.conf;
|
||||
}
|
||||
EOF
|
||||
sed -i 's|https://apify.github.io/crawlee|http://localhost:3000|g' default.conf
|
||||
mkdir -p "${PWD_PATH}/logs"
|
||||
nginx -c "${PWD_PATH}/default.conf"
|
||||
sleep 1
|
||||
|
||||
- name: Run header assertions
|
||||
run: |
|
||||
set -euo pipefail
|
||||
function assert_header() {
|
||||
url=$1
|
||||
header=$2
|
||||
expected=$3
|
||||
shift 3
|
||||
extra_args=("$@")
|
||||
actual=$(curl -s -D - -o /dev/null "${extra_args[@]}" "$url" | grep -i "^$header" | tr -d '\r' || true)
|
||||
echo "→ $url → $actual"
|
||||
echo "$actual" | grep -q "$expected" || (echo "❌ Expected '$expected' in '$header' for $url" && exit 1)
|
||||
}
|
||||
|
||||
function assert_status() {
|
||||
url=$1
|
||||
expected=$2
|
||||
shift 2
|
||||
extra_args=("$@")
|
||||
actual=$(curl -s -o /dev/null -w "%{http_code}" "${extra_args[@]}" "$url")
|
||||
echo "→ $url → HTTP $actual"
|
||||
[ "$actual" = "$expected" ] || (echo "❌ Expected HTTP $expected but got $actual for $url" && exit 1)
|
||||
}
|
||||
|
||||
function assert_no_redirect() {
|
||||
url=$1
|
||||
shift
|
||||
extra_args=("$@")
|
||||
response=$(curl -s -D - -o /dev/null -w "\n%{http_code}" "${extra_args[@]}" "$url" 2>/dev/null)
|
||||
status=$(echo "$response" | tail -1)
|
||||
location=$(echo "$response" | grep -i "^location:" | tr -d '\r' || true)
|
||||
echo "→ $url → HTTP $status ${location:+(${location})}"
|
||||
if [ "$status" = "301" ] || [ "$status" = "302" ]; then
|
||||
echo "❌ Got redirect for $url: $location" && exit 1
|
||||
fi
|
||||
}
|
||||
|
||||
echo "🧪 Checking open redirect protection..."
|
||||
assert_no_redirect "http://localhost:8080///%5Cevil.com/"
|
||||
assert_no_redirect "http://localhost:8080/%5Cevil.com/"
|
||||
assert_no_redirect "http://localhost:8080///%5cevil.com/"
|
||||
assert_no_redirect "http://localhost:8080" --request-target '/\evil.com/'
|
||||
assert_no_redirect "http://localhost:8080" --request-target '///\evil.com/'
|
||||
assert_status "http://localhost:8080/js/docs/quick-start/" "302"
|
||||
|
||||
echo "🧪 Checking Nginx responses... (crawlee JS)"
|
||||
assert_header "http://localhost:8080/" "Content-Type" "text/html"
|
||||
assert_header "http://localhost:8080/" "Content-Type" "text/markdown" -H "Accept: text/markdown"
|
||||
assert_header "http://localhost:8080/js/docs/quick-start" "Content-Type" "text/html"
|
||||
assert_header "http://localhost:8080/js/docs/quick-start.md" "Content-Type" "text/markdown"
|
||||
assert_header "http://localhost:8080/js/docs/quick-start" "Content-Type" "text/markdown" -H "Accept: text/markdown"
|
||||
assert_header "http://localhost:8080/llms.txt" "Content-Type" "text/markdown"
|
||||
assert_header "http://localhost:8080/llms-full.txt" "Content-Type" "text/markdown"
|
||||
|
||||
echo "🧪 Checking Nginx responses... (crawlee Python)"
|
||||
assert_header "http://localhost:8080/python/docs/quick-start" "Content-Type" "text/html"
|
||||
assert_header "http://localhost:8080/python/docs/quick-start.md" "Content-Type" "text/markdown"
|
||||
assert_header "http://localhost:8080/python/docs/quick-start" "Content-Type" "text/markdown" -H "Accept: text/markdown"
|
||||
assert_header "http://localhost:8080/python/llms.txt" "Content-Type" "text/markdown"
|
||||
assert_header "http://localhost:8080/python/llms-full.txt" "Content-Type" "text/markdown"
|
||||
|
||||
echo "✅ All Nginx header checks passed."
|
||||
|
||||
- name: Stop Nginx
|
||||
if: always()
|
||||
run: nginx -c "$(pwd)/default.conf" -s stop
|
||||
|
||||
lint:
|
||||
name: Lint
|
||||
runs-on: ubuntu-22.04
|
||||
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@v7
|
||||
|
||||
- name: Use Node.js 24
|
||||
uses: actions/setup-node@v6
|
||||
with:
|
||||
node-version: 24
|
||||
package-manager-cache: false
|
||||
|
||||
- name: Enable corepack
|
||||
run: |
|
||||
corepack enable
|
||||
corepack prepare yarn@stable --activate
|
||||
|
||||
- name: Activate cache for Node.js 24
|
||||
uses: actions/setup-node@v6
|
||||
with:
|
||||
cache: 'yarn'
|
||||
|
||||
- name: Turbo cache
|
||||
id: turbo-cache
|
||||
uses: actions/cache@v6
|
||||
with:
|
||||
path: .turbo
|
||||
key: turbo-${{ github.job }}-${{ github.ref_name }}-${{ github.sha }}
|
||||
restore-keys: |
|
||||
turbo-${{ github.job }}-${{ github.ref_name }}-
|
||||
|
||||
- name: Install Dependencies
|
||||
run: yarn
|
||||
|
||||
- name: ESLint
|
||||
run: yarn lint
|
||||
|
||||
- name: Biome format
|
||||
run: yarn format:check
|
||||
|
||||
release_next:
|
||||
name: Release @next
|
||||
if: github.event_name == 'push' && contains(github.event.ref, 'master') && (!contains(github.event.head_commit.message, '[skip ci]') && !contains(github.event.head_commit.message, 'docs:'))
|
||||
needs: build_and_test
|
||||
runs-on: ubuntu-22.04
|
||||
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@v7
|
||||
with:
|
||||
token: ${{ secrets.APIFY_SERVICE_ACCOUNT_GITHUB_TOKEN }}
|
||||
fetch-depth: 0
|
||||
|
||||
- name: Use Node.js 24
|
||||
uses: actions/setup-node@v6
|
||||
with:
|
||||
node-version: 24
|
||||
package-manager-cache: false
|
||||
|
||||
- name: Enable corepack
|
||||
run: |
|
||||
corepack enable
|
||||
corepack prepare yarn@stable --activate
|
||||
|
||||
- name: Activate cache for Node.js 24
|
||||
uses: actions/setup-node@v6
|
||||
with:
|
||||
cache: 'yarn'
|
||||
|
||||
- name: Turbo cache
|
||||
id: turbo-cache
|
||||
uses: actions/cache@v6
|
||||
with:
|
||||
path: .turbo
|
||||
key: turbo-${{ github.job }}-${{ github.ref_name }}-${{ github.sha }}
|
||||
restore-keys: |
|
||||
turbo-${{ github.job }}-${{ github.ref_name }}-
|
||||
|
||||
- name: Install Dependencies
|
||||
run: yarn
|
||||
|
||||
- name: Build
|
||||
run: yarn ci:build
|
||||
|
||||
- name: Generate changed packages list
|
||||
id: changed-packages
|
||||
run: |
|
||||
echo "changed_packages=$(node ./node_modules/.bin/lerna changed -p | wc -l | xargs)" | tee -a $GITHUB_OUTPUT
|
||||
|
||||
- name: Report nothing to release
|
||||
if: steps.changed-packages.outputs.changed_packages == '0'
|
||||
run: echo "Nothing to release"
|
||||
|
||||
- name: Publish packages
|
||||
if: steps.changed-packages.outputs.changed_packages != '0'
|
||||
uses: apify/actions/execute-workflow@v1.4.0
|
||||
with:
|
||||
workflow: publish-to-npm.yml
|
||||
inputs: >
|
||||
{
|
||||
"ref": "${{ steps.commit.outputs.commit_long_sha || github.sha }}",
|
||||
"dist-tag": "next"
|
||||
}
|
||||
|
||||
- name: Collect versions for Docker images
|
||||
id: versions
|
||||
run: |
|
||||
crawlee=`node -p "require('./packages/crawlee/package.json').version"`
|
||||
echo "crawlee=$crawlee" | tee -a $GITHUB_OUTPUT
|
||||
|
||||
- name: Trigger Docker image builds
|
||||
uses: peter-evans/repository-dispatch@v4
|
||||
# Trigger next images only if we have something new pushed
|
||||
if: steps.changed-packages.outputs.changed_packages != '0'
|
||||
with:
|
||||
token: ${{ secrets.APIFY_SERVICE_ACCOUNT_GITHUB_TOKEN }}
|
||||
repository: apify/apify-actor-docker
|
||||
event-type: build-node-images
|
||||
client-payload: >
|
||||
{
|
||||
"crawlee_version": "${{ steps.versions.outputs.crawlee }}",
|
||||
"release_tag": "beta"
|
||||
}
|
||||
@@ -0,0 +1,92 @@
|
||||
name: E2E tests
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
schedule:
|
||||
# Runs at 2 am every day
|
||||
- cron: '0 2 * * *'
|
||||
|
||||
env:
|
||||
YARN_IGNORE_NODE: 1
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.ref }}
|
||||
cancel-in-progress: true
|
||||
|
||||
jobs:
|
||||
# NPM install is done in a separate job and cached to speed up the following jobs.
|
||||
build_and_test:
|
||||
name: Build & Test
|
||||
runs-on: ubuntu-22.04
|
||||
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
storage: [ LOCAL, MEMORY, PLATFORM ]
|
||||
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@v7
|
||||
|
||||
- name: Use Node.js 24
|
||||
uses: actions/setup-node@v6
|
||||
with:
|
||||
node-version: 24
|
||||
package-manager-cache: false
|
||||
|
||||
- name: Enable corepack
|
||||
run: |
|
||||
corepack enable
|
||||
corepack prepare yarn@stable --activate
|
||||
|
||||
- name: Activate cache for Node.js 24
|
||||
uses: actions/setup-node@v6
|
||||
with:
|
||||
cache: 'yarn'
|
||||
|
||||
- name: Turbo cache
|
||||
id: turbo-cache
|
||||
uses: actions/cache@v6
|
||||
with:
|
||||
path: .turbo
|
||||
key: turbo-${{ github.job }}-${{ github.ref_name }}-${{ github.sha }}
|
||||
restore-keys: |
|
||||
turbo-${{ github.job }}-${{ github.ref_name }}-
|
||||
|
||||
|
||||
- name: Setup Apify CLI
|
||||
uses: apify/setup-apify-cli-action@main
|
||||
with:
|
||||
version: 'beta'
|
||||
token: ${{ secrets.APIFY_SCRAPER_TESTS_API_TOKEN }}
|
||||
|
||||
- name: Add Apify secrets for E2E tests
|
||||
run: apify secrets add anthropicApiKey ${{ secrets.ANTHROPIC_API_KEY }}
|
||||
|
||||
- name: Install Dependencies
|
||||
run: yarn
|
||||
|
||||
- name: Cache Playwright browsers
|
||||
if: (matrix.storage != 'PLATFORM')
|
||||
uses: actions/cache@v6
|
||||
with:
|
||||
path: ~/.cache/ms-playwright
|
||||
key: playwright-${{ runner.os }}-${{ hashFiles('**/yarn.lock') }}
|
||||
restore-keys: |
|
||||
playwright-${{ runner.os }}-
|
||||
|
||||
- name: Install Playwright Dependencies
|
||||
if: (matrix.storage != 'PLATFORM')
|
||||
timeout-minutes: 10
|
||||
run: yarn playwright install --with-deps
|
||||
|
||||
- name: Build
|
||||
run: yarn ci:build
|
||||
|
||||
- name: Test with storage ${{ matrix.storage }}
|
||||
run: yarn test:e2e
|
||||
env:
|
||||
STORAGE_IMPLEMENTATION: ${{ matrix.storage }}
|
||||
APIFY_HTTPBIN_TOKEN: ${{ secrets.APIFY_HTTPBIN_TOKEN }}
|
||||
ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
|
||||
GITHUB_TOKEN: ${{ github.token }}
|
||||
@@ -0,0 +1,25 @@
|
||||
name: Update new issue
|
||||
|
||||
on:
|
||||
issues:
|
||||
types:
|
||||
- opened
|
||||
|
||||
jobs:
|
||||
label_issues:
|
||||
name: Label issues
|
||||
runs-on: ubuntu-22.04
|
||||
permissions:
|
||||
issues: write
|
||||
|
||||
steps:
|
||||
# Add the "t-tooling" label to all new issues
|
||||
- uses: actions/github-script@v9
|
||||
with:
|
||||
script: |
|
||||
github.rest.issues.addLabels({
|
||||
issue_number: context.issue.number,
|
||||
owner: context.repo.owner,
|
||||
repo: context.repo.repo,
|
||||
labels: ["t-tooling"]
|
||||
})
|
||||
Reference in New Issue
Block a user