wehub-resource-sync
370 lines
14 KiB
YAML
370 lines
14 KiB
YAML
name: Check
|
|
|
|
on:
|
|
push:
|
|
branches: [ master, renovate/** ]
|
|
pull_request:
|
|
branches: [ master ]
|
|
|
|
env:
|
|
YARN_IGNORE_NODE: 1
|
|
RETRY_TESTS: 1
|
|
|
|
concurrency:
|
|
group: ${{ github.workflow }}-${{ github.ref }}
|
|
cancel-in-progress: true
|
|
|
|
jobs:
|
|
# `yarn install` is done in a separate job and cached to speed up the following jobs.
|
|
build_and_test:
|
|
name: Build & Test
|
|
if: (!contains(github.event.head_commit.message, '[skip ci]') && !contains(github.event.head_commit.message, 'docs:'))
|
|
runs-on: ${{ matrix.os }}
|
|
|
|
strategy:
|
|
fail-fast: false
|
|
matrix:
|
|
# tests on windows are extremely unstable
|
|
# os: [ ubuntu-22.04, windows-2019 ]
|
|
os: [ ubuntu-22.04 ]
|
|
node-version: [ 20, 22, 24 ]
|
|
|
|
steps:
|
|
- name: Checkout repository
|
|
uses: actions/checkout@v7
|
|
|
|
- name: Use Node.js ${{ matrix.node-version }}
|
|
uses: actions/setup-node@v6
|
|
with:
|
|
node-version: ${{ matrix.node-version }}
|
|
package-manager-cache: false
|
|
|
|
- name: Enable corepack
|
|
run: |
|
|
corepack enable
|
|
corepack prepare yarn@stable --activate
|
|
|
|
- name: Activate cache for Node.js ${{ matrix.node-version }}
|
|
uses: actions/setup-node@v6
|
|
with:
|
|
cache: 'yarn'
|
|
|
|
- name: Turbo cache
|
|
id: turbo-cache
|
|
uses: actions/cache@v6
|
|
with:
|
|
path: .turbo
|
|
key: turbo-${{ github.job }}-${{ matrix.node-version }}-${{ github.ref_name }}-${{ github.sha }}
|
|
restore-keys: |
|
|
turbo-${{ github.job }}-${{ matrix.node-version }}-${{ github.ref_name }}-
|
|
|
|
- name: Install Dependencies
|
|
run: yarn
|
|
env:
|
|
YARN_IGNORE_NODE: 1
|
|
|
|
- name: Cache Playwright browsers
|
|
uses: actions/cache@v6
|
|
with:
|
|
path: ~/.cache/ms-playwright
|
|
key: playwright-${{ runner.os }}-${{ hashFiles('**/yarn.lock') }}
|
|
restore-keys: |
|
|
playwright-${{ runner.os }}-
|
|
|
|
- name: Install Playwright browsers
|
|
timeout-minutes: 10
|
|
run: yarn playwright install --with-deps
|
|
env:
|
|
YARN_IGNORE_NODE: 1
|
|
|
|
- name: Build
|
|
run: yarn ci:build
|
|
env:
|
|
YARN_IGNORE_NODE: 1
|
|
|
|
- name: Test TS
|
|
run: yarn tsc-check-tests
|
|
env:
|
|
YARN_IGNORE_NODE: 1
|
|
|
|
- name: Typecheck documentation examples
|
|
working-directory: ./docs
|
|
run: |
|
|
yarn
|
|
yarn typecheck
|
|
env:
|
|
YARN_IGNORE_NODE: 1
|
|
|
|
- name: Tests
|
|
run: yarn test
|
|
env:
|
|
YARN_IGNORE_NODE: 1
|
|
|
|
docs:
|
|
name: Docs build
|
|
if: (!contains(github.event.head_commit.message, '[skip ci]') && github.ref != 'refs/heads/master')
|
|
runs-on: ubuntu-22.04
|
|
steps:
|
|
- name: Checkout Source code
|
|
uses: actions/checkout@v7
|
|
|
|
- name: Use Node.js 24
|
|
uses: actions/setup-node@v6
|
|
with:
|
|
node-version: 24
|
|
package-manager-cache: false
|
|
|
|
- name: Enable corepack
|
|
run: |
|
|
corepack enable
|
|
corepack prepare yarn@stable --activate
|
|
|
|
- name: Activate cache for Node.js 24
|
|
uses: actions/setup-node@v6
|
|
with:
|
|
cache: 'yarn'
|
|
|
|
- name: Turbo cache
|
|
id: turbo-cache
|
|
uses: actions/cache@v6
|
|
with:
|
|
path: .turbo
|
|
key: turbo-${{ github.job }}-${{ github.ref_name }}-${{ github.sha }}
|
|
restore-keys: |
|
|
turbo-${{ github.job }}-${{ github.ref_name }}-
|
|
|
|
- name: Install Dependencies
|
|
run: yarn
|
|
|
|
- name: Build & deploy docs
|
|
run: |
|
|
cd website
|
|
yarn
|
|
yarn build
|
|
env:
|
|
APIFY_SIGNING_TOKEN: ${{ secrets.APIFY_SIGNING_TOKEN }}
|
|
SEGMENT_TOKEN: ${{ secrets.SEGMENT_TOKEN }}
|
|
|
|
- name: Install Nginx
|
|
run: |
|
|
sudo apt-get update
|
|
sudo apt-get install -y nginx
|
|
|
|
- name: Start Docusaurus server
|
|
run: |
|
|
cd website
|
|
nohup yarn docusaurus serve --port 3000 --no-open &
|
|
sleep 5
|
|
curl -f http://localhost:3000 > /dev/null
|
|
|
|
- name: Start Nginx with project config
|
|
run: |
|
|
PWD_PATH="$(pwd)"
|
|
cat > default.conf <<EOF
|
|
worker_processes auto;
|
|
error_log ${PWD_PATH}/logs/error.log;
|
|
pid ${PWD_PATH}/logs/nginx.pid;
|
|
events {}
|
|
http {
|
|
access_log ${PWD_PATH}/logs/access.log;
|
|
include ${PWD_PATH}/website/nginx.conf;
|
|
}
|
|
EOF
|
|
sed -i 's|https://apify.github.io/crawlee|http://localhost:3000|g' default.conf
|
|
mkdir -p "${PWD_PATH}/logs"
|
|
nginx -c "${PWD_PATH}/default.conf"
|
|
sleep 1
|
|
|
|
- name: Run header assertions
|
|
run: |
|
|
set -euo pipefail
|
|
function assert_header() {
|
|
url=$1
|
|
header=$2
|
|
expected=$3
|
|
shift 3
|
|
extra_args=("$@")
|
|
actual=$(curl -s -D - -o /dev/null "${extra_args[@]}" "$url" | grep -i "^$header" | tr -d '\r' || true)
|
|
echo "→ $url → $actual"
|
|
echo "$actual" | grep -q "$expected" || (echo "❌ Expected '$expected' in '$header' for $url" && exit 1)
|
|
}
|
|
|
|
function assert_status() {
|
|
url=$1
|
|
expected=$2
|
|
shift 2
|
|
extra_args=("$@")
|
|
actual=$(curl -s -o /dev/null -w "%{http_code}" "${extra_args[@]}" "$url")
|
|
echo "→ $url → HTTP $actual"
|
|
[ "$actual" = "$expected" ] || (echo "❌ Expected HTTP $expected but got $actual for $url" && exit 1)
|
|
}
|
|
|
|
function assert_no_redirect() {
|
|
url=$1
|
|
shift
|
|
extra_args=("$@")
|
|
response=$(curl -s -D - -o /dev/null -w "\n%{http_code}" "${extra_args[@]}" "$url" 2>/dev/null)
|
|
status=$(echo "$response" | tail -1)
|
|
location=$(echo "$response" | grep -i "^location:" | tr -d '\r' || true)
|
|
echo "→ $url → HTTP $status ${location:+(${location})}"
|
|
if [ "$status" = "301" ] || [ "$status" = "302" ]; then
|
|
echo "❌ Got redirect for $url: $location" && exit 1
|
|
fi
|
|
}
|
|
|
|
echo "🧪 Checking open redirect protection..."
|
|
assert_no_redirect "http://localhost:8080///%5Cevil.com/"
|
|
assert_no_redirect "http://localhost:8080/%5Cevil.com/"
|
|
assert_no_redirect "http://localhost:8080///%5cevil.com/"
|
|
assert_no_redirect "http://localhost:8080" --request-target '/\evil.com/'
|
|
assert_no_redirect "http://localhost:8080" --request-target '///\evil.com/'
|
|
assert_status "http://localhost:8080/js/docs/quick-start/" "302"
|
|
|
|
echo "🧪 Checking Nginx responses... (crawlee JS)"
|
|
assert_header "http://localhost:8080/" "Content-Type" "text/html"
|
|
assert_header "http://localhost:8080/" "Content-Type" "text/markdown" -H "Accept: text/markdown"
|
|
assert_header "http://localhost:8080/js/docs/quick-start" "Content-Type" "text/html"
|
|
assert_header "http://localhost:8080/js/docs/quick-start.md" "Content-Type" "text/markdown"
|
|
assert_header "http://localhost:8080/js/docs/quick-start" "Content-Type" "text/markdown" -H "Accept: text/markdown"
|
|
assert_header "http://localhost:8080/llms.txt" "Content-Type" "text/markdown"
|
|
assert_header "http://localhost:8080/llms-full.txt" "Content-Type" "text/markdown"
|
|
|
|
echo "🧪 Checking Nginx responses... (crawlee Python)"
|
|
assert_header "http://localhost:8080/python/docs/quick-start" "Content-Type" "text/html"
|
|
assert_header "http://localhost:8080/python/docs/quick-start.md" "Content-Type" "text/markdown"
|
|
assert_header "http://localhost:8080/python/docs/quick-start" "Content-Type" "text/markdown" -H "Accept: text/markdown"
|
|
assert_header "http://localhost:8080/python/llms.txt" "Content-Type" "text/markdown"
|
|
assert_header "http://localhost:8080/python/llms-full.txt" "Content-Type" "text/markdown"
|
|
|
|
echo "✅ All Nginx header checks passed."
|
|
|
|
- name: Stop Nginx
|
|
if: always()
|
|
run: nginx -c "$(pwd)/default.conf" -s stop
|
|
|
|
lint:
|
|
name: Lint
|
|
runs-on: ubuntu-22.04
|
|
|
|
steps:
|
|
- name: Checkout repository
|
|
uses: actions/checkout@v7
|
|
|
|
- name: Use Node.js 24
|
|
uses: actions/setup-node@v6
|
|
with:
|
|
node-version: 24
|
|
package-manager-cache: false
|
|
|
|
- name: Enable corepack
|
|
run: |
|
|
corepack enable
|
|
corepack prepare yarn@stable --activate
|
|
|
|
- name: Activate cache for Node.js 24
|
|
uses: actions/setup-node@v6
|
|
with:
|
|
cache: 'yarn'
|
|
|
|
- name: Turbo cache
|
|
id: turbo-cache
|
|
uses: actions/cache@v6
|
|
with:
|
|
path: .turbo
|
|
key: turbo-${{ github.job }}-${{ github.ref_name }}-${{ github.sha }}
|
|
restore-keys: |
|
|
turbo-${{ github.job }}-${{ github.ref_name }}-
|
|
|
|
- name: Install Dependencies
|
|
run: yarn
|
|
|
|
- name: ESLint
|
|
run: yarn lint
|
|
|
|
- name: Biome format
|
|
run: yarn format:check
|
|
|
|
release_next:
|
|
name: Release @next
|
|
if: github.event_name == 'push' && contains(github.event.ref, 'master') && (!contains(github.event.head_commit.message, '[skip ci]') && !contains(github.event.head_commit.message, 'docs:'))
|
|
needs: build_and_test
|
|
runs-on: ubuntu-22.04
|
|
|
|
steps:
|
|
- name: Checkout repository
|
|
uses: actions/checkout@v7
|
|
with:
|
|
token: ${{ secrets.APIFY_SERVICE_ACCOUNT_GITHUB_TOKEN }}
|
|
fetch-depth: 0
|
|
|
|
- name: Use Node.js 24
|
|
uses: actions/setup-node@v6
|
|
with:
|
|
node-version: 24
|
|
package-manager-cache: false
|
|
|
|
- name: Enable corepack
|
|
run: |
|
|
corepack enable
|
|
corepack prepare yarn@stable --activate
|
|
|
|
- name: Activate cache for Node.js 24
|
|
uses: actions/setup-node@v6
|
|
with:
|
|
cache: 'yarn'
|
|
|
|
- name: Turbo cache
|
|
id: turbo-cache
|
|
uses: actions/cache@v6
|
|
with:
|
|
path: .turbo
|
|
key: turbo-${{ github.job }}-${{ github.ref_name }}-${{ github.sha }}
|
|
restore-keys: |
|
|
turbo-${{ github.job }}-${{ github.ref_name }}-
|
|
|
|
- name: Install Dependencies
|
|
run: yarn
|
|
|
|
- name: Build
|
|
run: yarn ci:build
|
|
|
|
- name: Generate changed packages list
|
|
id: changed-packages
|
|
run: |
|
|
echo "changed_packages=$(node ./node_modules/.bin/lerna changed -p | wc -l | xargs)" | tee -a $GITHUB_OUTPUT
|
|
|
|
- name: Report nothing to release
|
|
if: steps.changed-packages.outputs.changed_packages == '0'
|
|
run: echo "Nothing to release"
|
|
|
|
- name: Publish packages
|
|
if: steps.changed-packages.outputs.changed_packages != '0'
|
|
uses: apify/actions/execute-workflow@v1.4.0
|
|
with:
|
|
workflow: publish-to-npm.yml
|
|
inputs: >
|
|
{
|
|
"ref": "${{ steps.commit.outputs.commit_long_sha || github.sha }}",
|
|
"dist-tag": "next"
|
|
}
|
|
|
|
- name: Collect versions for Docker images
|
|
id: versions
|
|
run: |
|
|
crawlee=`node -p "require('./packages/crawlee/package.json').version"`
|
|
echo "crawlee=$crawlee" | tee -a $GITHUB_OUTPUT
|
|
|
|
- name: Trigger Docker image builds
|
|
uses: peter-evans/repository-dispatch@v4
|
|
# Trigger next images only if we have something new pushed
|
|
if: steps.changed-packages.outputs.changed_packages != '0'
|
|
with:
|
|
token: ${{ secrets.APIFY_SERVICE_ACCOUNT_GITHUB_TOKEN }}
|
|
repository: apify/apify-actor-docker
|
|
event-type: build-node-images
|
|
client-payload: >
|
|
{
|
|
"crawlee_version": "${{ steps.versions.outputs.crawlee }}",
|
|
"release_tag": "beta"
|
|
}
|