wehub-resource-sync
497 lines
17 KiB
YAML
497 lines
17 KiB
YAML
name: Build & Test
|
|
on:
|
|
pull_request:
|
|
paths-ignore:
|
|
# ignore markdown files (CHANGELOG.md, README.md, etc.)
|
|
- '**/*.md'
|
|
- 'COPYRIGHT'
|
|
- 'LICENSE'
|
|
- '.github/workflows/release.yml'
|
|
- 'changelog/**'
|
|
- 'kong.conf.default'
|
|
push:
|
|
paths-ignore:
|
|
# ignore markdown files (CHANGELOG.md, README.md, etc.)
|
|
- '**/*.md'
|
|
# ignore PRs for the generated COPYRIGHT file
|
|
- 'COPYRIGHT'
|
|
- 'LICENSE'
|
|
branches:
|
|
- master
|
|
- release/*
|
|
- test-please/*
|
|
workflow_dispatch:
|
|
inputs:
|
|
coverage:
|
|
description: 'Coverage enabled'
|
|
required: false
|
|
type: boolean
|
|
default: false
|
|
|
|
# cancel previous runs if new commits are pushed to the PR, but run for each commit on master
|
|
concurrency:
|
|
group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
|
|
cancel-in-progress: true
|
|
|
|
env:
|
|
BUILD_ROOT: ${{ github.workspace }}/bazel-bin/build
|
|
KONG_TEST_COVERAGE: ${{ inputs.coverage == true || github.event_name == 'schedule' }}
|
|
RUNNER_COUNT: 7
|
|
|
|
jobs:
|
|
metadata:
|
|
name: Metadata
|
|
runs-on: ubuntu-22.04
|
|
outputs:
|
|
old-kong-version: ${{ steps.old-kong-version.outputs.ref }}
|
|
|
|
steps:
|
|
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
|
with:
|
|
fetch-depth: 0 # `git merge-base` requires the history
|
|
|
|
- name: Get Old Kong Version
|
|
id: old-kong-version
|
|
run: |
|
|
KONG_VERSION=$(bash scripts/grep-kong-version.sh)
|
|
major=$(echo "$KONG_VERSION" | cut -d. -f1)
|
|
minor=$(echo "$KONG_VERSION" | cut -d. -f2)
|
|
# if the minor version isn't 0, use the first release or starting point of the previous minor branch;
|
|
# otherwise just leave it empty, so later the default branch or commit will be used.
|
|
if [ "$minor" -ne 0 ]; then
|
|
minor=$((minor - 1))
|
|
git fetch origin master -t
|
|
if [ $(git tag -l "$major.$minor.0") ]; then
|
|
echo "ref=$major.$minor.0" >> $GITHUB_OUTPUT
|
|
else
|
|
git fetch origin release/$major.$minor.x
|
|
COMMIT_HASH=$(git merge-base origin/master origin/release/$major.$minor.x)
|
|
echo "ref=$COMMIT_HASH" >> $GITHUB_OUTPUT
|
|
fi
|
|
else
|
|
echo "ref=" >> $GITHUB_OUTPUT
|
|
fi
|
|
|
|
build:
|
|
uses: ./.github/workflows/build.yml
|
|
with:
|
|
relative-build-root: bazel-bin/build
|
|
|
|
lint-and-doc-tests:
|
|
name: Lint and Doc tests
|
|
runs-on: ubuntu-22.04
|
|
needs: build
|
|
|
|
steps:
|
|
- name: Bump max open files
|
|
run: |
|
|
sudo echo 'kong soft nofile 65536' | sudo tee -a /etc/security/limits.d/kong-ci.conf
|
|
sudo echo 'kong hard nofile 65536' | sudo tee -a /etc/security/limits.d/kong-ci.conf
|
|
sudo echo "$(whoami) soft nofile 65536" | sudo tee -a /etc/security/limits.d/kong-ci.conf
|
|
sudo echo "$(whoami) hard nofile 65536" | sudo tee -a /etc/security/limits.d/kong-ci.conf
|
|
|
|
- name: Checkout Kong source code
|
|
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
|
|
|
- name: Lookup build cache
|
|
id: cache-deps
|
|
uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5
|
|
with:
|
|
path: ${{ env.BUILD_ROOT }}
|
|
key: ${{ needs.build.outputs.cache-key }}
|
|
|
|
- name: Check test-helpers doc generation
|
|
run: |
|
|
source ${{ env.BUILD_ROOT }}/kong-dev-venv.sh
|
|
pushd ./spec && ldoc .
|
|
|
|
- name: Check autodoc generation
|
|
run: |
|
|
source ${{ env.BUILD_ROOT }}/kong-dev-venv.sh
|
|
scripts/autodoc
|
|
|
|
- name: Lint Lua code
|
|
run: |
|
|
make lint
|
|
|
|
- name: Validate rockspec file
|
|
run: |
|
|
source ${{ env.BUILD_ROOT }}/kong-dev-venv.sh
|
|
scripts/validate-rockspec
|
|
|
|
- name: Check spec file misspelling
|
|
run: |
|
|
scripts/check_spec_files_spelling.sh
|
|
|
|
- name: Check labeler configuration
|
|
run: scripts/check-labeler.pl .github/labeler.yml
|
|
|
|
schedule:
|
|
name: Schedule busted tests to run
|
|
runs-on: ubuntu-22.04
|
|
needs: build
|
|
|
|
env:
|
|
WORKFLOW_ID: ${{ github.run_id }}
|
|
|
|
outputs:
|
|
runners: ${{ steps.generate-runner-array.outputs.RUNNERS }}
|
|
|
|
steps:
|
|
- name: Checkout source code
|
|
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
|
|
|
- name: Download runtimes file
|
|
uses: Kong/gh-storage/download@29d5de7b2b87c63016daae5680aefab30812a318 # main (node24)
|
|
env:
|
|
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
|
with:
|
|
repo-path: Kong/gateway-action-storage/main/.ci/runtimes.json
|
|
|
|
- name: Schedule tests
|
|
uses: Kong/gateway-test-scheduler/schedule@7f296716e385b5fce9573ebc5875f13c09504791 # main (node24)
|
|
with:
|
|
test-suites-file: .ci/test_suites.json
|
|
test-file-runtime-file: .ci/runtimes.json
|
|
output-prefix: test-chunk.
|
|
runner-count: ${{ env.RUNNER_COUNT }}
|
|
static-mode: ${{ github.run_attempt > 1 }}
|
|
|
|
- name: Upload schedule files
|
|
uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0
|
|
continue-on-error: true
|
|
with:
|
|
name: schedule-test-files
|
|
path: test-chunk.*
|
|
retention-days: 7
|
|
|
|
- name: Generate runner array
|
|
id: generate-runner-array
|
|
run: |
|
|
echo "RUNNERS=[$(seq -s "," 1 $(( "$RUNNER_COUNT" )))]" >> "$GITHUB_OUTPUT"
|
|
|
|
busted-tests:
|
|
name: Busted test runner ${{ matrix.runner }}
|
|
runs-on: ubuntu-22.04
|
|
needs: [metadata,build,schedule]
|
|
|
|
strategy:
|
|
fail-fast: false
|
|
matrix:
|
|
runner: ${{ fromJSON(needs.schedule.outputs.runners) }}
|
|
|
|
services:
|
|
postgres:
|
|
image: postgres:13
|
|
env:
|
|
POSTGRES_USER: kong
|
|
POSTGRES_DB: kong
|
|
POSTGRES_HOST_AUTH_METHOD: trust
|
|
ports:
|
|
- 5432:5432
|
|
options: --health-cmd pg_isready --health-interval 5s --health-timeout 5s --health-retries 8
|
|
|
|
grpcbin:
|
|
image: kong/grpcbin
|
|
ports:
|
|
- 15002:9000
|
|
- 15003:9001
|
|
|
|
redis:
|
|
image: redis
|
|
ports:
|
|
- 6379:6379
|
|
- 6380:6380
|
|
options: >-
|
|
--name kong_redis
|
|
|
|
zipkin:
|
|
image: openzipkin/zipkin:2
|
|
ports:
|
|
- 9411:9411
|
|
|
|
redis-auth:
|
|
image: redis/redis-stack-server
|
|
# Set health checks to wait until redis has started
|
|
options: >-
|
|
--health-cmd "redis-cli ping"
|
|
--health-interval 10s
|
|
--health-timeout 5s
|
|
--health-retries 5
|
|
ports:
|
|
- 6385:6379
|
|
env:
|
|
REDIS_ARGS: "--requirepass passdefault"
|
|
|
|
steps:
|
|
- name: Bump max open files
|
|
run: |
|
|
sudo echo 'kong soft nofile 65536' | sudo tee -a /etc/security/limits.d/kong-ci.conf
|
|
sudo echo 'kong hard nofile 65536' | sudo tee -a /etc/security/limits.d/kong-ci.conf
|
|
sudo echo "$(whoami) soft nofile 65536" | sudo tee -a /etc/security/limits.d/kong-ci.conf
|
|
sudo echo "$(whoami) hard nofile 65536" | sudo tee -a /etc/security/limits.d/kong-ci.conf
|
|
|
|
- name: Checkout Kong source code
|
|
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
|
|
|
# used for plugin compatibility test
|
|
- name: Checkout old version Kong source code
|
|
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
|
with:
|
|
path: kong-old
|
|
# if the minor version is 0, `ref` will default to ''
|
|
# which is same as in the previous step
|
|
ref: ${{ needs.metadata.outputs.old-kong-version }}
|
|
|
|
- name: Lookup build cache
|
|
id: cache-deps
|
|
uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5
|
|
with:
|
|
path: ${{ env.BUILD_ROOT }}
|
|
key: ${{ needs.build.outputs.cache-key }}
|
|
|
|
- name: Add gRPC test host names
|
|
run: |
|
|
echo "127.0.0.1 grpcs_1.test" | sudo tee -a /etc/hosts
|
|
echo "127.0.0.1 grpcs_2.test" | sudo tee -a /etc/hosts
|
|
|
|
- name: Enable SSL for Redis
|
|
run: |
|
|
docker cp ${{ github.workspace }} kong_redis:/workspace
|
|
docker cp ${{ github.workspace }}/spec/fixtures/redis/docker-entrypoint.sh kong_redis:/usr/local/bin/docker-entrypoint.sh
|
|
docker restart kong_redis
|
|
docker logs kong_redis
|
|
|
|
- name: Run OpenTelemetry Collector
|
|
run: |
|
|
mkdir -p ${{ github.workspace }}/tmp/otel
|
|
touch ${{ github.workspace }}/tmp/otel/file_exporter.json
|
|
sudo chmod 777 -R ${{ github.workspace }}/tmp/otel
|
|
docker run -p 4317:4317 -p 4318:4318 -p 55679:55679 \
|
|
-v ${{ github.workspace }}/spec/fixtures/opentelemetry/otelcol.yaml:/etc/otel-collector-config.yaml \
|
|
-v ${{ github.workspace }}/tmp/otel:/etc/otel \
|
|
--name opentelemetry-collector -d \
|
|
otel/opentelemetry-collector-contrib:0.52.0 \
|
|
--config=/etc/otel-collector-config.yaml
|
|
sleep 2
|
|
docker logs opentelemetry-collector
|
|
|
|
- name: Install AWS SAM cli tool
|
|
run: |
|
|
curl -L -s -o /tmp/aws-sam-cli.zip https://github.com/aws/aws-sam-cli/releases/latest/download/aws-sam-cli-linux-x86_64.zip
|
|
unzip -o /tmp/aws-sam-cli.zip -d /tmp/aws-sam-cli
|
|
sudo /tmp/aws-sam-cli/install --update
|
|
|
|
- name: Update PATH
|
|
run: |
|
|
echo "$BUILD_ROOT/kong-dev/bin" >> $GITHUB_PATH
|
|
echo "$BUILD_ROOT/kong-dev/openresty/nginx/sbin" >> $GITHUB_PATH
|
|
echo "$BUILD_ROOT/kong-dev/openresty/bin" >> $GITHUB_PATH
|
|
|
|
- name: Debug (nginx)
|
|
run: |
|
|
echo nginx: $(which nginx)
|
|
nginx -V 2>&1 | sed -re 's/ --/\n--/g'
|
|
ldd $(which nginx)
|
|
|
|
- name: Debug (luarocks)
|
|
run: |
|
|
echo luarocks: $(which luarocks)
|
|
luarocks --version
|
|
luarocks config
|
|
|
|
- name: Tune up postgres max_connections
|
|
run: |
|
|
# arm64 runners may use more connections due to more worker cores
|
|
psql -hlocalhost -Ukong kong -tAc 'alter system set max_connections = 5000;'
|
|
|
|
- name: Download test schedule file
|
|
uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0
|
|
with:
|
|
name: schedule-test-files
|
|
|
|
- name: Generate helper environment variables
|
|
run: |
|
|
echo FAILED_TEST_FILES_FILE=failed-tests.json >> $GITHUB_ENV
|
|
echo TEST_FILE_RUNTIME_FILE=test-runtime.json >> $GITHUB_ENV
|
|
echo SPEC_ERRLOG_CACHE_DIR=/tmp/${{ github.run_id }}/build_test/${{ matrix.runner }} >> $GITHUB_ENV
|
|
|
|
- name: Build & install dependencies
|
|
run: |
|
|
make dev
|
|
# python pluginserver tests dependency
|
|
pip install kong-pdk
|
|
|
|
- name: Download test rerun information
|
|
uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0
|
|
continue-on-error: true
|
|
with:
|
|
name: test-rerun-info-${{ matrix.runner }}
|
|
|
|
- name: Download test runtime statistics from previous runs
|
|
uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0
|
|
continue-on-error: true
|
|
with:
|
|
name: test-runtime-statistics-${{ matrix.runner }}
|
|
|
|
- name: Run Tests
|
|
env:
|
|
KONG_TEST_PG_DATABASE: kong
|
|
KONG_TEST_PG_USER: kong
|
|
KONG_TEST_DATABASE: postgres
|
|
KONG_SPEC_TEST_GRPCBIN_PORT: "15002"
|
|
KONG_SPEC_TEST_GRPCBIN_SSL_PORT: "15003"
|
|
KONG_SPEC_TEST_OTELCOL_FILE_EXPORTER_PATH: ${{ github.workspace }}/tmp/otel/file_exporter.json
|
|
KONG_SPEC_TEST_OLD_VERSION_KONG_PATH: ${{ github.workspace }}/kong-old
|
|
DD_ENV: ci
|
|
DD_SERVICE: kong-ce-ci
|
|
DD_CIVISIBILITY_MANUAL_API_ENABLED: 1
|
|
DD_CIVISIBILITY_AGENTLESS_ENABLED: true
|
|
DD_TRACE_GIT_METADATA_ENABLED: true
|
|
DD_API_KEY: ${{ secrets.DATADOG_API_KEY }}
|
|
SPEC_ERRLOG_CACHE_DIR: ${{ env.SPEC_ERRLOG_CACHE_DIR }}
|
|
uses: Kong/gateway-test-scheduler/runner@7f296716e385b5fce9573ebc5875f13c09504791 # main (node24)
|
|
with:
|
|
tests-to-run-file: test-chunk.${{ matrix.runner }}.json
|
|
failed-test-files-file: ${{ env.FAILED_TEST_FILES_FILE }}
|
|
test-file-runtime-file: ${{ env.TEST_FILE_RUNTIME_FILE }}
|
|
build-root: ${{ env.BUILD_ROOT }}
|
|
|
|
- name: Upload error logs
|
|
if: failure()
|
|
uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0
|
|
with:
|
|
name: busted-test-errlogs-${{ matrix.runner }}
|
|
path: ${{ env.SPEC_ERRLOG_CACHE_DIR }}
|
|
retention-days: 1
|
|
|
|
- name: Upload test rerun information
|
|
if: always()
|
|
uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0
|
|
with:
|
|
name: test-rerun-info-${{ matrix.runner }}
|
|
path: ${{ env.FAILED_TEST_FILES_FILE }}
|
|
retention-days: 2
|
|
|
|
- name: Upload test runtime statistics for offline scheduling
|
|
if: always()
|
|
uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0
|
|
with:
|
|
name: test-runtime-statistics-${{ matrix.runner }}
|
|
path: ${{ env.TEST_FILE_RUNTIME_FILE }}
|
|
retention-days: 7
|
|
|
|
- name: Archive coverage stats file
|
|
uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0
|
|
if: ${{ always() && (inputs.coverage == true || github.event_name == 'schedule') }}
|
|
with:
|
|
name: luacov-stats-out-${{ github.job }}-${{ github.run_id }}-${{ matrix.runner }}
|
|
retention-days: 1
|
|
path: |
|
|
luacov.stats.out
|
|
|
|
- name: Get kernel message
|
|
if: failure()
|
|
run: |
|
|
sudo dmesg -T
|
|
|
|
pdk-tests:
|
|
name: PDK tests
|
|
runs-on: ubuntu-22.04
|
|
needs: build
|
|
|
|
steps:
|
|
- name: Bump max open files
|
|
run: |
|
|
sudo echo 'kong soft nofile 65536' | sudo tee -a /etc/security/limits.d/kong-ci.conf
|
|
sudo echo 'kong hard nofile 65536' | sudo tee -a /etc/security/limits.d/kong-ci.conf
|
|
sudo echo "$(whoami) soft nofile 65536" | sudo tee -a /etc/security/limits.d/kong-ci.conf
|
|
sudo echo "$(whoami) hard nofile 65536" | sudo tee -a /etc/security/limits.d/kong-ci.conf
|
|
|
|
- name: Checkout Kong source code
|
|
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
|
|
|
- name: Lookup build cache
|
|
id: cache-deps
|
|
uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5
|
|
with:
|
|
path: ${{ env.BUILD_ROOT }}
|
|
key: ${{ needs.build.outputs.cache-key }}
|
|
|
|
- name: Install Test::Nginx
|
|
run: |
|
|
CPAN_DOWNLOAD=./cpanm
|
|
mkdir -p $CPAN_DOWNLOAD
|
|
curl -o $CPAN_DOWNLOAD/cpanm https://cpanmin.us
|
|
chmod +x $CPAN_DOWNLOAD/cpanm
|
|
|
|
echo "Installing CPAN dependencies..."
|
|
$CPAN_DOWNLOAD/cpanm --notest --local-lib=$HOME/perl5 local::lib && eval $(perl -I $HOME/perl5/lib/perl5/ -Mlocal::lib)
|
|
$CPAN_DOWNLOAD/cpanm --notest Test::Nginx
|
|
|
|
- name: Generate environment variables
|
|
run: |
|
|
echo SPEC_ERRLOG_CACHE_DIR=/tmp/${{ github.run_id }}/PDK_test >> $GITHUB_ENV
|
|
|
|
- name: Tests
|
|
env:
|
|
TEST_SUITE: pdk
|
|
run: |
|
|
source ${{ env.BUILD_ROOT }}/kong-dev-venv.sh
|
|
if [[ $KONG_TEST_COVERAGE = true ]]; then
|
|
export PDK_LUACOV=1
|
|
fi
|
|
eval $(perl -I $HOME/perl5/lib/perl5/ -Mlocal::lib)
|
|
prove -I. -r t
|
|
|
|
- name: Upload error logs
|
|
if: failure()
|
|
uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0
|
|
with:
|
|
name: PDK-test-errlogs
|
|
path: ${{ env.SPEC_ERRLOG_CACHE_DIR }}
|
|
retention-days: 1
|
|
|
|
- name: Archive coverage stats file
|
|
uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0
|
|
if: ${{ always() && (inputs.coverage == true || github.event_name == 'schedule') }}
|
|
with:
|
|
name: luacov-stats-out-${{ github.job }}-${{ github.run_id }}
|
|
retention-days: 1
|
|
path: |
|
|
luacov.stats.out
|
|
|
|
- name: Get kernel message
|
|
if: failure()
|
|
run: |
|
|
sudo dmesg -T
|
|
|
|
cleanup-and-aggregate-stats:
|
|
needs: [lint-and-doc-tests,pdk-tests,busted-tests]
|
|
name: Cleanup and Luacov stats aggregator
|
|
if: ${{ always() && (inputs.coverage == true || github.event_name == 'schedule') }}
|
|
runs-on: ubuntu-22.04
|
|
|
|
steps:
|
|
- name: Checkout source code
|
|
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
|
|
|
- name: Install requirements
|
|
run: |
|
|
sudo apt-get update && sudo apt-get install -y luarocks
|
|
sudo luarocks install luacov
|
|
sudo luarocks install luafilesystem
|
|
|
|
# Download all archived coverage stats files
|
|
- uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0
|
|
|
|
- name: Stats aggregation
|
|
shell: bash
|
|
run: |
|
|
lua .ci/luacov-stats-aggregator.lua "luacov-stats-out-" "luacov.stats.out" ${{ github.workspace }}/
|
|
# The following prints a report with each file sorted by coverage percentage, and the total coverage
|
|
printf "\n\nCoverage File\n\n"
|
|
awk -v RS='Coverage\n-+\n' 'NR>1{print $0}' luacov.report.out | grep -vE "^-|^$" > summary.out
|
|
cat summary.out | grep -v "^Total" | awk '{printf "%7d%% %s\n", $4, $1}' | sort -n
|
|
cat summary.out | grep "^Total" | awk '{printf "%7d%% %s\n", $4, $1}'
|