chore: import upstream snapshot with attribution
Integration Tests - MySQL + Elasticsearch / Detect Changes (push) Has been cancelled
Integration Tests - MySQL + Elasticsearch / integration-tests-mysql-elasticsearch (push) Has been cancelled
Integration Tests - PostgreSQL + Elasticsearch + Redis / Detect Changes (push) Has been cancelled
Integration Tests - PostgreSQL + Elasticsearch + Redis / integration-tests-postgres-elasticsearch-redis (push) Has been cancelled
Integration Tests - PostgreSQL + OpenSearch / Detect Changes (push) Has been cancelled
Integration Tests - PostgreSQL + OpenSearch / integration-tests-postgres-opensearch (push) Has been cancelled
Java Checkstyle / java-checkstyle (push) Has been cancelled
Maven Collate Tests / maven-collate-ci (push) Has been cancelled
OpenMetadata Service Unit Tests / openmetadata-service-unit-tests-status (push) Has been cancelled
Publish Package to Maven Central Repository / publish-maven-packages (push) Has been cancelled
OpenMetadata Service Unit Tests / Detect Changes (push) Has been cancelled
OpenMetadata Service Unit Tests / openmetadata-service-unit-tests (push) Has been cancelled
OpenMetadata Service Unit Tests / k8s_operator-unit-tests (push) Has been cancelled
Integration Tests - MySQL + Elasticsearch / Detect Changes (push) Has been cancelled
Integration Tests - MySQL + Elasticsearch / integration-tests-mysql-elasticsearch (push) Has been cancelled
Integration Tests - PostgreSQL + Elasticsearch + Redis / Detect Changes (push) Has been cancelled
Integration Tests - PostgreSQL + Elasticsearch + Redis / integration-tests-postgres-elasticsearch-redis (push) Has been cancelled
Integration Tests - PostgreSQL + OpenSearch / Detect Changes (push) Has been cancelled
Integration Tests - PostgreSQL + OpenSearch / integration-tests-postgres-opensearch (push) Has been cancelled
Java Checkstyle / java-checkstyle (push) Has been cancelled
Maven Collate Tests / maven-collate-ci (push) Has been cancelled
OpenMetadata Service Unit Tests / openmetadata-service-unit-tests-status (push) Has been cancelled
Publish Package to Maven Central Repository / publish-maven-packages (push) Has been cancelled
OpenMetadata Service Unit Tests / Detect Changes (push) Has been cancelled
OpenMetadata Service Unit Tests / openmetadata-service-unit-tests (push) Has been cancelled
OpenMetadata Service Unit Tests / k8s_operator-unit-tests (push) Has been cancelled
This commit is contained in:
wehub-resource-sync
@@ -0,0 +1,162 @@
|
||||
|
||||
# Copyright 2021 Collate
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
name: airflow-apis-tests
|
||||
on:
|
||||
pull_request_target:
|
||||
types: [labeled, opened, synchronize, reopened, ready_for_review]
|
||||
paths:
|
||||
- 'openmetadata-airflow-apis/**'
|
||||
workflow_dispatch:
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
concurrency:
|
||||
group: airflow-apis-tests-${{ github.event.pull_request.number || github.run_id }}
|
||||
cancel-in-progress: ${{ github.event_name != 'pull_request_target' || github.event.action != 'labeled' || github.event.label.name == 'safe to test' }}
|
||||
|
||||
env:
|
||||
SONAR_OPTS: >-
|
||||
-Dproject.settings=openmetadata-airflow-apis/sonar-project.properties
|
||||
-Dsonar.pullrequest.key=${{ github.event.pull_request.number }}
|
||||
-Dsonar.pullrequest.branch=${{ github.event.pull_request.head.ref }}
|
||||
-Dsonar.pullrequest.github.repository=OpenMetadata
|
||||
-Dsonar.scm.revision=${{ github.event.pull_request.head.sha }}
|
||||
-Dsonar.pullrequest.provider=github
|
||||
|
||||
jobs:
|
||||
airflow-apis-tests:
|
||||
runs-on: ubuntu-latest
|
||||
if: ${{ !github.event.pull_request.draft && (github.event_name != 'pull_request_target' || github.event.action != 'labeled' || github.event.label.name == 'safe to test') }}
|
||||
steps:
|
||||
- name: Free Disk Space (Ubuntu)
|
||||
uses: jlumbroso/free-disk-space@main
|
||||
with:
|
||||
tool-cache: false
|
||||
android: true
|
||||
dotnet: true
|
||||
haskell: true
|
||||
large-packages: false
|
||||
docker-images: false
|
||||
swap-storage: true
|
||||
- name: Wait for the labeler
|
||||
uses: lewagon/wait-on-check-action@v1.7.0
|
||||
if: ${{ github.event_name == 'pull_request_target' }}
|
||||
with:
|
||||
ref: ${{ github.event.pull_request.head.sha }}
|
||||
check-name: Team Label
|
||||
repo-token: ${{ secrets.GITHUB_TOKEN }}
|
||||
wait-interval: 90
|
||||
|
||||
- name: Verify PR labels
|
||||
uses: jesusvasquez333/verify-pr-label-action@v1.4.0
|
||||
if: ${{ github.event_name == 'pull_request_target' }}
|
||||
with:
|
||||
github-token: '${{ secrets.GITHUB_TOKEN }}'
|
||||
valid-labels: 'safe to test'
|
||||
pull-request-number: '${{ github.event.pull_request.number }}'
|
||||
disable-reviews: true # To not auto approve changes
|
||||
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
ref: ${{ github.event.pull_request.head.sha }}
|
||||
fetch-depth: 0
|
||||
filter: blob:none
|
||||
|
||||
- name: Set up JDK 21
|
||||
uses: actions/setup-java@v4
|
||||
with:
|
||||
java-version: '21'
|
||||
distribution: 'temurin'
|
||||
|
||||
- name: Set up Python 3.10
|
||||
uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: '3.10'
|
||||
|
||||
- name: Install Ubuntu dependencies
|
||||
run: |
|
||||
# stop relying on apt cache of GitHub runners
|
||||
sudo apt-get update
|
||||
sudo apt-get install -y unixodbc-dev python3-venv librdkafka-dev gcc libsasl2-dev build-essential libssl-dev libffi-dev \
|
||||
librdkafka-dev unixodbc-dev libevent-dev libkrb5-dev
|
||||
|
||||
- name: Generate models
|
||||
run: |
|
||||
python3 -m venv env
|
||||
source env/bin/activate
|
||||
sudo make install_antlr_cli
|
||||
make install_dev generate
|
||||
|
||||
- name: Install open-metadata dependencies
|
||||
run: |
|
||||
source env/bin/activate
|
||||
make install_all install_test
|
||||
|
||||
- name: Start Server and Ingest Sample Data
|
||||
uses: nick-fields/retry@v3.0.2
|
||||
env:
|
||||
INGESTION_DEPENDENCY: "mysql,elasticsearch,sample-data"
|
||||
with:
|
||||
timeout_minutes: 60
|
||||
max_attempts: 2
|
||||
retry_on: error
|
||||
command: ./docker/run_local_docker.sh -m no-ui
|
||||
|
||||
- name: Run Python Tests & Record Coverage
|
||||
run: |
|
||||
source env/bin/activate
|
||||
make install_apis
|
||||
make coverage_apis
|
||||
rm pom.xml
|
||||
# fix coverage xml report for github
|
||||
sed -i 's/openmetadata_managed_apis/\/github\/workspace\/openmetadata-airflow-apis\/openmetadata_managed_apis/g' openmetadata-airflow-apis/ci-coverage.xml
|
||||
|
||||
- name: Push Results in PR to Sonar
|
||||
id: push-to-sonar
|
||||
if: ${{ github.event_name == 'pull_request_target' }}
|
||||
continue-on-error: true
|
||||
uses: SonarSource/sonarqube-scan-action@v7
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
SONAR_TOKEN: ${{ secrets.AIRFLOW_APIS_SONAR_TOKEN }}
|
||||
with:
|
||||
projectBaseDir: openmetadata-airflow-apis/
|
||||
args: ${{ env.SONAR_OPTS }}
|
||||
|
||||
# next two steps are for retrying "Push Results in PR to Sonar" step in case it fails
|
||||
- name: Wait to retry 'Push Results in PR to Sonar'
|
||||
if: ${{ github.event_name == 'pull_request_target' && steps.push-to-sonar.outcome != 'success' }}
|
||||
run: sleep 20s
|
||||
shell: bash
|
||||
|
||||
- name: Retry 'Push Results in PR to Sonar'
|
||||
uses: SonarSource/sonarqube-scan-action@v7
|
||||
if: ${{ github.event_name == 'pull_request_target' && steps.push-to-sonar.outcome != 'success' }}
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
SONAR_TOKEN: ${{ secrets.AIRFLOW_APIS_SONAR_TOKEN }}
|
||||
with:
|
||||
projectBaseDir: openmetadata-airflow-apis/
|
||||
args: ${{ env.SONAR_OPTS }}
|
||||
|
||||
- name: Push Results to Sonar
|
||||
uses: SonarSource/sonarqube-scan-action@v7
|
||||
if: ${{ github.event_name == 'push' }}
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
SONAR_TOKEN: ${{ secrets.AIRFLOW_APIS_SONAR_TOKEN }}
|
||||
with:
|
||||
projectBaseDir: openmetadata-airflow-apis/
|
||||
args: -Dproject.settings=openmetadata-airflow-apis/sonar-project.properties
|
||||
@@ -0,0 +1,89 @@
|
||||
---
|
||||
name: Cherry-pick labeled PRs to OpenMetadata release branch on merge
|
||||
# yamllint disable-line rule:comments
|
||||
run-name: OpenMetadata release cherry-pick PR #${{ github.event.pull_request.number }}
|
||||
|
||||
# yamllint disable-line rule:truthy
|
||||
on:
|
||||
pull_request_target:
|
||||
types: [closed]
|
||||
branches:
|
||||
- main
|
||||
permissions:
|
||||
contents: write
|
||||
pull-requests: write
|
||||
issues: write
|
||||
env:
|
||||
CURRENT_RELEASE_ENDPOINT: ${{ vars.CURRENT_RELEASE_ENDPOINT }} # Endpoint that returns the current release version in json format
|
||||
jobs:
|
||||
get_release_branch:
|
||||
if: github.event.pull_request.merged == true &&
|
||||
contains(github.event.pull_request.labels.*.name, 'To release')
|
||||
runs-on: ubuntu-latest
|
||||
outputs:
|
||||
release_branches: ${{ steps.get_release_version.outputs.release_branches }}
|
||||
steps:
|
||||
- name: Get the release version
|
||||
id: get_release_version
|
||||
run: |
|
||||
CURRENT_RELEASE=$(curl -s $CURRENT_RELEASE_ENDPOINT | jq -c '.collate_branches // []')
|
||||
echo "release_branches=${CURRENT_RELEASE}" >> $GITHUB_OUTPUT
|
||||
|
||||
cherry_pick_to_release_branch:
|
||||
needs: get_release_branch
|
||||
if: needs.get_release_branch.outputs.release_branches != '' && needs.get_release_branch.outputs.release_branches != '[]'
|
||||
runs-on: ubuntu-latest # Running it on ubuntu-latest on purpose (we're not using all the free minutes)
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
branch: ${{ fromJson(needs.get_release_branch.outputs.release_branches) }}
|
||||
steps:
|
||||
- name: Checkout main branch
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
ref: main
|
||||
fetch-depth: 0
|
||||
- name: Cherry-pick changes from PR
|
||||
id: cherry_pick
|
||||
continue-on-error: true
|
||||
run: |
|
||||
git config --global user.email "release-bot@open-metadata.org"
|
||||
git config --global user.name "OpenMetadata Release Bot"
|
||||
git fetch origin ${{ matrix.branch }}
|
||||
git checkout ${{ matrix.branch }}
|
||||
git cherry-pick -x ${{ github.event.pull_request.merge_commit_sha }}
|
||||
- name: Push changes to release branch
|
||||
id: push_changes
|
||||
continue-on-error: true
|
||||
if: steps.cherry_pick.outcome == 'success'
|
||||
run: |
|
||||
git push origin ${{ matrix.branch }}
|
||||
- name: Post a comment on failure
|
||||
if: steps.cherry_pick.outcome != 'success' || steps.push_changes.outcome != 'success'
|
||||
uses: actions/github-script@v7
|
||||
with:
|
||||
script: |
|
||||
const prNumber = context.payload.pull_request.number;
|
||||
const releaseBranch = '${{ matrix.branch }}';
|
||||
const workflowRunUrl = `${process.env.GITHUB_SERVER_URL}/${process.env.GITHUB_REPOSITORY}/actions/runs/${process.env.GITHUB_RUN_ID}`;
|
||||
github.rest.issues.createComment({
|
||||
owner: context.repo.owner,
|
||||
repo: context.repo.repo,
|
||||
issue_number: prNumber,
|
||||
body: `Failed to cherry-pick changes to the ${releaseBranch} branch.
|
||||
Please cherry-pick the changes manually.
|
||||
You can find more details [here](${workflowRunUrl}).`
|
||||
})
|
||||
- name: Post a comment on success
|
||||
if: steps.cherry_pick.outcome == 'success' && steps.push_changes.outcome == 'success'
|
||||
uses: actions/github-script@v7
|
||||
with:
|
||||
script: |
|
||||
const prNumber = context.payload.pull_request.number;
|
||||
const releaseBranch = '${{ matrix.branch }}';
|
||||
github.rest.issues.createComment({
|
||||
owner: context.repo.owner,
|
||||
repo: context.repo.repo,
|
||||
issue_number: prNumber,
|
||||
body: `Changes have been cherry-picked to the ${releaseBranch} branch.`
|
||||
})
|
||||
@@ -0,0 +1,64 @@
|
||||
name: Claude Code
|
||||
|
||||
on:
|
||||
issue_comment:
|
||||
types: [created]
|
||||
pull_request_review_comment:
|
||||
types: [created]
|
||||
issues:
|
||||
types: [opened, assigned]
|
||||
pull_request_review:
|
||||
types: [submitted]
|
||||
|
||||
jobs:
|
||||
claude:
|
||||
if: |
|
||||
(github.event_name == 'issue_comment' && contains(github.event.comment.body, '@claude')) ||
|
||||
(github.event_name == 'pull_request_review_comment' && contains(github.event.comment.body, '@claude')) ||
|
||||
(github.event_name == 'pull_request_review' && contains(github.event.review.body, '@claude')) ||
|
||||
(github.event_name == 'issues' && (contains(github.event.issue.body, '@claude') || contains(github.event.issue.title, '@claude')))
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
contents: read
|
||||
pull-requests: read
|
||||
issues: read
|
||||
id-token: write
|
||||
actions: read # Required for Claude to read CI results on PRs
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
fetch-depth: 1
|
||||
|
||||
- name: Run Claude Code
|
||||
id: claude
|
||||
uses: anthropics/claude-code-action@beta
|
||||
with:
|
||||
claude_code_oauth_token: ${{ secrets.CLAUDE_CODE_OAUTH_TOKEN }}
|
||||
|
||||
# This is an optional setting that allows Claude to read CI results on PRs
|
||||
additional_permissions: |
|
||||
actions: read
|
||||
|
||||
# Optional: Specify model (defaults to Claude Sonnet 4, uncomment for Claude Opus 4)
|
||||
# model: "claude-opus-4-20250514"
|
||||
|
||||
# Optional: Customize the trigger phrase (default: @claude)
|
||||
# trigger_phrase: "/claude"
|
||||
|
||||
# Optional: Trigger when specific user is assigned to an issue
|
||||
# assignee_trigger: "claude-bot"
|
||||
|
||||
# Optional: Allow Claude to run specific commands
|
||||
# allowed_tools: "Bash(npm install),Bash(npm run build),Bash(npm run test:*),Bash(npm run lint:*)"
|
||||
|
||||
# Optional: Add custom instructions for Claude to customize its behavior for your project
|
||||
# custom_instructions: |
|
||||
# Follow our coding standards
|
||||
# Ensure all new code has tests
|
||||
# Use TypeScript for new files
|
||||
|
||||
# Optional: Custom environment variables for Claude
|
||||
# claude_env: |
|
||||
# NODE_ENV: test
|
||||
|
||||
@@ -0,0 +1,23 @@
|
||||
name: CodeQL Advanced
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
branch:
|
||||
description: "Branch to run the workflow on"
|
||||
required: true
|
||||
default: "main"
|
||||
type: string
|
||||
jobs:
|
||||
analyze:
|
||||
runs-on: ubuntu-latest
|
||||
if: ${{ !github.event.pull_request.draft }}
|
||||
permissions:
|
||||
contents: write
|
||||
pull-requests: write
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
ref: ${{ github.event.inputs.branch || github.event.pull_request.head.ref || github.ref }}
|
||||
token: ${{ secrets.GITHUB_TOKEN }}
|
||||
@@ -0,0 +1,81 @@
|
||||
# Copyright 2026 Collate
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
name: Workflows and Data Access Request E2E tests
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
pull_request_target:
|
||||
types: [opened, synchronize, reopened, labeled, ready_for_review]
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
pull-requests: read
|
||||
checks: write
|
||||
|
||||
concurrency:
|
||||
group: data-access-request-e2e-${{ github.event.pull_request.number || github.run_id }}
|
||||
cancel-in-progress: ${{ github.event_name != 'pull_request_target' || github.event.action != 'labeled' || github.event.label.name == 'safe to test' }}
|
||||
|
||||
jobs:
|
||||
check-changes:
|
||||
runs-on: ubuntu-latest
|
||||
outputs:
|
||||
dar: ${{ steps.filter.outputs.dar }}
|
||||
steps:
|
||||
- uses: dorny/paths-filter@v4
|
||||
id: filter
|
||||
with:
|
||||
filters: |
|
||||
dar:
|
||||
- 'openmetadata-service/src/main/java/org/openmetadata/service/resources/tasks/TaskResource.java'
|
||||
- 'openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/TaskRepository.java'
|
||||
- 'openmetadata-service/src/main/java/org/openmetadata/service/resources/feeds/FeedResource.java'
|
||||
- 'openmetadata-service/src/main/java/org/openmetadata/service/governance/workflows/**'
|
||||
|
||||
data-access-request-e2e:
|
||||
needs: check-changes
|
||||
runs-on: ubuntu-latest
|
||||
if: |
|
||||
!github.event.pull_request.draft &&
|
||||
(github.event_name == 'workflow_dispatch' || needs.check-changes.outputs.dar == 'true') &&
|
||||
(github.event_name != 'pull_request_target' || github.event.action != 'labeled' || github.event.label.name == 'safe to test')
|
||||
steps:
|
||||
- name: Wait for the labeler
|
||||
uses: lewagon/wait-on-check-action@v1.7.0
|
||||
if: ${{ github.event_name == 'pull_request_target' }}
|
||||
with:
|
||||
ref: ${{ github.event.pull_request.head.sha }}
|
||||
check-name: Team Label
|
||||
repo-token: ${{ secrets.GITHUB_TOKEN }}
|
||||
wait-interval: 90
|
||||
|
||||
- name: Verify PR labels
|
||||
uses: jesusvasquez333/verify-pr-label-action@v1.4.0
|
||||
if: ${{ github.event_name == 'pull_request_target' }}
|
||||
with:
|
||||
github-token: "${{ secrets.GITHUB_TOKEN }}"
|
||||
valid-labels: "safe to test"
|
||||
pull-request-number: "${{ github.event.pull_request.number }}"
|
||||
disable-reviews: true # To not auto approve changes
|
||||
|
||||
- name: Trigger Collate Playwright run & wait
|
||||
uses: the-actions-org/workflow-dispatch@v4
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
SHA: ${{ github.event.pull_request.head.sha || github.sha }}
|
||||
with:
|
||||
workflow: Workflows and Data Access Request E2E tests (from OpenMetadata PR)
|
||||
ref: main
|
||||
repo: open-metadata/openmetadata-collate
|
||||
token: ${{ secrets.COLLATE_PAT }}
|
||||
wait-for-completion: true
|
||||
inputs: '{ "sha": "${{ env.SHA }}", "event": "${{ github.event_name }}" }'
|
||||
@@ -0,0 +1,147 @@
|
||||
# Copyright 2021 Collate
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
name: docker-k8s-operator release app
|
||||
on:
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
docker_release_tag:
|
||||
description: "K8s Operator Docker Image Tag"
|
||||
required: true
|
||||
push_latest_tag_to_release:
|
||||
description: "Do you want to update docker image latest tag as well ?"
|
||||
type: boolean
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
|
||||
cancel-in-progress: true
|
||||
|
||||
jobs:
|
||||
maven-build:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
ref: ${{ github.event.pull_request.head.sha || github.ref }}
|
||||
|
||||
- name: Set up JDK 21
|
||||
uses: actions/setup-java@v4
|
||||
with:
|
||||
java-version: 21
|
||||
distribution: 'temurin'
|
||||
|
||||
- name: Build K8s Operator Application
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
run: |
|
||||
mvn -DskipTests clean package -pl openmetadata-k8s-operator -am
|
||||
|
||||
- name: Run K8s Operator Tests
|
||||
run: |
|
||||
mvn test -pl openmetadata-k8s-operator
|
||||
|
||||
- name: Upload K8s Operator JAR to Artifact
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: k8s-operator-binary
|
||||
path: openmetadata-k8s-operator/target/*.jar
|
||||
|
||||
release-project-event-workflow_dispatch:
|
||||
if: github.event_name == 'workflow_dispatch'
|
||||
name: Release k8s operator with workflow_dispatch event
|
||||
runs-on: ubuntu-latest
|
||||
needs: maven-build
|
||||
steps:
|
||||
- name: Fetch Release Data
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
id: attach_release
|
||||
run: |
|
||||
echo "UPLOAD_URL=$(curl -H "Accept: application/vnd.github+json" -H "Authorization: Bearer $GITHUB_TOKEN" -H "X-GitHub-Api-Version: 2022-11-28" https://api.github.com/repos/open-metadata/OpenMetadata/releases/tags/${{ inputs.docker_release_tag }}-release | jq .upload_url | tr -d '"' )" >> $GITHUB_OUTPUT
|
||||
|
||||
- name: Download application from Artifact
|
||||
uses: actions/download-artifact@v4
|
||||
with:
|
||||
name: k8s-operator-binary
|
||||
|
||||
- name: Upload k8s operator to release
|
||||
uses: actions/upload-release-asset@v1
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
with:
|
||||
asset_path: ./openmetadata-k8s-operator-${{ inputs.docker_release_tag }}-boot.jar
|
||||
upload_url: ${{ steps.attach_release.outputs.UPLOAD_URL }}
|
||||
asset_name: openmetadata-k8s-operator-${{ inputs.docker_release_tag }}.jar
|
||||
asset_content_type: application/java-archive
|
||||
|
||||
push_to_docker_hub:
|
||||
runs-on: ubuntu-latest
|
||||
if: ${{ always() && contains(join(needs.*.result, ','), 'success') }}
|
||||
needs: [ maven-build ]
|
||||
steps:
|
||||
- name: Wait for the labeler
|
||||
uses: lewagon/wait-on-check-action@v1.7.0
|
||||
if: ${{ github.event_name == 'pull_request_target' }}
|
||||
with:
|
||||
ref: ${{ github.event.pull_request.head.sha }}
|
||||
check-name: Team Label
|
||||
repo-token: ${{ secrets.GITHUB_TOKEN }}
|
||||
wait-interval: 90
|
||||
|
||||
- name: Verify PR labels
|
||||
uses: jesusvasquez333/verify-pr-label-action@v1.4.0
|
||||
if: ${{ github.event_name == 'pull_request_target' }}
|
||||
with:
|
||||
github-token: "${{ secrets.GITHUB_TOKEN }}"
|
||||
valid-labels: "safe to test"
|
||||
pull-request-number: "${{ github.event.pull_request.number }}"
|
||||
disable-reviews: true
|
||||
|
||||
- name: Check out the Repo
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
ref: ${{ github.event.pull_request.head.sha || github.ref }}
|
||||
|
||||
- name: Download application from Artifact
|
||||
uses: actions/download-artifact@v4
|
||||
with:
|
||||
name: k8s-operator-binary
|
||||
path: openmetadata-k8s-operator/target/
|
||||
|
||||
- name: Set build arguments
|
||||
id: build-args
|
||||
run: |
|
||||
echo "BUILD_DATE=$(date -u +'%Y-%m-%dT%H:%M:%SZ')" >> $GITHUB_OUTPUT
|
||||
|
||||
- name: Prepare for Docker Build&Push
|
||||
id: prepare
|
||||
uses: ./.github/actions/prepare-for-docker-build-and-push
|
||||
with:
|
||||
image: openmetadata/omjob-operator
|
||||
tag: ${{ inputs.docker_release_tag || 'pr-test' }}
|
||||
push_latest: ${{ inputs.push_latest_tag_to_release || false }}
|
||||
dockerhub_username: ${{ secrets.DOCKERHUB_OPENMETADATA_USERNAME }}
|
||||
dockerhub_token: ${{ secrets.DOCKERHUB_OPENMETADATA_TOKEN }}
|
||||
|
||||
- name: Build and push Docker image
|
||||
uses: docker/build-push-action@v6
|
||||
env:
|
||||
DOCKER_BUILD_NO_SUMMARY: true
|
||||
with:
|
||||
context: ./openmetadata-k8s-operator
|
||||
platforms: linux/amd64,linux/arm64
|
||||
push: ${{ github.event_name == 'workflow_dispatch' }}
|
||||
tags: ${{ steps.prepare.outputs.tags }}
|
||||
file: ./openmetadata-k8s-operator/Dockerfile
|
||||
build-args: |
|
||||
BUILD_DATE=${{ steps.build-args.outputs.BUILD_DATE }}
|
||||
COMMIT_ID=${{ github.sha }}
|
||||
@@ -0,0 +1,51 @@
|
||||
# Copyright 2021 Collate
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
name: docker-openmetadata-db docker
|
||||
on:
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
docker_release_tag:
|
||||
description: "OpenMetadata MySQL Docker Image Tag"
|
||||
required: true
|
||||
push_latest_tag_to_release:
|
||||
description: "Do you want to update docker image latest tag as well ?"
|
||||
type: boolean
|
||||
|
||||
jobs:
|
||||
push_to_docker_hub:
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
steps:
|
||||
- name: Check out the Repo
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Prepare for Docker Build&Push
|
||||
id: prepare
|
||||
uses: ./.github/actions/prepare-for-docker-build-and-push
|
||||
with:
|
||||
image: openmetadata/db
|
||||
tag: ${{ inputs.docker_release_tag }}
|
||||
push_latest: ${{ inputs.push_latest_tag_to_release }}
|
||||
dockerhub_username: ${{ secrets.DOCKERHUB_OPENMETADATA_USERNAME }}
|
||||
dockerhub_token: ${{ secrets.DOCKERHUB_OPENMETADATA_TOKEN }}
|
||||
|
||||
|
||||
- name: Build and push if event is workflow_dispatch and input is checked
|
||||
uses: docker/build-push-action@v6
|
||||
env:
|
||||
DOCKER_BUILD_NO_SUMMARY: true
|
||||
with:
|
||||
context: .
|
||||
platforms: linux/amd64,linux/arm64
|
||||
push: true
|
||||
tags: ${{ steps.prepare.outputs.tags }}
|
||||
file: ./docker/mysql/Dockerfile_mysql
|
||||
@@ -0,0 +1,57 @@
|
||||
# Copyright 2021 Collate
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
name: docker-openmetadata-ingestion-base-slim docker
|
||||
on:
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
docker_release_tag:
|
||||
description: "Ingestion Base Slim Docker Image Tag (3 digit docker image tag)"
|
||||
required: true
|
||||
pypi_release_version:
|
||||
description: "Provide the Release Version (4 digit docker image tag)"
|
||||
required: true
|
||||
push_latest_tag_to_release:
|
||||
description: "Do you want to update docker image latest tag as well ?"
|
||||
type: boolean
|
||||
|
||||
jobs:
|
||||
push_to_docker_hub:
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
steps:
|
||||
- name: Check out the Repo
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Prepare for Docker Build&Push
|
||||
id: prepare
|
||||
uses: ./.github/actions/prepare-for-docker-build-and-push
|
||||
with:
|
||||
image: openmetadata/ingestion-base-slim
|
||||
tag: ${{ inputs.docker_release_tag }}
|
||||
push_latest: ${{ inputs.push_latest_tag_to_release }}
|
||||
is_ingestion: true
|
||||
release_version: ${{ inputs.pypi_release_version }}
|
||||
dockerhub_username: ${{ secrets.DOCKERHUB_OPENMETADATA_USERNAME }}
|
||||
dockerhub_token: ${{ secrets.DOCKERHUB_OPENMETADATA_TOKEN }}
|
||||
|
||||
- name: Build and push if event is workflow_dispatch and input is checked
|
||||
uses: docker/build-push-action@v6
|
||||
env:
|
||||
DOCKER_BUILD_NO_SUMMARY: true
|
||||
with:
|
||||
context: .
|
||||
platforms: linux/amd64,linux/arm64
|
||||
push: true
|
||||
tags: ${{ steps.prepare.outputs.tags }}
|
||||
file: ./ingestion/operators/docker/Dockerfile
|
||||
build-args: |
|
||||
INGESTION_DEPENDENCY=slim
|
||||
@@ -0,0 +1,66 @@
|
||||
# Copyright 2021 Collate
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
name: docker-openmetadata-ingestion-base docker
|
||||
on:
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
docker_release_tag:
|
||||
description: "Ingestion Base Docker Image Tag (3 digit docker image tag)"
|
||||
required: true
|
||||
pypi_release_version:
|
||||
description: "Provide the Release Version (4 digit docker image tag)"
|
||||
required: true
|
||||
push_latest_tag_to_release:
|
||||
description: "Do you want to update docker image latest tag as well ?"
|
||||
type: boolean
|
||||
|
||||
jobs:
|
||||
push_to_docker_hub:
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
steps:
|
||||
- name: Free Disk Space (Ubuntu)
|
||||
uses: jlumbroso/free-disk-space@main
|
||||
with:
|
||||
tool-cache: false
|
||||
android: true
|
||||
dotnet: true
|
||||
haskell: true
|
||||
large-packages: false
|
||||
swap-storage: true
|
||||
docker-images: false
|
||||
|
||||
- name: Check out the Repo
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Prepare for Docker Build&Push
|
||||
id: prepare
|
||||
uses: ./.github/actions/prepare-for-docker-build-and-push
|
||||
with:
|
||||
image: openmetadata/ingestion-base
|
||||
tag: ${{ inputs.docker_release_tag }}
|
||||
push_latest: ${{ inputs.push_latest_tag_to_release }}
|
||||
is_ingestion: true
|
||||
release_version: ${{ inputs.pypi_release_version }}
|
||||
dockerhub_username: ${{ secrets.DOCKERHUB_OPENMETADATA_USERNAME }}
|
||||
dockerhub_token: ${{ secrets.DOCKERHUB_OPENMETADATA_TOKEN }}
|
||||
|
||||
- name: Build and push if event is workflow_dispatch and input is checked
|
||||
uses: docker/build-push-action@v6
|
||||
env:
|
||||
DOCKER_BUILD_NO_SUMMARY: true
|
||||
with:
|
||||
context: .
|
||||
platforms: linux/amd64,linux/arm64
|
||||
push: true
|
||||
tags: ${{ steps.prepare.outputs.tags }}
|
||||
file: ./ingestion/operators/docker/Dockerfile
|
||||
@@ -0,0 +1,91 @@
|
||||
# Copyright 2021 Collate
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
name: docker-openmetadata-ingestion docker
|
||||
on:
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
docker_release_tag:
|
||||
description: "Ingestion Docker Image Tag (3 digit docker image tag)"
|
||||
required: true
|
||||
pypi_release_version:
|
||||
description: "Provide the Release Version (4 digit docker image tag)"
|
||||
required: true
|
||||
push_latest_tag_to_release:
|
||||
description: "Mark this as latest tag as well ?"
|
||||
type: boolean
|
||||
|
||||
jobs:
|
||||
push_to_docker_hub:
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
steps:
|
||||
- name: Free Disk Space (Ubuntu)
|
||||
uses: jlumbroso/free-disk-space@main
|
||||
with:
|
||||
tool-cache: false
|
||||
android: true
|
||||
dotnet: true
|
||||
haskell: true
|
||||
large-packages: false
|
||||
swap-storage: true
|
||||
docker-images: false
|
||||
- name: Check out the Repo
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Prepare for Docker Build&Push
|
||||
id: prepare
|
||||
uses: ./.github/actions/prepare-for-docker-build-and-push
|
||||
with:
|
||||
image: openmetadata/ingestion
|
||||
tag: ${{ inputs.docker_release_tag }}
|
||||
push_latest: ${{ inputs.push_latest_tag_to_release }}
|
||||
is_ingestion: true
|
||||
release_version: ${{ inputs.pypi_release_version }}
|
||||
dockerhub_username: ${{ secrets.DOCKERHUB_OPENMETADATA_USERNAME }}
|
||||
dockerhub_token: ${{ secrets.DOCKERHUB_OPENMETADATA_TOKEN }}
|
||||
|
||||
- name: Build and push if event is workflow_dispatch and input is checked
|
||||
env:
|
||||
DOCKER_BUILD_NO_SUMMARY: true
|
||||
uses: docker/build-push-action@v6
|
||||
with:
|
||||
context: .
|
||||
platforms: linux/amd64,linux/arm64
|
||||
push: true
|
||||
tags: ${{ steps.prepare.outputs.tags }}
|
||||
file: ./ingestion/Dockerfile
|
||||
|
||||
# Publish the openmetadata/ingestion-slim image with fewer dependencies
|
||||
- name: Prepare Slim for Docker Build&Push
|
||||
id: prepare-sim
|
||||
uses: ./.github/actions/prepare-for-docker-build-and-push
|
||||
with:
|
||||
image: openmetadata/ingestion-slim
|
||||
tag: ${{ inputs.docker_release_tag }}
|
||||
push_latest: ${{ inputs.push_latest_tag_to_release }}
|
||||
is_ingestion: true
|
||||
release_version: ${{ inputs.pypi_release_version }}
|
||||
dockerhub_username: ${{ secrets.DOCKERHUB_OPENMETADATA_USERNAME }}
|
||||
dockerhub_token: ${{ secrets.DOCKERHUB_OPENMETADATA_TOKEN }}
|
||||
|
||||
- name: Build and push Slim if event is workflow_dispatch and input is checked
|
||||
env:
|
||||
DOCKER_BUILD_NO_SUMMARY: true
|
||||
uses: docker/build-push-action@v6
|
||||
with:
|
||||
context: .
|
||||
platforms: linux/amd64,linux/arm64
|
||||
push: true
|
||||
tags: ${{ steps.prepare-sim.outputs.tags }}
|
||||
file: ./ingestion/Dockerfile
|
||||
build-args: |
|
||||
INGESTION_DEPENDENCY=slim
|
||||
@@ -0,0 +1,50 @@
|
||||
# Copyright 2021 Collate
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
name: docker-openmetadata-postgres-db docker
|
||||
on:
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
docker_release_tag:
|
||||
description: "OpenMetadata PostgreSQL Docker Image Tag"
|
||||
required: true
|
||||
push_latest_tag_to_release:
|
||||
description: "Do you want to update docker image latest tag as well ?"
|
||||
type: boolean
|
||||
|
||||
jobs:
|
||||
push_to_docker_hub:
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
steps:
|
||||
- name: Check out the Repo
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Prepare for Docker Build&Push
|
||||
id: prepare
|
||||
uses: ./.github/actions/prepare-for-docker-build-and-push
|
||||
with:
|
||||
image: openmetadata/postgresql
|
||||
tag: ${{ inputs.docker_release_tag }}
|
||||
push_latest: ${{ inputs.push_latest_tag_to_release }}
|
||||
dockerhub_username: ${{ secrets.DOCKERHUB_OPENMETADATA_USERNAME }}
|
||||
dockerhub_token: ${{ secrets.DOCKERHUB_OPENMETADATA_TOKEN }}
|
||||
|
||||
- name: Build and push if event is workflow_dispatch and input is checked
|
||||
uses: docker/build-push-action@v6
|
||||
env:
|
||||
DOCKER_BUILD_NO_SUMMARY: true
|
||||
with:
|
||||
context: .
|
||||
platforms: linux/amd64,linux/arm64
|
||||
push: true
|
||||
tags: ${{ steps.prepare.outputs.tags }}
|
||||
file: ./docker/postgresql/Dockerfile_postgres
|
||||
@@ -0,0 +1,121 @@
|
||||
# Copyright 2021 Collate
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
name: docker-openmetadata-server release app
|
||||
on:
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
docker_release_tag:
|
||||
description: "Server Docker Image Tag"
|
||||
required: true
|
||||
push_latest_tag_to_release:
|
||||
description: "Do you want to update docker image latest tag as well ?"
|
||||
type: boolean
|
||||
jobs:
|
||||
maven-build:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Set up JDK 21
|
||||
uses: actions/setup-java@v4
|
||||
with:
|
||||
java-version: 21
|
||||
distribution: 'temurin'
|
||||
|
||||
- name: Install antlr cli
|
||||
run: |
|
||||
sudo make install_antlr_cli
|
||||
|
||||
- name: Setup jq
|
||||
run: |
|
||||
sudo apt-get update
|
||||
sudo apt-get install jq
|
||||
|
||||
- name: Build OpenMetadata Server Application
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
run: |
|
||||
mvn -DskipTests clean package
|
||||
|
||||
- name: Upload OpenMetadata application to Artifact
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: openmetadata-binary
|
||||
path: /home/runner/work/OpenMetadata/OpenMetadata/openmetadata-dist/target/*.tar.gz
|
||||
|
||||
release-project-event-workflow_dispatch:
|
||||
if: github.event_name == 'workflow_dispatch'
|
||||
name: Release app with workflow_dispatch event
|
||||
runs-on: ubuntu-latest
|
||||
needs: maven-build
|
||||
steps:
|
||||
|
||||
- name: Fetch Release Data
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
id: attach_release
|
||||
run: |
|
||||
echo "UPLOAD_URL=$(curl -H "Accept: application/vnd.github+json" -H "Authorization: Bearer $GITHUB_TOKEN" -H "X-GitHub-Api-Version: 2022-11-28" https://api.github.com/repos/open-metadata/OpenMetadata/releases/tags/${{ inputs.DOCKER_RELEASE_TAG }}-release | jq .upload_url | tr -d '"' )" >> $GITHUB_OUTPUT
|
||||
|
||||
- name: Download application from Artifact
|
||||
uses: actions/download-artifact@v4
|
||||
with:
|
||||
name: openmetadata-binary
|
||||
|
||||
- name: Upload app to release
|
||||
uses: actions/upload-release-asset@v1
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
with:
|
||||
asset_path: ./openmetadata-${{ inputs.DOCKER_RELEASE_TAG }}.tar.gz
|
||||
upload_url: ${{ steps.attach_release.outputs.UPLOAD_URL }}
|
||||
asset_name: openmetadata-${{ inputs.DOCKER_RELEASE_TAG }}.tar.gz
|
||||
asset_content_type: application/tar.gz
|
||||
|
||||
push_to_docker_hub:
|
||||
runs-on: ubuntu-latest
|
||||
if: ${{ always() && contains(join(needs.*.result, ','), 'success') }}
|
||||
needs: [ release-project-event-workflow_dispatch ]
|
||||
steps:
|
||||
- name: Check out the Repo
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Set build arguments
|
||||
id: build-args
|
||||
run: |
|
||||
echo "BUILD_DATE=$(date -u +'%Y-%m-%dT%H:%M:%SZ')" >> $GITHUB_OUTPUT
|
||||
|
||||
- name: Prepare for Docker Build&Push
|
||||
id: prepare
|
||||
uses: ./.github/actions/prepare-for-docker-build-and-push
|
||||
with:
|
||||
image: openmetadata/server
|
||||
tag: ${{ inputs.docker_release_tag }}
|
||||
push_latest: ${{ inputs.push_latest_tag_to_release }}
|
||||
dockerhub_username: ${{ secrets.DOCKERHUB_OPENMETADATA_USERNAME }}
|
||||
dockerhub_token: ${{ secrets.DOCKERHUB_OPENMETADATA_TOKEN }}
|
||||
|
||||
- name: Build and push if event is workflow_dispatch and input is checked
|
||||
uses: docker/build-push-action@v6
|
||||
env:
|
||||
DOCKER_BUILD_NO_SUMMARY: true
|
||||
with:
|
||||
context: .
|
||||
platforms: linux/amd64,linux/arm64
|
||||
push: true
|
||||
tags: ${{ steps.prepare.outputs.tags }}
|
||||
file: ./docker/docker-compose-quickstart/Dockerfile
|
||||
build-args: |
|
||||
BUILD_DATE=${{ steps.build-args.outputs.BUILD_DATE }}
|
||||
COMMIT_ID=${{ github.sha }}
|
||||
@@ -0,0 +1,33 @@
|
||||
name: Create Release Branches
|
||||
run-name: Create Release Branch ${{ inputs.release_branch_name }}
|
||||
on:
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
release_branch_name:
|
||||
description: "Github Release Branch Name"
|
||||
required: true
|
||||
base_branch_name:
|
||||
description: "Base Branch for Release Branch"
|
||||
required: true
|
||||
default: "main"
|
||||
permissions:
|
||||
contents: write
|
||||
jobs:
|
||||
create-release-branch:
|
||||
name: Create Release Branch ${{ inputs.release_branch_name }}
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Checkout Repo
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
ref: ${{ inputs.base_branch_name }}
|
||||
- name: Update application versions
|
||||
run: |
|
||||
make update_all RELEASE_VERSION=${{ inputs.release_branch_name }}
|
||||
- name: Commit changes to ${{ inputs.release_branch_name }} branch
|
||||
uses: EndBug/add-and-commit@v9
|
||||
with:
|
||||
default_author: github_actions
|
||||
message: 'chore(release): Prepare Branch for `${{ inputs.release_branch_name }}`'
|
||||
add: '.'
|
||||
new_branch: ${{ inputs.release_branch_name }}
|
||||
@@ -0,0 +1,155 @@
|
||||
# Copyright 2021 Collate
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
name: Integration Tests - MySQL + Elasticsearch
|
||||
|
||||
on:
|
||||
merge_group:
|
||||
workflow_dispatch:
|
||||
push:
|
||||
branches:
|
||||
- main
|
||||
paths:
|
||||
- "openmetadata-service/**"
|
||||
- "openmetadata-integration-tests/**"
|
||||
- "openmetadata-spec/src/main/resources/json/schema/**"
|
||||
- "openmetadata-sdk/**"
|
||||
- "common/**"
|
||||
- "pom.xml"
|
||||
- "bootstrap/**"
|
||||
pull_request_target:
|
||||
types: [labeled, opened, synchronize, reopened, ready_for_review]
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
checks: write
|
||||
|
||||
concurrency:
|
||||
group: integration-tests-mysql-es-${{ github.event.pull_request.number || github.run_id }}
|
||||
cancel-in-progress: ${{ github.event_name != 'pull_request_target' || github.event.action != 'labeled' || github.event.label.name == 'safe to test' }}
|
||||
jobs:
|
||||
# Detect whether relevant paths changed. When no matching files are modified
|
||||
# the downstream job is skipped via its `if` condition.
|
||||
# A job skipped by `if` reports as "Success", so required checks still pass.
|
||||
changes:
|
||||
name: Detect Changes
|
||||
runs-on: ubuntu-latest
|
||||
if: ${{ !github.event.pull_request.draft && (github.event_name != 'pull_request_target' || github.event.action != 'labeled' || github.event.label.name == 'safe to test') }}
|
||||
outputs:
|
||||
backend: ${{ github.event_name == 'workflow_dispatch' && 'true' || steps.filter.outputs.backend }}
|
||||
steps:
|
||||
- uses: dorny/paths-filter@v3
|
||||
id: filter
|
||||
if: ${{ github.event_name != 'workflow_dispatch' }}
|
||||
with:
|
||||
filters: |
|
||||
backend:
|
||||
- 'openmetadata-service/**'
|
||||
- 'openmetadata-integration-tests/**'
|
||||
- 'openmetadata-spec/src/main/resources/json/schema/**'
|
||||
- 'openmetadata-sdk/**'
|
||||
- 'common/**'
|
||||
- 'pom.xml'
|
||||
- 'bootstrap/**'
|
||||
|
||||
integration-tests-mysql-elasticsearch:
|
||||
needs: changes
|
||||
runs-on: ubuntu-latest
|
||||
if: ${{ needs.changes.outputs.backend == 'true' }}
|
||||
steps:
|
||||
- name: Free Disk Space (Ubuntu)
|
||||
uses: jlumbroso/free-disk-space@main
|
||||
with:
|
||||
tool-cache: true
|
||||
android: true
|
||||
dotnet: true
|
||||
haskell: true
|
||||
large-packages: true
|
||||
docker-images: false
|
||||
swap-storage: true
|
||||
|
||||
- name: Wait for the labeler
|
||||
uses: lewagon/wait-on-check-action@v1.7.0
|
||||
if: ${{ github.event_name == 'pull_request_target' }}
|
||||
with:
|
||||
ref: ${{ github.event.pull_request.head.sha }}
|
||||
check-name: Team Label
|
||||
repo-token: ${{ secrets.GITHUB_TOKEN }}
|
||||
wait-interval: 90
|
||||
|
||||
- name: Verify PR labels
|
||||
uses: jesusvasquez333/verify-pr-label-action@v1.4.0
|
||||
if: ${{ github.event_name == 'pull_request_target' }}
|
||||
with:
|
||||
github-token: '${{ secrets.GITHUB_TOKEN }}'
|
||||
valid-labels: 'safe to test'
|
||||
pull-request-number: '${{ github.event.pull_request.number }}'
|
||||
disable-reviews: true # To not auto approve changes
|
||||
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
ref: ${{ github.event_name == 'merge_group' && github.sha || github.event.pull_request.head.sha }}
|
||||
|
||||
- name: Cache Maven dependencies
|
||||
id: cache-output
|
||||
uses: actions/cache@v4
|
||||
with:
|
||||
path: ~/.m2
|
||||
key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}
|
||||
restore-keys: |
|
||||
${{ runner.os }}-maven-
|
||||
|
||||
- name: Set up JDK 21
|
||||
if: steps.cache-output.outputs.exit-code == 0
|
||||
uses: actions/setup-java@v4
|
||||
with:
|
||||
java-version: '21'
|
||||
distribution: 'temurin'
|
||||
|
||||
- name: Install Ubuntu dependencies
|
||||
if: steps.cache-output.outputs.exit-code == 0
|
||||
run: |
|
||||
sudo apt-get update
|
||||
sudo apt-get install -y unixodbc-dev python3-venv librdkafka-dev gcc libsasl2-dev build-essential libssl-dev libffi-dev \
|
||||
librdkafka-dev unixodbc-dev libevent-dev jq
|
||||
sudo make install_antlr_cli
|
||||
|
||||
- name: Build for Integration Tests (MySQL + Elasticsearch)
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
run: mvn -DskipTests clean install -pl :openmetadata-integration-tests -am
|
||||
|
||||
- name: Free build artifacts
|
||||
run: |
|
||||
rm -rf openmetadata-service/target/lib openmetadata-service/target/classes
|
||||
rm -rf openmetadata-spec/target openmetadata-sdk/target common/target
|
||||
rm -rf openmetadata-shaded-deps/*/target
|
||||
df -h /
|
||||
|
||||
- name: Run Integration Tests (MySQL + Elasticsearch)
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
run: mvn verify -pl :openmetadata-integration-tests -Pmysql-elasticsearch
|
||||
|
||||
- name: Clean Up
|
||||
run: |
|
||||
cd ./docker/development
|
||||
docker compose down --remove-orphans
|
||||
sudo rm -rf ${PWD}/docker-volume
|
||||
|
||||
- name: Publish Test Report
|
||||
if: ${{ always() }}
|
||||
uses: scacap/action-surefire-report@v1
|
||||
with:
|
||||
github_token: ${{ secrets.GITHUB_TOKEN }}
|
||||
fail_on_test_failures: true
|
||||
report_paths: 'openmetadata-integration-tests/target/failsafe-reports/TEST-*.xml'
|
||||
@@ -0,0 +1,178 @@
|
||||
# Copyright 2026 Collate
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
# Runs the full integration test suite with the Redis cache enabled (postgres + elasticsearch +
|
||||
# redis), via the cache-tests Maven profile. Catches cache-invalidation and stale-data bugs that
|
||||
# only surface when every test path goes through the cache layer.
|
||||
#
|
||||
# Security note (CodeQL "pull_request_target + checkout untrusted code"):
|
||||
# This workflow uses `pull_request_target` so PRs from forks can produce a required check.
|
||||
# CodeQL flags the pattern as risky because it checks out PR-controlled code while having
|
||||
# access to secrets. The mitigation is the explicit `safe to test` label gate below — the
|
||||
# verify-pr-label step rejects the workflow run before any PR code is checked out unless a
|
||||
# maintainer has applied the label. This matches the mitigation used by every other
|
||||
# integration-tests-*.yml workflow in this repo. If you remove the label gate, you reopen
|
||||
# the vulnerability.
|
||||
name: Integration Tests - PostgreSQL + Elasticsearch + Redis
|
||||
|
||||
on:
|
||||
merge_group:
|
||||
workflow_dispatch:
|
||||
push:
|
||||
branches:
|
||||
- main
|
||||
paths:
|
||||
- "openmetadata-service/**"
|
||||
- "openmetadata-integration-tests/**"
|
||||
- "openmetadata-spec/src/main/resources/json/schema/**"
|
||||
- "openmetadata-sdk/**"
|
||||
- "common/**"
|
||||
- "pom.xml"
|
||||
- "bootstrap/**"
|
||||
# `pull_request_target` is intentional and required so the workflow runs against PRs from
|
||||
# forks (which `pull_request` cannot for security reasons). The `safe to test` label gate
|
||||
# below is what makes this safe — see security note in the file header.
|
||||
pull_request_target:
|
||||
types: [labeled, opened, synchronize, reopened, ready_for_review]
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
checks: write
|
||||
|
||||
concurrency:
|
||||
group: integration-tests-pg-es-redis-${{ github.event.pull_request.number || github.run_id }}
|
||||
cancel-in-progress: ${{ github.event_name != 'pull_request_target' || github.event.action != 'labeled' || github.event.label.name == 'safe to test' }}
|
||||
jobs:
|
||||
# Detect whether relevant paths changed. When no matching files are modified
|
||||
# the downstream job is skipped via its `if` condition.
|
||||
# A job skipped by `if` reports as "Success", so required checks still pass.
|
||||
changes:
|
||||
name: Detect Changes
|
||||
runs-on: ubuntu-latest
|
||||
if: ${{ !github.event.pull_request.draft && (github.event_name != 'pull_request_target' || github.event.action != 'labeled' || github.event.label.name == 'safe to test') }}
|
||||
outputs:
|
||||
backend: ${{ github.event_name == 'workflow_dispatch' && 'true' || steps.filter.outputs.backend }}
|
||||
steps:
|
||||
- uses: dorny/paths-filter@v3
|
||||
id: filter
|
||||
if: ${{ github.event_name != 'workflow_dispatch' }}
|
||||
with:
|
||||
filters: |
|
||||
backend:
|
||||
- 'openmetadata-service/**'
|
||||
- 'openmetadata-integration-tests/**'
|
||||
- 'openmetadata-spec/src/main/resources/json/schema/**'
|
||||
- 'openmetadata-sdk/**'
|
||||
- 'common/**'
|
||||
- 'pom.xml'
|
||||
- 'bootstrap/**'
|
||||
|
||||
integration-tests-postgres-elasticsearch-redis:
|
||||
needs: changes
|
||||
runs-on: ubuntu-latest
|
||||
if: ${{ needs.changes.outputs.backend == 'true' }}
|
||||
steps:
|
||||
- name: Free Disk Space (Ubuntu)
|
||||
uses: jlumbroso/free-disk-space@main
|
||||
with:
|
||||
tool-cache: true
|
||||
android: true
|
||||
dotnet: true
|
||||
haskell: true
|
||||
large-packages: true
|
||||
docker-images: false
|
||||
swap-storage: true
|
||||
|
||||
- name: Wait for the labeler
|
||||
uses: lewagon/wait-on-check-action@v1.7.0
|
||||
if: ${{ github.event_name == 'pull_request_target' }}
|
||||
with:
|
||||
ref: ${{ github.event.pull_request.head.sha }}
|
||||
check-name: Team Label
|
||||
repo-token: ${{ secrets.GITHUB_TOKEN }}
|
||||
wait-interval: 90
|
||||
|
||||
- name: Verify PR labels
|
||||
uses: jesusvasquez333/verify-pr-label-action@v1.4.0
|
||||
if: ${{ github.event_name == 'pull_request_target' }}
|
||||
with:
|
||||
github-token: '${{ secrets.GITHUB_TOKEN }}'
|
||||
valid-labels: 'safe to test'
|
||||
pull-request-number: '${{ github.event.pull_request.number }}'
|
||||
disable-reviews: true # To not auto approve changes
|
||||
|
||||
# SECURITY: this step checks out PR-controlled code while the workflow runs with
|
||||
# `pull_request_target` privileges (secrets access). The `Verify PR labels` step above
|
||||
# gates this — the workflow halts before we get here unless a maintainer has applied
|
||||
# the `safe to test` label. CodeQL flags the pattern; the label gate is the accepted
|
||||
# mitigation, mirroring how every other integration-tests-*.yml workflow in this repo
|
||||
# handles fork PRs.
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
ref: ${{ github.event_name == 'merge_group' && github.sha || github.event.pull_request.head.sha }}
|
||||
|
||||
- name: Cache Maven dependencies
|
||||
id: cache-output
|
||||
uses: actions/cache@v4
|
||||
with:
|
||||
path: ~/.m2
|
||||
key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}
|
||||
restore-keys: |
|
||||
${{ runner.os }}-maven-
|
||||
|
||||
# Run unconditionally. The previous `if: steps.cache-output.outputs.exit-code == 0` was a
|
||||
# bug — `actions/cache@v4` exposes `cache-hit` (boolean) and `cache-primary-key`, never
|
||||
# `exit-code`. The expression always evaluated to false and the steps never ran. Maven
|
||||
# then ran against whatever JDK the runner happened to ship with, masking the issue.
|
||||
- name: Set up JDK 21
|
||||
uses: actions/setup-java@v4
|
||||
with:
|
||||
java-version: '21'
|
||||
distribution: 'temurin'
|
||||
|
||||
- name: Install Ubuntu dependencies
|
||||
run: |
|
||||
sudo apt-get update
|
||||
sudo apt-get install -y unixodbc-dev python3-venv librdkafka-dev gcc libsasl2-dev build-essential libssl-dev libffi-dev \
|
||||
librdkafka-dev unixodbc-dev libevent-dev jq
|
||||
sudo make install_antlr_cli
|
||||
|
||||
- name: Build for Integration Tests (PostgreSQL + Elasticsearch + Redis)
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
run: mvn -DskipTests clean install -pl :openmetadata-integration-tests -am
|
||||
|
||||
- name: Free build artifacts
|
||||
run: |
|
||||
rm -rf openmetadata-service/target/lib openmetadata-service/target/classes
|
||||
rm -rf openmetadata-spec/target openmetadata-sdk/target common/target
|
||||
rm -rf openmetadata-shaded-deps/*/target
|
||||
df -h /
|
||||
|
||||
- name: Run Integration Tests (PostgreSQL + Elasticsearch + Redis)
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
run: mvn verify -pl :openmetadata-integration-tests -Pcache-tests
|
||||
|
||||
- name: Clean Up
|
||||
run: |
|
||||
cd ./docker/development
|
||||
docker compose down --remove-orphans
|
||||
sudo rm -rf ${PWD}/docker-volume
|
||||
|
||||
- name: Publish Test Report
|
||||
if: ${{ always() }}
|
||||
uses: scacap/action-surefire-report@v1
|
||||
with:
|
||||
github_token: ${{ secrets.GITHUB_TOKEN }}
|
||||
fail_on_test_failures: true
|
||||
report_paths: 'openmetadata-integration-tests/target/failsafe-reports/TEST-*.xml'
|
||||
@@ -0,0 +1,155 @@
|
||||
# Copyright 2021 Collate
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
name: Integration Tests - PostgreSQL + OpenSearch
|
||||
|
||||
on:
|
||||
merge_group:
|
||||
workflow_dispatch:
|
||||
push:
|
||||
branches:
|
||||
- main
|
||||
paths:
|
||||
- "openmetadata-service/**"
|
||||
- "openmetadata-integration-tests/**"
|
||||
- "openmetadata-spec/src/main/resources/json/schema/**"
|
||||
- "openmetadata-sdk/**"
|
||||
- "common/**"
|
||||
- "pom.xml"
|
||||
- "bootstrap/**"
|
||||
pull_request_target:
|
||||
types: [labeled, opened, synchronize, reopened, ready_for_review]
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
checks: write
|
||||
|
||||
concurrency:
|
||||
group: integration-tests-pg-os-${{ github.event.pull_request.number || github.run_id }}
|
||||
cancel-in-progress: ${{ github.event_name != 'pull_request_target' || github.event.action != 'labeled' || github.event.label.name == 'safe to test' }}
|
||||
jobs:
|
||||
# Detect whether relevant paths changed. When no matching files are modified
|
||||
# the downstream job is skipped via its `if` condition.
|
||||
# A job skipped by `if` reports as "Success", so required checks still pass.
|
||||
changes:
|
||||
name: Detect Changes
|
||||
runs-on: ubuntu-latest
|
||||
if: ${{ !github.event.pull_request.draft && (github.event_name != 'pull_request_target' || github.event.action != 'labeled' || github.event.label.name == 'safe to test') }}
|
||||
outputs:
|
||||
backend: ${{ github.event_name == 'workflow_dispatch' && 'true' || steps.filter.outputs.backend }}
|
||||
steps:
|
||||
- uses: dorny/paths-filter@v3
|
||||
id: filter
|
||||
if: ${{ github.event_name != 'workflow_dispatch' }}
|
||||
with:
|
||||
filters: |
|
||||
backend:
|
||||
- 'openmetadata-service/**'
|
||||
- 'openmetadata-integration-tests/**'
|
||||
- 'openmetadata-spec/src/main/resources/json/schema/**'
|
||||
- 'openmetadata-sdk/**'
|
||||
- 'common/**'
|
||||
- 'pom.xml'
|
||||
- 'bootstrap/**'
|
||||
|
||||
integration-tests-postgres-opensearch:
|
||||
needs: changes
|
||||
runs-on: ubuntu-latest
|
||||
if: ${{ needs.changes.outputs.backend == 'true' }}
|
||||
steps:
|
||||
- name: Free Disk Space (Ubuntu)
|
||||
uses: jlumbroso/free-disk-space@main
|
||||
with:
|
||||
tool-cache: true
|
||||
android: true
|
||||
dotnet: true
|
||||
haskell: true
|
||||
large-packages: true
|
||||
docker-images: false
|
||||
swap-storage: true
|
||||
|
||||
- name: Wait for the labeler
|
||||
uses: lewagon/wait-on-check-action@v1.7.0
|
||||
if: ${{ github.event_name == 'pull_request_target' }}
|
||||
with:
|
||||
ref: ${{ github.event.pull_request.head.sha }}
|
||||
check-name: Team Label
|
||||
repo-token: ${{ secrets.GITHUB_TOKEN }}
|
||||
wait-interval: 90
|
||||
|
||||
- name: Verify PR labels
|
||||
uses: jesusvasquez333/verify-pr-label-action@v1.4.0
|
||||
if: ${{ github.event_name == 'pull_request_target' }}
|
||||
with:
|
||||
github-token: '${{ secrets.GITHUB_TOKEN }}'
|
||||
valid-labels: 'safe to test'
|
||||
pull-request-number: '${{ github.event.pull_request.number }}'
|
||||
disable-reviews: true # To not auto approve changes
|
||||
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
ref: ${{ github.event_name == 'merge_group' && github.sha || github.event.pull_request.head.sha }}
|
||||
|
||||
- name: Cache Maven dependencies
|
||||
id: cache-output
|
||||
uses: actions/cache@v4
|
||||
with:
|
||||
path: ~/.m2
|
||||
key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}
|
||||
restore-keys: |
|
||||
${{ runner.os }}-maven-
|
||||
|
||||
- name: Set up JDK 21
|
||||
if: steps.cache-output.outputs.exit-code == 0
|
||||
uses: actions/setup-java@v4
|
||||
with:
|
||||
java-version: '21'
|
||||
distribution: 'temurin'
|
||||
|
||||
- name: Install Ubuntu dependencies
|
||||
if: steps.cache-output.outputs.exit-code == 0
|
||||
run: |
|
||||
sudo apt-get update
|
||||
sudo apt-get install -y unixodbc-dev python3-venv librdkafka-dev gcc libsasl2-dev build-essential libssl-dev libffi-dev \
|
||||
librdkafka-dev unixodbc-dev libevent-dev jq
|
||||
sudo make install_antlr_cli
|
||||
|
||||
- name: Build for Integration Tests (PostgreSQL + OpenSearch)
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
run: mvn -DskipTests clean install -pl :openmetadata-integration-tests -am
|
||||
|
||||
- name: Free build artifacts
|
||||
run: |
|
||||
rm -rf openmetadata-service/target/lib openmetadata-service/target/classes
|
||||
rm -rf openmetadata-spec/target openmetadata-sdk/target common/target
|
||||
rm -rf openmetadata-shaded-deps/*/target
|
||||
df -h /
|
||||
|
||||
- name: Run Integration Tests (PostgreSQL + OpenSearch)
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
run: mvn verify -pl :openmetadata-integration-tests -Ppostgres-opensearch
|
||||
|
||||
- name: Clean Up
|
||||
run: |
|
||||
cd ./docker/development
|
||||
docker compose down --remove-orphans
|
||||
sudo rm -rf ${PWD}/docker-volume
|
||||
|
||||
- name: Publish Test Report
|
||||
if: ${{ always() }}
|
||||
uses: scacap/action-surefire-report@v1
|
||||
with:
|
||||
github_token: ${{ secrets.GITHUB_TOKEN }}
|
||||
fail_on_test_failures: true
|
||||
report_paths: 'openmetadata-integration-tests/target/failsafe-reports/TEST-*.xml'
|
||||
@@ -0,0 +1,106 @@
|
||||
# Copyright 2021 Collate
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
name: Java Checkstyle
|
||||
|
||||
on:
|
||||
merge_group:
|
||||
# Trigger analysis when pushing in master or pull requests, and when creating
|
||||
# a pull request.
|
||||
push:
|
||||
branches:
|
||||
- main
|
||||
- "0.[0-9]+.[0-9]+"
|
||||
pull_request_target:
|
||||
types: [opened, synchronize, reopened, labeled]
|
||||
paths-ignore:
|
||||
- "openmetadata-ui/src/main/resources/ui/playwright/doc-generator/**"
|
||||
- "openmetadata-ui/src/main/resources/ui/playwright/docs/**"
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
concurrency:
|
||||
group: java-checkstyle-${{ github.event.pull_request.number || github.run_id }}
|
||||
cancel-in-progress: ${{ github.event_name != 'pull_request_target' || github.event.action != 'labeled' || github.event.label.name == 'safe to test' }}
|
||||
jobs:
|
||||
java-checkstyle:
|
||||
runs-on: ubuntu-latest
|
||||
if: ${{ github.event_name != 'pull_request_target' || github.event.action != 'labeled' || github.event.label.name == 'safe to test' }}
|
||||
permissions:
|
||||
pull-requests: write
|
||||
|
||||
steps:
|
||||
- name: Free Disk Space (Ubuntu)
|
||||
uses: jlumbroso/free-disk-space@main
|
||||
with:
|
||||
tool-cache: false
|
||||
android: true
|
||||
dotnet: true
|
||||
haskell: true
|
||||
large-packages: false
|
||||
swap-storage: true
|
||||
docker-images: false
|
||||
- name: Wait for the labeler
|
||||
uses: lewagon/wait-on-check-action@v1.7.0
|
||||
if: ${{ github.event_name == 'pull_request_target' }}
|
||||
with:
|
||||
ref: ${{ github.event.pull_request.head.sha }}
|
||||
check-name: Team Label
|
||||
repo-token: ${{ secrets.GITHUB_TOKEN }}
|
||||
wait-interval: 90
|
||||
|
||||
- name: Verify PR labels
|
||||
uses: jesusvasquez333/verify-pr-label-action@v1.4.0
|
||||
if: ${{ github.event_name == 'pull_request_target' }}
|
||||
with:
|
||||
github-token: '${{ secrets.GITHUB_TOKEN }}'
|
||||
valid-labels: 'safe to test'
|
||||
pull-request-number: '${{ github.event.pull_request.number }}'
|
||||
disable-reviews: true # To not auto approve changes
|
||||
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
ref: ${{ github.event_name == 'merge_group' && github.sha || github.event.pull_request.head.sha }}
|
||||
|
||||
- name: Set up JDK 21
|
||||
uses: actions/setup-java@v4
|
||||
with:
|
||||
java-version: '21'
|
||||
distribution: 'temurin'
|
||||
|
||||
- name: Run checkstyle
|
||||
run: mvn spotless:apply
|
||||
|
||||
- name: Save checkstyle outcome
|
||||
id: git
|
||||
continue-on-error: true
|
||||
run: |
|
||||
git diff-files --quiet
|
||||
|
||||
- name: Create a comment in the PR with the instructions
|
||||
if: ${{ steps.git.outcome != 'success' && github.event_name == 'pull_request_target' }}
|
||||
uses: peter-evans/create-or-update-comment@v1
|
||||
with:
|
||||
issue-number: ${{ github.event.pull_request.number }}
|
||||
body: |
|
||||
**The Java checkstyle failed.**
|
||||
|
||||
Please run `mvn spotless:apply` in the root of your repository and commit the changes to this PR.
|
||||
You can also use [pre-commit](https://pre-commit.com/) to automate the Java code formatting.
|
||||
|
||||
You can install the pre-commit hooks with `make install_test precommit_install`.
|
||||
|
||||
- name: Java checkstyle failed, check the comment in the PR
|
||||
if: steps.git.outcome != 'success'
|
||||
run: |
|
||||
exit 1
|
||||
@@ -0,0 +1,537 @@
|
||||
# Copyright 2026 Collate
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
# Runs the Java integration suites against an ALREADY-RUNNING external OpenMetadata cluster
|
||||
# instead of a containerized server, across three jobs:
|
||||
# - ui-it-external : *UIIT.java (Playwright) via the `ui-it` profile
|
||||
# - search-it-external : tests/search/*IT.java via the `search-it` profile
|
||||
# - scale-it-external : tests/search/scale/*IT.java via `scale-it`, matrixed over cohort
|
||||
# sizes (scaleSeeds, default [100000]).
|
||||
#
|
||||
# All three jobs share ONE cluster, so each starts with a "Purge orphaned test data" step
|
||||
# (purge-it profile) to clear any bulk cohort a cancelled prior run left behind. Scale runs in
|
||||
# jpw.data.mode=ingest: it creates AND cleans up its own 100k (TestNamespaceExtension), so the
|
||||
# cluster stays clean for ui-it/search-it — whose own reindexes then only ever process their small
|
||||
# per-test fixtures. (Persistent static-100k reuse is for a SEPARATE scale-only cluster, not here.)
|
||||
#
|
||||
# The harness switches to external mode automatically when OM_URL + OM_ADMIN_TOKEN are set (see
|
||||
# UiTestServer.get / ExternalServer.fromEnv / OssTestServer.defaultHandle): no Docker image is
|
||||
# built and no testcontainers (MySQL/ES/OS) are started. Both the SDK clients and the Playwright
|
||||
# browser authenticate with the JWT acquired below.
|
||||
#
|
||||
# Search engine access: the search/scale ITs would normally talk to OpenSearch directly on :9200,
|
||||
# which a remote cluster doesn't expose. In external mode SearchClient routes index introspection
|
||||
# through the server's admin-only, typed /v1/test-support/search endpoints (TestSupportSearchResource,
|
||||
# auto-registered via @Collection). The deployed image on the target cluster MUST therefore (a) be
|
||||
# built from this branch or later, AND (b) set OM_TEST_SUPPORT_SEARCH_ENABLED=true in the SERVER's
|
||||
# environment — the resource is disabled by default so it never ships enabled in production, and
|
||||
# without the flag every external search assertion fails. The flag belongs on the cluster
|
||||
# deployment, not on this CI runner.
|
||||
#
|
||||
# Auth: the target cluster uses basic auth, so we exchange username/password for a JWT via
|
||||
# POST /api/v1/users/login (password base64-encoded) and feed the accessToken as OM_ADMIN_TOKEN.
|
||||
#
|
||||
# Excluded / skipped:
|
||||
# - @Tag("sso-bootstrap") — spin up their own OM lifecycle; pointless against external.
|
||||
# - @Tag("search-direct") — SearchAvailable*/DistributedAutoTune*UIIT read the engine directly;
|
||||
# not yet routed through the passthrough (Phase 2 follow-on).
|
||||
# - LiveIndexRetryIT + ReindexStatsIT#orphanedSchemaDoesNotFailReindex self-skip in external
|
||||
# mode (they need the embedded container / in-JVM DAO).
|
||||
# - GoogleSsoSignInUIIT self-skips under non-SSO backends.
|
||||
|
||||
name: Java Integration Tests (External Cluster)
|
||||
|
||||
on:
|
||||
schedule:
|
||||
# Run daily at midnight (00:00 UTC).
|
||||
- cron: '0 0 * * *'
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
runUiIt:
|
||||
description: 'Run the ui-it (Playwright) job'
|
||||
required: false
|
||||
type: boolean
|
||||
default: true
|
||||
runSearchIt:
|
||||
description: 'Run the search-it job'
|
||||
required: false
|
||||
type: boolean
|
||||
default: true
|
||||
runScaleIt:
|
||||
description: 'Run the scale-it matrix job'
|
||||
required: false
|
||||
type: boolean
|
||||
default: true
|
||||
itTest:
|
||||
description: 'Optional single class/glob to run for ui-it (failsafe -Dit.test). Empty = full suite.'
|
||||
required: false
|
||||
type: string
|
||||
default: ''
|
||||
scaleSeeds:
|
||||
description: 'JSON array of scale table cohort sizes (matrix). E.g. [100000] or [100000, 500000].'
|
||||
required: false
|
||||
type: string
|
||||
default: '[100000]'
|
||||
scaleTimeoutMin:
|
||||
description: 'Per-run reindex timeout (minutes) for scale tests'
|
||||
required: false
|
||||
type: string
|
||||
default: '300'
|
||||
reindexTimeoutMin:
|
||||
description: 'Per-run reindex completion/acceptance timeout (minutes) for ui-it & search-it'
|
||||
required: false
|
||||
type: string
|
||||
default: '60'
|
||||
entityLoaderMaxWorkers:
|
||||
description: 'Cap on EntityLoader create concurrency (keep low for a shared/proxied cluster to avoid 504s)'
|
||||
required: false
|
||||
type: string
|
||||
default: '8'
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
checks: write
|
||||
|
||||
jobs:
|
||||
ui-it-external:
|
||||
if: ${{ github.event.inputs.runUiIt != 'false' }}
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 90
|
||||
steps:
|
||||
- name: Free Disk Space (Ubuntu)
|
||||
uses: jlumbroso/free-disk-space@main
|
||||
with:
|
||||
tool-cache: true
|
||||
android: true
|
||||
dotnet: true
|
||||
haskell: true
|
||||
large-packages: true
|
||||
docker-images: false
|
||||
swap-storage: true
|
||||
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
ref: ${{ github.ref }}
|
||||
|
||||
- name: Set up JDK 21
|
||||
uses: actions/setup-java@v4
|
||||
with:
|
||||
java-version: '21'
|
||||
distribution: 'temurin'
|
||||
|
||||
- name: Cache Maven dependencies
|
||||
uses: actions/cache@v4
|
||||
with:
|
||||
path: ~/.m2
|
||||
key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}
|
||||
restore-keys: |
|
||||
${{ runner.os }}-maven-
|
||||
|
||||
- name: Install Ubuntu dependencies
|
||||
run: |
|
||||
sudo apt-get update
|
||||
sudo apt-get install -y unixodbc-dev python3-venv librdkafka-dev gcc libsasl2-dev build-essential libssl-dev libffi-dev \
|
||||
libevent-dev jq
|
||||
sudo make install_antlr_cli
|
||||
|
||||
- name: Build dependencies for integration-tests
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
run: mvn -DskipTests install -pl :openmetadata-integration-tests -am
|
||||
|
||||
- name: Install Playwright browsers
|
||||
run: |
|
||||
mvn -pl :openmetadata-integration-tests dependency:build-classpath -Dmdep.outputFile=/tmp/cp.txt -q
|
||||
java -cp "$(cat /tmp/cp.txt)" com.microsoft.playwright.CLI install --with-deps chromium
|
||||
|
||||
- name: Acquire admin JWT from external cluster
|
||||
env:
|
||||
OM_EXTERNAL_URL: ${{ secrets.OM_EXTERNAL_URL }}
|
||||
OM_EXTERNAL_USERNAME: ${{ secrets.OM_EXTERNAL_USERNAME }}
|
||||
OM_EXTERNAL_PASSWORD: ${{ secrets.OM_EXTERNAL_PASSWORD }}
|
||||
run: |
|
||||
if [ -z "$OM_EXTERNAL_URL" ] || [ -z "$OM_EXTERNAL_USERNAME" ] || [ -z "$OM_EXTERNAL_PASSWORD" ]; then
|
||||
echo "Missing OM_EXTERNAL_URL / OM_EXTERNAL_USERNAME / OM_EXTERNAL_PASSWORD secrets"
|
||||
exit 1
|
||||
fi
|
||||
# OM basic-auth login expects a base64-encoded password (BasicAuthServletHandler).
|
||||
PW_B64=$(printf '%s' "$OM_EXTERNAL_PASSWORD" | base64 | tr -d '\n')
|
||||
TOKEN=$(curl -fsS -X POST "${OM_EXTERNAL_URL%/}/api/v1/users/login" \
|
||||
-H 'Content-Type: application/json' \
|
||||
-d "{\"email\":\"${OM_EXTERNAL_USERNAME}\",\"password\":\"${PW_B64}\"}" \
|
||||
| jq -r '.accessToken')
|
||||
if [ -z "$TOKEN" ] || [ "$TOKEN" = "null" ]; then
|
||||
echo "Login to ${OM_EXTERNAL_URL} failed — no accessToken returned"
|
||||
exit 1
|
||||
fi
|
||||
echo "::add-mask::$TOKEN"
|
||||
echo "OM_ADMIN_TOKEN=$TOKEN" >> "$GITHUB_ENV"
|
||||
echo "OM_URL=${OM_EXTERNAL_URL%/}" >> "$GITHUB_ENV"
|
||||
|
||||
- name: Purge orphaned test data (clean cluster before run)
|
||||
# Functional suites must start on a cluster free of leftover bulk cohorts — a cancelled prior
|
||||
# run skips per-test cleanup, and a stale 100k makes every reindex reprocess it (~20m each).
|
||||
# purge-it hard-deletes every test-namespace root (matched by the __<run-id>__ signature), so
|
||||
# reindexes here only ever process this run's own small fixtures.
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
OM_URL: ${{ env.OM_URL }}
|
||||
OM_ADMIN_TOKEN: ${{ env.OM_ADMIN_TOKEN }}
|
||||
OM_EXTERNAL_USERNAME: ${{ secrets.OM_EXTERNAL_USERNAME }}
|
||||
OM_EXTERNAL_PASSWORD: ${{ secrets.OM_EXTERNAL_PASSWORD }}
|
||||
run: |
|
||||
mvn verify -P purge-it -pl :openmetadata-integration-tests -Dskip.embedded.bootstrap=true
|
||||
|
||||
- name: Run UI integration tests (external cluster)
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
OM_URL: ${{ env.OM_URL }}
|
||||
OM_ADMIN_TOKEN: ${{ env.OM_ADMIN_TOKEN }}
|
||||
# Credentials so the harness can re-login and auto-renew the token on long runs.
|
||||
OM_EXTERNAL_USERNAME: ${{ secrets.OM_EXTERNAL_USERNAME }}
|
||||
OM_EXTERNAL_PASSWORD: ${{ secrets.OM_EXTERNAL_PASSWORD }}
|
||||
PW_VIDEO: 'true'
|
||||
run: |
|
||||
STRESS_PROPS="-Djpw.simpleReindex.tables=${{ github.event.inputs.simpleReindexTables || '5000' }} \
|
||||
-Djpw.simpleReindex.topics=${{ github.event.inputs.simpleReindexTopics || '1500' }} \
|
||||
-Djpw.simpleReindex.dashboards=${{ github.event.inputs.simpleReindexDashboards || '1500' }} \
|
||||
-Djpw.simpleReindex.pipelines=${{ github.event.inputs.simpleReindexPipelines || '2000' }} \
|
||||
-Djpw.searchAvailable.tables=${{ github.event.inputs.searchAvailableTables || '5000' }}"
|
||||
IT_TEST_PROP=""
|
||||
if [ -n "${{ github.event.inputs.itTest }}" ]; then
|
||||
IT_TEST_PROP="-Dit.test=${{ github.event.inputs.itTest }}"
|
||||
fi
|
||||
mvn verify -P ui-it -pl :openmetadata-integration-tests $STRESS_PROPS $IT_TEST_PROP \
|
||||
-Djpw.loader.maxWorkers=${{ github.event.inputs.entityLoaderMaxWorkers || '8' }} \
|
||||
-Djpw.reindex.timeoutMin=${{ github.event.inputs.reindexTimeoutMin || '60' }} \
|
||||
-Dui.it.excludedGroups="sso-bootstrap,search-direct"
|
||||
|
||||
- name: Upload Playwright traces
|
||||
if: ${{ always() }}
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: playwright-traces-external-${{ github.run_id }}
|
||||
path: openmetadata-integration-tests/target/playwright-traces
|
||||
if-no-files-found: ignore
|
||||
retention-days: 14
|
||||
|
||||
- name: Upload Playwright videos
|
||||
if: ${{ always() }}
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: playwright-videos-external-${{ github.run_id }}
|
||||
path: openmetadata-integration-tests/target/playwright-videos
|
||||
if-no-files-found: ignore
|
||||
retention-days: 14
|
||||
|
||||
- name: Upload Failsafe Reports
|
||||
if: ${{ always() }}
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: failsafe-reports-ui-it-external-${{ github.run_id }}
|
||||
path: openmetadata-integration-tests/target/failsafe-reports
|
||||
if-no-files-found: ignore
|
||||
retention-days: 14
|
||||
|
||||
- name: Publish Test Report
|
||||
if: ${{ always() }}
|
||||
uses: scacap/action-surefire-report@v1
|
||||
with:
|
||||
github_token: ${{ secrets.GITHUB_TOKEN }}
|
||||
fail_on_test_failures: false
|
||||
report_paths: 'openmetadata-integration-tests/target/failsafe-reports/TEST-*.xml'
|
||||
|
||||
- name: Slack notification
|
||||
if: ${{ always() && env.SLACK_JAVA_PLAYWRIGHT_URL != '' }}
|
||||
uses: slackapi/slack-github-action@v2.0.0
|
||||
with:
|
||||
webhook: ${{ env.SLACK_JAVA_PLAYWRIGHT_URL }}
|
||||
webhook-type: incoming-webhook
|
||||
payload: |
|
||||
text: "${{ job.status == 'success' && ':white_check_mark:' || ':fire:' }} Java UIIT External: ${{ job.status }}\nRef: ${{ github.ref_name }}\nLogs: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"
|
||||
env:
|
||||
SLACK_JAVA_PLAYWRIGHT_URL: ${{ secrets.SLACK_JAVA_PLAYWRIGHT_URL }}
|
||||
|
||||
search-it-external:
|
||||
# These suites trigger the server-global SearchIndexingApplication, which is a singleton per
|
||||
# instance (one run at a time). Since all external jobs hit the SAME cluster, they must run
|
||||
# serially or they collide with "Job is already running". `needs` enforces ordering; `always()`
|
||||
# keeps each job's own toggle independent (so it still runs if ui-it was skipped/failed).
|
||||
needs: [ui-it-external]
|
||||
if: ${{ always() && github.event.inputs.runSearchIt != 'false' }}
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 90
|
||||
steps:
|
||||
- name: Free Disk Space (Ubuntu)
|
||||
uses: jlumbroso/free-disk-space@main
|
||||
with:
|
||||
tool-cache: true
|
||||
android: true
|
||||
dotnet: true
|
||||
haskell: true
|
||||
large-packages: true
|
||||
docker-images: false
|
||||
swap-storage: true
|
||||
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
ref: ${{ github.ref }}
|
||||
|
||||
- name: Set up JDK 21
|
||||
uses: actions/setup-java@v4
|
||||
with:
|
||||
java-version: '21'
|
||||
distribution: 'temurin'
|
||||
|
||||
- name: Cache Maven dependencies
|
||||
uses: actions/cache@v4
|
||||
with:
|
||||
path: ~/.m2
|
||||
key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}
|
||||
restore-keys: |
|
||||
${{ runner.os }}-maven-
|
||||
|
||||
- name: Install Ubuntu dependencies
|
||||
run: |
|
||||
sudo apt-get update
|
||||
sudo apt-get install -y unixodbc-dev python3-venv librdkafka-dev gcc libsasl2-dev build-essential libssl-dev libffi-dev \
|
||||
libevent-dev jq
|
||||
sudo make install_antlr_cli
|
||||
|
||||
- name: Build dependencies for integration-tests
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
run: mvn -DskipTests install -pl :openmetadata-integration-tests -am
|
||||
|
||||
- name: Acquire admin JWT from external cluster
|
||||
env:
|
||||
OM_EXTERNAL_URL: ${{ secrets.OM_EXTERNAL_URL }}
|
||||
OM_EXTERNAL_USERNAME: ${{ secrets.OM_EXTERNAL_USERNAME }}
|
||||
OM_EXTERNAL_PASSWORD: ${{ secrets.OM_EXTERNAL_PASSWORD }}
|
||||
run: |
|
||||
if [ -z "$OM_EXTERNAL_URL" ] || [ -z "$OM_EXTERNAL_USERNAME" ] || [ -z "$OM_EXTERNAL_PASSWORD" ]; then
|
||||
echo "Missing OM_EXTERNAL_URL / OM_EXTERNAL_USERNAME / OM_EXTERNAL_PASSWORD secrets"
|
||||
exit 1
|
||||
fi
|
||||
PW_B64=$(printf '%s' "$OM_EXTERNAL_PASSWORD" | base64 | tr -d '\n')
|
||||
TOKEN=$(curl -fsS -X POST "${OM_EXTERNAL_URL%/}/api/v1/users/login" \
|
||||
-H 'Content-Type: application/json' \
|
||||
-d "{\"email\":\"${OM_EXTERNAL_USERNAME}\",\"password\":\"${PW_B64}\"}" \
|
||||
| jq -r '.accessToken')
|
||||
if [ -z "$TOKEN" ] || [ "$TOKEN" = "null" ]; then
|
||||
echo "Login to ${OM_EXTERNAL_URL} failed — no accessToken returned"
|
||||
exit 1
|
||||
fi
|
||||
echo "::add-mask::$TOKEN"
|
||||
echo "OM_ADMIN_TOKEN=$TOKEN" >> "$GITHUB_ENV"
|
||||
echo "OM_URL=${OM_EXTERNAL_URL%/}" >> "$GITHUB_ENV"
|
||||
|
||||
- name: Purge orphaned test data (clean cluster before run)
|
||||
# search-it does ~14 full reindexes; on a cluster carrying a stale 100k each is ~20m and the
|
||||
# job blows its cap. Purge every test-namespace root first so reindexes only process this
|
||||
# run's own small per-test fixtures.
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
OM_URL: ${{ env.OM_URL }}
|
||||
OM_ADMIN_TOKEN: ${{ env.OM_ADMIN_TOKEN }}
|
||||
OM_EXTERNAL_USERNAME: ${{ secrets.OM_EXTERNAL_USERNAME }}
|
||||
OM_EXTERNAL_PASSWORD: ${{ secrets.OM_EXTERNAL_PASSWORD }}
|
||||
run: |
|
||||
mvn verify -P purge-it -pl :openmetadata-integration-tests -Dskip.embedded.bootstrap=true
|
||||
|
||||
- name: Run search integration tests (external cluster)
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
OM_URL: ${{ env.OM_URL }}
|
||||
OM_ADMIN_TOKEN: ${{ env.OM_ADMIN_TOKEN }}
|
||||
# Credentials so the harness can re-login and auto-renew the token on long runs.
|
||||
OM_EXTERNAL_USERNAME: ${{ secrets.OM_EXTERNAL_USERNAME }}
|
||||
OM_EXTERNAL_PASSWORD: ${{ secrets.OM_EXTERNAL_PASSWORD }}
|
||||
run: |
|
||||
mvn verify -P search-it -pl :openmetadata-integration-tests -Dskip.embedded.bootstrap=true \
|
||||
-Djpw.loader.maxWorkers=${{ github.event.inputs.entityLoaderMaxWorkers || '8' }} \
|
||||
-Djpw.reindex.timeoutMin=${{ github.event.inputs.reindexTimeoutMin || '60' }}
|
||||
|
||||
- name: Upload Failsafe Reports
|
||||
if: ${{ always() }}
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: failsafe-reports-search-it-external-${{ github.run_id }}
|
||||
path: openmetadata-integration-tests/target/failsafe-reports
|
||||
if-no-files-found: ignore
|
||||
retention-days: 14
|
||||
|
||||
- name: Publish Test Report
|
||||
if: ${{ always() }}
|
||||
uses: scacap/action-surefire-report@v1
|
||||
with:
|
||||
github_token: ${{ secrets.GITHUB_TOKEN }}
|
||||
fail_on_test_failures: false
|
||||
report_paths: 'openmetadata-integration-tests/target/failsafe-reports/TEST-*.xml'
|
||||
|
||||
- name: Slack notification
|
||||
if: ${{ always() && env.SLACK_JAVA_PLAYWRIGHT_URL != '' }}
|
||||
uses: slackapi/slack-github-action@v2.0.0
|
||||
with:
|
||||
webhook: ${{ env.SLACK_JAVA_PLAYWRIGHT_URL }}
|
||||
webhook-type: incoming-webhook
|
||||
payload: |
|
||||
text: "${{ job.status == 'success' && ':white_check_mark:' || ':fire:' }} Java Search ITs External: ${{ job.status }}\nRef: ${{ github.ref_name }}\nLogs: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"
|
||||
env:
|
||||
SLACK_JAVA_PLAYWRIGHT_URL: ${{ secrets.SLACK_JAVA_PLAYWRIGHT_URL }}
|
||||
|
||||
scale-it-external:
|
||||
# Runs after search-it (shared instance, singleton reindex app). max-parallel: 1 so the matrix
|
||||
# entries (e.g. 100k then 500k) seed + reindex one at a time — two concurrent reindexes on one
|
||||
# cluster collide and the db/es counts would mix cohorts.
|
||||
needs: [search-it-external]
|
||||
if: ${{ always() && github.event.inputs.runScaleIt != 'false' }}
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 360
|
||||
strategy:
|
||||
fail-fast: false
|
||||
max-parallel: 1
|
||||
matrix:
|
||||
seed: ${{ fromJSON(github.event.inputs.scaleSeeds || '[100000, 500000]') }}
|
||||
steps:
|
||||
- name: Free Disk Space (Ubuntu)
|
||||
uses: jlumbroso/free-disk-space@main
|
||||
with:
|
||||
tool-cache: true
|
||||
android: true
|
||||
dotnet: true
|
||||
haskell: true
|
||||
large-packages: true
|
||||
docker-images: false
|
||||
swap-storage: true
|
||||
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
ref: ${{ github.ref }}
|
||||
|
||||
- name: Set up JDK 21
|
||||
uses: actions/setup-java@v4
|
||||
with:
|
||||
java-version: '21'
|
||||
distribution: 'temurin'
|
||||
|
||||
- name: Cache Maven dependencies
|
||||
uses: actions/cache@v4
|
||||
with:
|
||||
path: ~/.m2
|
||||
key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}
|
||||
restore-keys: |
|
||||
${{ runner.os }}-maven-
|
||||
|
||||
- name: Install Ubuntu dependencies
|
||||
run: |
|
||||
sudo apt-get update
|
||||
sudo apt-get install -y unixodbc-dev python3-venv librdkafka-dev gcc libsasl2-dev build-essential libssl-dev libffi-dev \
|
||||
libevent-dev jq
|
||||
sudo make install_antlr_cli
|
||||
|
||||
- name: Build dependencies for integration-tests
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
run: mvn -DskipTests install -pl :openmetadata-integration-tests -am
|
||||
|
||||
- name: Acquire admin JWT from external cluster
|
||||
env:
|
||||
OM_EXTERNAL_URL: ${{ secrets.OM_EXTERNAL_URL }}
|
||||
OM_EXTERNAL_USERNAME: ${{ secrets.OM_EXTERNAL_USERNAME }}
|
||||
OM_EXTERNAL_PASSWORD: ${{ secrets.OM_EXTERNAL_PASSWORD }}
|
||||
run: |
|
||||
if [ -z "$OM_EXTERNAL_URL" ] || [ -z "$OM_EXTERNAL_USERNAME" ] || [ -z "$OM_EXTERNAL_PASSWORD" ]; then
|
||||
echo "Missing OM_EXTERNAL_URL / OM_EXTERNAL_USERNAME / OM_EXTERNAL_PASSWORD secrets"
|
||||
exit 1
|
||||
fi
|
||||
PW_B64=$(printf '%s' "$OM_EXTERNAL_PASSWORD" | base64 | tr -d '\n')
|
||||
TOKEN=$(curl -fsS -X POST "${OM_EXTERNAL_URL%/}/api/v1/users/login" \
|
||||
-H 'Content-Type: application/json' \
|
||||
-d "{\"email\":\"${OM_EXTERNAL_USERNAME}\",\"password\":\"${PW_B64}\"}" \
|
||||
| jq -r '.accessToken')
|
||||
if [ -z "$TOKEN" ] || [ "$TOKEN" = "null" ]; then
|
||||
echo "Login to ${OM_EXTERNAL_URL} failed — no accessToken returned"
|
||||
exit 1
|
||||
fi
|
||||
echo "::add-mask::$TOKEN"
|
||||
echo "OM_ADMIN_TOKEN=$TOKEN" >> "$GITHUB_ENV"
|
||||
echo "OM_URL=${OM_EXTERNAL_URL%/}" >> "$GITHUB_ENV"
|
||||
|
||||
- name: Purge orphaned test data (clean cluster before run)
|
||||
# Clear any bulk cohort a cancelled prior run left behind. The scale test (ingest mode)
|
||||
# creates and then cleans up its own 100k, so the cluster returns to clean; this purge just
|
||||
# guarantees a clean starting point.
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
OM_URL: ${{ env.OM_URL }}
|
||||
OM_ADMIN_TOKEN: ${{ env.OM_ADMIN_TOKEN }}
|
||||
OM_EXTERNAL_USERNAME: ${{ secrets.OM_EXTERNAL_USERNAME }}
|
||||
OM_EXTERNAL_PASSWORD: ${{ secrets.OM_EXTERNAL_PASSWORD }}
|
||||
run: |
|
||||
mvn verify -P purge-it -pl :openmetadata-integration-tests -Dskip.embedded.bootstrap=true
|
||||
|
||||
- name: Run scale test (${{ matrix.seed }} tables, external cluster)
|
||||
# Self-contained: ingest mode creates the cohort and cleans it up afterwards
|
||||
# (TestNamespaceExtension), so the cluster returns to clean for the next run.
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
OM_URL: ${{ env.OM_URL }}
|
||||
OM_ADMIN_TOKEN: ${{ env.OM_ADMIN_TOKEN }}
|
||||
# Credentials so the harness can re-login and auto-renew the token on long runs.
|
||||
OM_EXTERNAL_USERNAME: ${{ secrets.OM_EXTERNAL_USERNAME }}
|
||||
OM_EXTERNAL_PASSWORD: ${{ secrets.OM_EXTERNAL_PASSWORD }}
|
||||
run: |
|
||||
mvn verify -P scale-it -pl :openmetadata-integration-tests -Dskip.embedded.bootstrap=true \
|
||||
-Djpw.scale.tables=${{ matrix.seed }} \
|
||||
-Djpw.scale.timeoutMin=${{ github.event.inputs.scaleTimeoutMin || '300' }} \
|
||||
-Djpw.data.mode=ingest \
|
||||
-Djpw.loader.maxWorkers=${{ github.event.inputs.entityLoaderMaxWorkers || '8' }}
|
||||
|
||||
- name: Upload scale metrics
|
||||
if: ${{ always() }}
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: scale-metrics-${{ matrix.seed }}-${{ github.run_id }}
|
||||
path: openmetadata-integration-tests/target/benchmark
|
||||
if-no-files-found: ignore
|
||||
retention-days: 30
|
||||
|
||||
- name: Upload Failsafe Reports
|
||||
if: ${{ always() }}
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: failsafe-reports-scale-${{ matrix.seed }}-${{ github.run_id }}
|
||||
path: openmetadata-integration-tests/target/failsafe-reports
|
||||
if-no-files-found: ignore
|
||||
retention-days: 14
|
||||
|
||||
- name: Publish Test Report
|
||||
if: ${{ always() }}
|
||||
uses: scacap/action-surefire-report@v1
|
||||
with:
|
||||
github_token: ${{ secrets.GITHUB_TOKEN }}
|
||||
fail_on_test_failures: false
|
||||
report_paths: 'openmetadata-integration-tests/target/failsafe-reports/TEST-*.xml'
|
||||
|
||||
- name: Slack notification
|
||||
if: ${{ always() && env.SLACK_JAVA_PLAYWRIGHT_URL != '' }}
|
||||
uses: slackapi/slack-github-action@v2.0.0
|
||||
with:
|
||||
webhook: ${{ env.SLACK_JAVA_PLAYWRIGHT_URL }}
|
||||
webhook-type: incoming-webhook
|
||||
payload: |
|
||||
text: "${{ job.status == 'success' && ':white_check_mark:' || ':fire:' }} Java Scale IT External (${{ matrix.seed }} tables): ${{ job.status }}\nRef: ${{ github.ref_name }}\nLogs: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"
|
||||
env:
|
||||
SLACK_JAVA_PLAYWRIGHT_URL: ${{ secrets.SLACK_JAVA_PLAYWRIGHT_URL }}
|
||||
@@ -0,0 +1,455 @@
|
||||
# Copyright 2026 Collate
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
# Full run of the UI integration suite (*UIIT.java) — lives inside
|
||||
# openmetadata-integration-tests under the `ui-it` Maven profile. Runs the
|
||||
# external-mode matrix (ES + OS). Tracks EPIC #3731 / tickets #3767, #3792.
|
||||
#
|
||||
# The schedule trigger is intentionally disabled while the suite stabilises. Run it
|
||||
# on demand via workflow_dispatch (pick the branch to test as the ref); it also runs
|
||||
# automatically on any PR that touches search-indexing code (see the pull_request
|
||||
# `paths` filter below), exercising this whole suite (both engines + search-it) against
|
||||
# the PR head in embedded mode. Re-add `schedule: - cron: '0 2 * * *'` once green on main.
|
||||
#
|
||||
# Topology:
|
||||
# - build-image: builds the openmetadata-dist tarball and bakes the local
|
||||
# openmetadata-server:jpw-snapshot image (BuildKit + GHA layer cache), saves it
|
||||
# to a workflow artifact, and primes the Maven cache for downstream jobs.
|
||||
# - ui-it-nightly (x2): matrix over [opensearch, elasticsearch]. Both download the
|
||||
# pre-built image artifact, restore the Maven cache, then run `mvn verify -P ui-it`
|
||||
# with OM_TEST_IMAGE pointing at the pre-loaded tag so ContainerizedServer skips
|
||||
# its own build.
|
||||
# - search-it-nightly: server-global destructive search ITs (tests/search/*IT.java) via
|
||||
# `mvn verify -P search-it`. They run on the embedded bootstrap (testcontainers), not the
|
||||
# baked image, so this job is independent of build-image and runs them serially.
|
||||
|
||||
name: JavaUIIT Integration Tests (Nightly)
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
includeSsoBootstrap:
|
||||
description: 'Also run @Tag("sso-bootstrap") UIITs (slower; second OM lifecycle)'
|
||||
required: false
|
||||
type: boolean
|
||||
default: false
|
||||
simpleReindexTables:
|
||||
description: 'SimpleReindexTriggerUIIT table cohort size'
|
||||
required: false
|
||||
type: string
|
||||
default: '5000'
|
||||
simpleReindexTopics:
|
||||
description: 'SimpleReindexTriggerUIIT topic cohort size'
|
||||
required: false
|
||||
type: string
|
||||
default: '1500'
|
||||
simpleReindexDashboards:
|
||||
description: 'SimpleReindexTriggerUIIT dashboard cohort size'
|
||||
required: false
|
||||
type: string
|
||||
default: '1500'
|
||||
simpleReindexPipelines:
|
||||
description: 'SimpleReindexTriggerUIIT pipeline cohort size'
|
||||
required: false
|
||||
type: string
|
||||
default: '2000'
|
||||
searchAvailableTables:
|
||||
description: 'SearchAvailableDuringReindexUIIT table cohort size'
|
||||
required: false
|
||||
type: string
|
||||
default: '5000'
|
||||
reindexTimeoutMin:
|
||||
description: 'Per-run reindex completion/acceptance timeout (minutes) for ui-it & search-it'
|
||||
required: false
|
||||
type: string
|
||||
default: '60'
|
||||
# Auto-run on PRs that modify search-indexing code (paths below): the full suite runs against
|
||||
# the PR head in embedded mode (testcontainers), NOT the external cluster — so a change to the
|
||||
# reindex app, search clients, or index mappings is validated before merge without anyone
|
||||
# remembering a label. Uses `pull_request` (NOT pull_request_target) on purpose: fork PRs run
|
||||
# WITHOUT secrets, so untrusted code never executes with credentials; same-repo branch PRs (the
|
||||
# common case here) still get secrets and full check reporting.
|
||||
pull_request:
|
||||
paths:
|
||||
# --- Production search-indexing code (openmetadata-service) ---
|
||||
- "openmetadata-service/src/main/java/org/openmetadata/service/search/**"
|
||||
- "openmetadata-service/src/main/java/org/openmetadata/service/apps/bundles/searchIndex/**"
|
||||
- "openmetadata-service/src/main/java/org/openmetadata/service/resources/searchindex/**"
|
||||
- "openmetadata-service/src/main/java/org/openmetadata/service/resources/search/**"
|
||||
- "openmetadata-service/src/main/java/org/openmetadata/service/resources/testsupport/**"
|
||||
- "openmetadata-service/src/main/java/org/openmetadata/service/events/lifecycle/handlers/SearchIndexHandler.java"
|
||||
# --- Index mappings + loader (canonical source lives in the spec module) ---
|
||||
- "openmetadata-spec/src/main/resources/elasticsearch/**"
|
||||
- "openmetadata-spec/src/main/java/org/openmetadata/search/**"
|
||||
# --- The search ITs / UIITs, their test harness, and the Maven profiles that run them ---
|
||||
- "openmetadata-integration-tests/src/test/java/org/openmetadata/it/tests/search/**"
|
||||
- "openmetadata-integration-tests/src/test/java/org/openmetadata/it/search/**"
|
||||
- "openmetadata-integration-tests/src/test/java/org/openmetadata/playwright/scenarios/search/**"
|
||||
- "openmetadata-integration-tests/pom.xml"
|
||||
# --- This workflow itself ---
|
||||
- ".github/workflows/java-playwright-nightly.yml"
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
checks: write
|
||||
|
||||
concurrency:
|
||||
# One run per PR (or per ref for dispatch/cron); a new push cancels the prior in-flight run.
|
||||
group: java-playwright-nightly-${{ github.event.pull_request.number || github.ref }}
|
||||
cancel-in-progress: true
|
||||
|
||||
env:
|
||||
SERVER_IMAGE_TAG: openmetadata-server:jpw-snapshot
|
||||
|
||||
jobs:
|
||||
build-image:
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 30
|
||||
outputs:
|
||||
ref-sha: ${{ steps.checkout.outputs.commit }}
|
||||
steps:
|
||||
- name: Free Disk Space (Ubuntu)
|
||||
uses: jlumbroso/free-disk-space@main
|
||||
with:
|
||||
tool-cache: true
|
||||
android: true
|
||||
dotnet: true
|
||||
haskell: true
|
||||
large-packages: true
|
||||
docker-images: false
|
||||
swap-storage: true
|
||||
|
||||
- name: Checkout
|
||||
id: checkout
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
# On a PR, test the PR head (the changed code). workflow_dispatch honours the
|
||||
# dispatched ref so feature branches can validate the nightly matrix before merge.
|
||||
# Cron/anything else runs against main (EPIC #3731 / PR #28008).
|
||||
ref: ${{ github.event.pull_request.head.sha || (github.event_name == 'workflow_dispatch' && github.ref || 'main') }}
|
||||
|
||||
- name: Set up JDK 21
|
||||
uses: actions/setup-java@v4
|
||||
with:
|
||||
java-version: '21'
|
||||
distribution: 'temurin'
|
||||
|
||||
- name: Cache Maven dependencies
|
||||
uses: actions/cache@v4
|
||||
with:
|
||||
path: ~/.m2
|
||||
key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}
|
||||
restore-keys: |
|
||||
${{ runner.os }}-maven-
|
||||
|
||||
- name: Install Ubuntu dependencies
|
||||
run: |
|
||||
sudo apt-get update
|
||||
sudo apt-get install -y unixodbc-dev python3-venv librdkafka-dev gcc libsasl2-dev build-essential libssl-dev libffi-dev \
|
||||
libevent-dev jq
|
||||
sudo make install_antlr_cli
|
||||
|
||||
- name: Build dependencies for integration-tests
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
run: mvn -DskipTests clean install -pl :openmetadata-integration-tests -am
|
||||
|
||||
- name: Build openmetadata-dist tarball
|
||||
# ContainerizedServer.buildLocalImageContainer needs
|
||||
# openmetadata-dist/target/openmetadata-*.tar.gz to bake the local image.
|
||||
# Dist is NOT a transitive dep of integration-tests, so the previous step
|
||||
# doesn't produce it — build it explicitly here.
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
run: mvn -DskipTests install -pl :openmetadata-dist -am
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@v3
|
||||
|
||||
- name: Build openmetadata-server image (with GHA layer cache)
|
||||
uses: docker/build-push-action@v6
|
||||
with:
|
||||
context: .
|
||||
file: docker/development/Dockerfile
|
||||
tags: ${{ env.SERVER_IMAGE_TAG }}
|
||||
load: true
|
||||
# Shared scope across matrix entries so the cache primed by this build
|
||||
# is reused by everything that runs this workflow on the same SHA (or
|
||||
# later runs whose Dockerfile + dist tarball haven't changed).
|
||||
cache-from: type=gha,scope=jpw-server-image
|
||||
cache-to: type=gha,scope=jpw-server-image,mode=max
|
||||
|
||||
- name: Export server image as artifact
|
||||
run: |
|
||||
docker save "${SERVER_IMAGE_TAG}" -o /tmp/jpw-server-image.tar
|
||||
ls -lh /tmp/jpw-server-image.tar
|
||||
|
||||
- name: Upload server image artifact
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: jpw-server-image-${{ github.run_id }}
|
||||
path: /tmp/jpw-server-image.tar
|
||||
retention-days: 1
|
||||
|
||||
ui-it-nightly:
|
||||
needs: build-image
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 90
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
searchEngine: [opensearch, elasticsearch]
|
||||
steps:
|
||||
- name: Free Disk Space (Ubuntu)
|
||||
uses: jlumbroso/free-disk-space@main
|
||||
with:
|
||||
tool-cache: true
|
||||
android: true
|
||||
dotnet: true
|
||||
haskell: true
|
||||
large-packages: true
|
||||
docker-images: false
|
||||
swap-storage: true
|
||||
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
ref: ${{ github.event.pull_request.head.sha || (github.event_name == 'workflow_dispatch' && github.ref || 'main') }}
|
||||
|
||||
- name: Set up JDK 21
|
||||
uses: actions/setup-java@v4
|
||||
with:
|
||||
java-version: '21'
|
||||
distribution: 'temurin'
|
||||
|
||||
- name: Cache Maven dependencies
|
||||
uses: actions/cache@v4
|
||||
with:
|
||||
path: ~/.m2
|
||||
key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}
|
||||
restore-keys: |
|
||||
${{ runner.os }}-maven-
|
||||
|
||||
- name: Install Ubuntu dependencies
|
||||
run: |
|
||||
sudo apt-get update
|
||||
sudo apt-get install -y unixodbc-dev python3-venv librdkafka-dev gcc libsasl2-dev build-essential libssl-dev libffi-dev \
|
||||
libevent-dev jq
|
||||
sudo make install_antlr_cli
|
||||
|
||||
- name: Add /etc/hosts entry for mock OIDC server
|
||||
# The SSO test infrastructure (MockOidcServer) needs `om-mock-idp` to resolve to
|
||||
# loopback on the host so the same URL works inside the Docker network and from
|
||||
# the host-side Playwright browser, keeping the issued tokens' `iss` claim
|
||||
# consistent across actors.
|
||||
run: echo "127.0.0.1 om-mock-idp" | sudo tee -a /etc/hosts
|
||||
|
||||
- name: Build dependencies for integration-tests
|
||||
# Maven cache primed by build-image makes this a fast restore-only invocation.
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
run: mvn -DskipTests install -pl :openmetadata-integration-tests -am
|
||||
|
||||
- name: Download server image artifact
|
||||
uses: actions/download-artifact@v4
|
||||
with:
|
||||
name: jpw-server-image-${{ github.run_id }}
|
||||
path: /tmp
|
||||
|
||||
- name: Load server image into local Docker
|
||||
run: |
|
||||
docker load -i /tmp/jpw-server-image.tar
|
||||
docker image ls "${SERVER_IMAGE_TAG}"
|
||||
|
||||
- name: Install Playwright browsers
|
||||
run: |
|
||||
mvn -pl :openmetadata-integration-tests dependency:build-classpath -Dmdep.outputFile=/tmp/cp.txt -q
|
||||
java -cp "$(cat /tmp/cp.txt)" com.microsoft.playwright.CLI install --with-deps chromium
|
||||
|
||||
- name: Free build artifacts
|
||||
run: |
|
||||
rm -rf openmetadata-service/target/lib openmetadata-service/target/classes
|
||||
rm -rf openmetadata-spec/target openmetadata-sdk/target common/target
|
||||
rm -rf openmetadata-shaded-deps/*/target
|
||||
df -h /
|
||||
|
||||
- name: Run UI integration tests (${{ matrix.searchEngine }})
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
# OM_TEST_IMAGE makes ContainerizedServer skip its in-test `docker build`
|
||||
# and reuse the image artifact loaded above. Cuts ~10-90s per launch.
|
||||
OM_TEST_IMAGE: ${{ env.SERVER_IMAGE_TAG }}
|
||||
# UiSessionExtension reads PW_VIDEO and records every test's BrowserContext
|
||||
# to target/playwright-videos when true. Kept off locally; on in CI for triage.
|
||||
PW_VIDEO: 'true'
|
||||
run: |
|
||||
# Nightly bumps the per-test cohort sizes up to the historical stress numbers.
|
||||
# PR runs use the smaller defaults baked into each test so `mvn verify` is fast.
|
||||
# Sizes are workflow_dispatch inputs (defaults below) so they're tunable per run.
|
||||
# Embedded nightly recreates a fresh stack each run, so data never persists — these
|
||||
# small-footprint suites always create on the fly (jpw.data.mode=ingest). Static/ensure
|
||||
# reuse is only for the persistent external cluster (see java-playwright-external.yml).
|
||||
STRESS_PROPS="-Djpw.simpleReindex.tables=${{ github.event.inputs.simpleReindexTables || '5000' }} \
|
||||
-Djpw.simpleReindex.topics=${{ github.event.inputs.simpleReindexTopics || '1500' }} \
|
||||
-Djpw.simpleReindex.dashboards=${{ github.event.inputs.simpleReindexDashboards || '1500' }} \
|
||||
-Djpw.simpleReindex.pipelines=${{ github.event.inputs.simpleReindexPipelines || '2000' }} \
|
||||
-Djpw.searchAvailable.tables=${{ github.event.inputs.searchAvailableTables || '5000' }} \
|
||||
-Djpw.data.mode=ingest \
|
||||
-Djpw.reindex.timeoutMin=${{ github.event.inputs.reindexTimeoutMin || '60' }}"
|
||||
# Opt-in for SSO bootstrap UIITs (second OM lifecycle); default off.
|
||||
if [ "${{ github.event.inputs.includeSsoBootstrap }}" = "true" ]; then
|
||||
SSO_PROPS="-Dui.it.excludedGroups="
|
||||
else
|
||||
SSO_PROPS=""
|
||||
fi
|
||||
if [ "${{ matrix.searchEngine }}" = "elasticsearch" ]; then
|
||||
mvn verify -P ui-it -pl :openmetadata-integration-tests $STRESS_PROPS $SSO_PROPS \
|
||||
-DsearchType=elasticsearch \
|
||||
-DsearchImage=docker.elastic.co/elasticsearch/elasticsearch:9.3.0
|
||||
else
|
||||
mvn verify -P ui-it -pl :openmetadata-integration-tests $STRESS_PROPS $SSO_PROPS
|
||||
fi
|
||||
|
||||
- name: Upload Playwright traces
|
||||
if: ${{ always() }}
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: playwright-traces-${{ matrix.searchEngine }}-${{ github.run_id }}
|
||||
path: openmetadata-integration-tests/target/playwright-traces
|
||||
if-no-files-found: ignore
|
||||
retention-days: 14
|
||||
|
||||
- name: Upload Playwright videos
|
||||
if: ${{ always() }}
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: playwright-videos-${{ matrix.searchEngine }}-${{ github.run_id }}
|
||||
path: openmetadata-integration-tests/target/playwright-videos
|
||||
if-no-files-found: ignore
|
||||
retention-days: 14
|
||||
|
||||
- name: Upload Failsafe Reports
|
||||
if: ${{ always() }}
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: failsafe-reports-${{ matrix.searchEngine }}-${{ github.run_id }}
|
||||
path: openmetadata-integration-tests/target/failsafe-reports
|
||||
if-no-files-found: ignore
|
||||
retention-days: 14
|
||||
|
||||
- name: Publish Test Report
|
||||
id: report
|
||||
if: ${{ always() }}
|
||||
uses: scacap/action-surefire-report@v1
|
||||
with:
|
||||
github_token: ${{ secrets.GITHUB_TOKEN }}
|
||||
# mvn verify already fails the job on test failures. This step just
|
||||
# publishes the check run with per-test details; it shouldn't itself
|
||||
# gate the job conclusion (otherwise a flake here breaks Slack).
|
||||
fail_on_test_failures: false
|
||||
report_paths: 'openmetadata-integration-tests/target/failsafe-reports/TEST-*.xml'
|
||||
|
||||
- name: Slack notification
|
||||
if: ${{ always() && github.event_name != 'pull_request' && env.SLACK_JAVA_PLAYWRIGHT_URL != '' }}
|
||||
uses: slackapi/slack-github-action@v2.0.0
|
||||
with:
|
||||
webhook: ${{ env.SLACK_JAVA_PLAYWRIGHT_URL }}
|
||||
webhook-type: incoming-webhook
|
||||
payload: |
|
||||
text: "${{ job.status == 'success' && ':white_check_mark:' || ':fire:' }} Java Playwright Nightly (${{ matrix.searchEngine }}): ${{ job.status }}\nRef: ${{ github.ref_name }}\nLogs: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"
|
||||
env:
|
||||
SLACK_JAVA_PLAYWRIGHT_URL: ${{ secrets.SLACK_JAVA_PLAYWRIGHT_URL }}
|
||||
|
||||
search-it-nightly:
|
||||
# Server-global destructive search ITs (tests/search/*IT.java): full reindex with
|
||||
# recreateIndex, alias swaps, and ES container pause. They mutate cluster-wide state, so
|
||||
# the default PR profiles exclude them; here the `search-it` Maven profile runs them
|
||||
# serially on their own embedded-bootstrap server (testcontainers). That bootstrap is
|
||||
# independent of the baked image, so this job does not need build-image.
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 90
|
||||
steps:
|
||||
- name: Free Disk Space (Ubuntu)
|
||||
uses: jlumbroso/free-disk-space@main
|
||||
with:
|
||||
tool-cache: true
|
||||
android: true
|
||||
dotnet: true
|
||||
haskell: true
|
||||
large-packages: true
|
||||
docker-images: false
|
||||
swap-storage: true
|
||||
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
ref: ${{ github.event.pull_request.head.sha || (github.event_name == 'workflow_dispatch' && github.ref || 'main') }}
|
||||
|
||||
- name: Set up JDK 21
|
||||
uses: actions/setup-java@v4
|
||||
with:
|
||||
java-version: '21'
|
||||
distribution: 'temurin'
|
||||
|
||||
- name: Cache Maven dependencies
|
||||
uses: actions/cache@v4
|
||||
with:
|
||||
path: ~/.m2
|
||||
key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}
|
||||
restore-keys: |
|
||||
${{ runner.os }}-maven-
|
||||
|
||||
- name: Install Ubuntu dependencies
|
||||
run: |
|
||||
sudo apt-get update
|
||||
sudo apt-get install -y unixodbc-dev python3-venv librdkafka-dev gcc libsasl2-dev build-essential libssl-dev libffi-dev \
|
||||
libevent-dev jq
|
||||
sudo make install_antlr_cli
|
||||
|
||||
- name: Build dependencies for integration-tests
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
run: mvn -DskipTests install -pl :openmetadata-integration-tests -am
|
||||
|
||||
- name: Run isolated search ITs (postgres + opensearch)
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
# Embedded bootstrap = fresh stack each run; these search ITs create their cohorts on the
|
||||
# fly (jpw.data.mode=ingest). Static/ensure reuse is for the external cluster only.
|
||||
run: mvn verify -P search-it -pl :openmetadata-integration-tests -DdatabaseType=postgres -DsearchType=opensearch -Djpw.data.mode=ingest -Djpw.reindex.timeoutMin=${{ github.event.inputs.reindexTimeoutMin || '60' }}
|
||||
|
||||
- name: Upload Failsafe Reports
|
||||
if: ${{ always() }}
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: failsafe-reports-search-it-${{ github.run_id }}
|
||||
path: openmetadata-integration-tests/target/failsafe-reports
|
||||
if-no-files-found: ignore
|
||||
retention-days: 14
|
||||
|
||||
- name: Publish Test Report
|
||||
if: ${{ always() }}
|
||||
uses: scacap/action-surefire-report@v1
|
||||
with:
|
||||
github_token: ${{ secrets.GITHUB_TOKEN }}
|
||||
fail_on_test_failures: false
|
||||
report_paths: 'openmetadata-integration-tests/target/failsafe-reports/TEST-*.xml'
|
||||
|
||||
- name: Slack notification
|
||||
if: ${{ always() && github.event_name != 'pull_request' && env.SLACK_JAVA_PLAYWRIGHT_URL != '' }}
|
||||
uses: slackapi/slack-github-action@v2.0.0
|
||||
with:
|
||||
webhook: ${{ env.SLACK_JAVA_PLAYWRIGHT_URL }}
|
||||
webhook-type: incoming-webhook
|
||||
payload: |
|
||||
text: "${{ job.status == 'success' && ':white_check_mark:' || ':fire:' }} Java Search ITs Nightly: ${{ job.status }}\nRef: ${{ github.ref_name }}\nLogs: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"
|
||||
env:
|
||||
SLACK_JAVA_PLAYWRIGHT_URL: ${{ secrets.SLACK_JAVA_PLAYWRIGHT_URL }}
|
||||
@@ -0,0 +1,25 @@
|
||||
name: Label connector bug
|
||||
|
||||
on:
|
||||
issues:
|
||||
types: [opened, edited]
|
||||
|
||||
permissions:
|
||||
issues: write
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
label:
|
||||
if: contains(github.event.issue.labels.*.name, 'Ingestion')
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
sparse-checkout: .github/scripts
|
||||
sparse-checkout-cone-mode: false
|
||||
- uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: '3.11'
|
||||
- run: python .github/scripts/label_connector.py
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
@@ -0,0 +1,93 @@
|
||||
# Copyright 2021 Collate
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
name: Maven Collate Tests
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
push:
|
||||
branches:
|
||||
- main
|
||||
paths:
|
||||
- "openmetadata-service/**"
|
||||
- "openmetadata-ui/**"
|
||||
- "!openmetadata-ui/src/main/resources/ui/playwright/doc-generator/**"
|
||||
- "!openmetadata-ui/src/main/resources/ui/playwright/docs/**"
|
||||
- "openmetadata-spec/src/main/resources/json/schema/**"
|
||||
- "openmetadata-dist/**"
|
||||
- "openmetadata-clients/**"
|
||||
- "openmetadata-sdk/**"
|
||||
- "openmetadata-integration-tests/**"
|
||||
- "openmetadata-mcp/**"
|
||||
- "common/**"
|
||||
- "pom.xml"
|
||||
- "yarn.lock"
|
||||
- "Makefile"
|
||||
- "bootstrap/**"
|
||||
pull_request_target:
|
||||
types: [opened, synchronize, labeled, ready_for_review]
|
||||
paths:
|
||||
- "openmetadata-service/**"
|
||||
- "openmetadata-ui/src/main/resources/ui/src/**"
|
||||
- "!openmetadata-ui/src/main/resources/ui/src/test/**"
|
||||
- "!openmetadata-ui/src/main/resources/ui/src/**/*.test.ts"
|
||||
- "!openmetadata-ui/src/main/resources/ui/src/**/*.test.tsx"
|
||||
- "openmetadata-spec/src/main/resources/json/schema/**"
|
||||
- "openmetadata-integration-tests/**"
|
||||
- "openmetadata-mcp/**"
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
checks: write
|
||||
|
||||
concurrency:
|
||||
group: maven-build-collate-${{ github.event.pull_request.number || github.run_id }}
|
||||
cancel-in-progress: ${{ github.event_name != 'pull_request_target' || github.event.action != 'labeled' || github.event.label.name == 'safe to test' }}
|
||||
jobs:
|
||||
maven-collate-ci:
|
||||
runs-on: ubuntu-latest
|
||||
if: ${{ !github.event.pull_request.draft && (github.event_name != 'pull_request_target' || github.event.action != 'labeled' || github.event.label.name == 'safe to test') }}
|
||||
steps:
|
||||
- name: Wait for the labeler
|
||||
uses: lewagon/wait-on-check-action@v1.7.0
|
||||
if: ${{ github.event_name == 'pull_request_target' }}
|
||||
with:
|
||||
ref: ${{ github.event.pull_request.head.sha }}
|
||||
check-name: Team Label
|
||||
repo-token: ${{ secrets.GITHUB_TOKEN }}
|
||||
wait-interval: 90
|
||||
|
||||
- name: Verify PR labels
|
||||
uses: jesusvasquez333/verify-pr-label-action@v1.4.0
|
||||
if: ${{ github.event_name == 'pull_request_target' }}
|
||||
with:
|
||||
github-token: "${{ secrets.GITHUB_TOKEN }}"
|
||||
valid-labels: "safe to test"
|
||||
pull-request-number: "${{ github.event.pull_request.number }}"
|
||||
disable-reviews: true # To not auto approve changes
|
||||
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
ref: ${{ github.event.pull_request.head.sha }}
|
||||
|
||||
- name: Trigger Collate build & wait
|
||||
uses: the-actions-org/workflow-dispatch@v4
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
SHA: ${{ github.event_name == 'push' && github.event.after || github.event.pull_request.head.sha }}
|
||||
with:
|
||||
workflow: OpenMetadata Collate Test
|
||||
ref: main
|
||||
repo: open-metadata/openmetadata-collate
|
||||
token: ${{ secrets.COLLATE_PAT }}
|
||||
wait-for-completion: true
|
||||
inputs: '{ "sha": "${{ env.SHA }}", "event": "${{ github.event_name }}" }'
|
||||
@@ -0,0 +1,92 @@
|
||||
# Copyright 2021 Collate
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
name: Maven SonarCloud CI
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
pull_request_target:
|
||||
types: [labeled, opened, synchronize, reopened, ready_for_review]
|
||||
paths:
|
||||
- "openmetadata-service/**"
|
||||
- "openmetadata-spec/src/main/resources/json/schema/**"
|
||||
- "openmetadata-dist/**"
|
||||
- "openmetadata-clients/**"
|
||||
- "openmetadata-sdk/**"
|
||||
- "common/**"
|
||||
- "pom.xml"
|
||||
- "yarn.lock"
|
||||
- "Makefile"
|
||||
- "bootstrap/**"
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
checks: write
|
||||
|
||||
concurrency:
|
||||
group: maven-sonar-build-${{ github.event.pull_request.number || github.run_id }}
|
||||
cancel-in-progress: ${{ github.event_name != 'pull_request_target' || github.event.action != 'labeled' || github.event.label.name == 'safe to test' }}
|
||||
jobs:
|
||||
maven-sonarcloud-ci:
|
||||
runs-on: ubuntu-latest
|
||||
if: ${{ !github.event.pull_request.draft && (github.event_name != 'pull_request_target' || github.event.action != 'labeled' || github.event.label.name == 'safe to test') }}
|
||||
steps:
|
||||
- name: Wait for the labeler
|
||||
uses: lewagon/wait-on-check-action@v1.7.0
|
||||
if: ${{ github.event_name == 'pull_request_target' }}
|
||||
with:
|
||||
ref: ${{ github.event.pull_request.head.sha }}
|
||||
check-name: Team Label
|
||||
repo-token: ${{ secrets.GITHUB_TOKEN }}
|
||||
wait-interval: 90
|
||||
|
||||
- name: Verify PR labels
|
||||
uses: jesusvasquez333/verify-pr-label-action@v1.4.0
|
||||
if: ${{ github.event_name == 'pull_request_target' }}
|
||||
with:
|
||||
github-token: "${{ secrets.GITHUB_TOKEN }}"
|
||||
valid-labels: "safe to test"
|
||||
pull-request-number: "${{ github.event.pull_request.number }}"
|
||||
disable-reviews: true # To not auto approve changes
|
||||
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
ref: ${{ github.event.pull_request.head.sha }}
|
||||
|
||||
- name: Cache Maven dependencies
|
||||
id: cache-output
|
||||
uses: actions/cache@v4
|
||||
with:
|
||||
path: ~/.m2
|
||||
key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}
|
||||
restore-keys: |
|
||||
${{ runner.os }}-maven-
|
||||
|
||||
- name: Set up JDK 21
|
||||
if: steps.cache-output.outputs.exit-code == 0
|
||||
uses: actions/setup-java@v4
|
||||
with:
|
||||
java-version: "21"
|
||||
distribution: "temurin"
|
||||
|
||||
- name: Install Ubuntu dependencies
|
||||
if: steps.cache-output.outputs.exit-code == 0
|
||||
run: |
|
||||
sudo apt-get update
|
||||
sudo apt-get install -y unixodbc-dev python3-venv librdkafka-dev gcc libsasl2-dev build-essential libssl-dev libffi-dev \
|
||||
librdkafka-dev unixodbc-dev libevent-dev jq
|
||||
sudo make install_antlr_cli
|
||||
|
||||
- name: Build with Maven
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
run: mvn -Pstatic-code-analysis clean verify -DskipTests -am
|
||||
@@ -0,0 +1,66 @@
|
||||
# Copyright 2021 Collate
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
name: monitor-slack-link
|
||||
on:
|
||||
schedule:
|
||||
# Run every 2 hours
|
||||
- cron: '0 */2 * * *'
|
||||
workflow_dispatch:
|
||||
|
||||
|
||||
# Grant least privilege permissions needed
|
||||
permissions:
|
||||
contents: read # Needed for checkout and reading requirements.txt
|
||||
|
||||
jobs:
|
||||
monitor-slack-link:
|
||||
runs-on: ubuntu-latest
|
||||
strategy:
|
||||
fail-fast: false # Ensure all matrix jobs run even if one fails
|
||||
# Only run specific matrix job when manually triggered with selection
|
||||
steps:
|
||||
# Step 1: Checkout repository code
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v4
|
||||
|
||||
# Step 2: Set up Python environment with caching
|
||||
- name: Set up Python 3.9 # Or consider a newer version like 3.11/3.12
|
||||
uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: 3.9 # Or e.g., '3.11'
|
||||
cache: 'pip'
|
||||
|
||||
# Step 3: Install dependencies
|
||||
- name: Install Dependencies
|
||||
run: |
|
||||
python -m venv env
|
||||
source env/bin/activate
|
||||
pip install --upgrade pip playwright slack_sdk==3.35.0
|
||||
|
||||
# Step 4: Install Playwright browsers
|
||||
- name: Install Playwright Browsers
|
||||
run: |
|
||||
source env/bin/activate
|
||||
playwright install chromium --with-deps
|
||||
|
||||
# Step 5: Run the monitoring script
|
||||
- name: Monitor Slack Link
|
||||
id: monitor
|
||||
env:
|
||||
PYTHONUNBUFFERED: "1"
|
||||
SLACK_WEBHOOK_URL: ${{ secrets.SLACK_MONITOR_SLACK_WEBHOOK }}
|
||||
GITHUB_SERVER_URL: ${{ github.server_url }}
|
||||
GITHUB_REPOSITORY: ${{ github.repository }}
|
||||
GITHUB_RUN_ID: ${{ github.run_id }}
|
||||
run: |
|
||||
source env/bin/activate
|
||||
python scripts/slack-link-monitor.py
|
||||
@@ -0,0 +1,234 @@
|
||||
# Copyright 2021 Collate
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
# This workflow executes end-to-end (e2e) tests using Playwright with MySQL as the database.
|
||||
# For more information see: https://github.com/actions/setup-java/blob/main/docs/advanced-usage.md#apache-maven-with-a-settings-path
|
||||
|
||||
name: MySQL Playwright E2E Tests
|
||||
on:
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
branch:
|
||||
description: "Branch to run tests on"
|
||||
required: true
|
||||
type: string
|
||||
default: "main"
|
||||
send_slack_notification:
|
||||
description: "Send Slack notification with test results"
|
||||
required: false
|
||||
type: boolean
|
||||
default: false
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
concurrency:
|
||||
group: pw-ci-mysql-branch-${{ inputs.branch }}
|
||||
cancel-in-progress: true
|
||||
|
||||
jobs:
|
||||
pw-ci-mysql-branch:
|
||||
runs-on: ubuntu-latest
|
||||
environment: test
|
||||
env:
|
||||
# Playwright logs the admin user in many times (performAdminLogin per test, parallel
|
||||
# workers, retries). The production default of 5 active sessions per user would evict the
|
||||
# long-lived storageState session the page fixtures rely on and 401 every request. Raise
|
||||
# the cap for E2E only; docker compose reads this for ${AUTHENTICATION_MAX_ACTIVE_SESSIONS_PER_USER:-5}.
|
||||
AUTHENTICATION_MAX_ACTIVE_SESSIONS_PER_USER: "10000"
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
shardIndex: [1, 2, 3, 4, 5, 6, 7]
|
||||
shardTotal: [7]
|
||||
steps:
|
||||
- name: Free Disk Space (Ubuntu)
|
||||
uses: jlumbroso/free-disk-space@main
|
||||
with:
|
||||
tool-cache: false
|
||||
android: true
|
||||
dotnet: true
|
||||
haskell: true
|
||||
large-packages: false
|
||||
swap-storage: true
|
||||
docker-images: false
|
||||
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
ref: ${{ inputs.branch }}
|
||||
|
||||
- name: Cache Maven Dependencies
|
||||
id: cache-output
|
||||
uses: actions/cache@v4
|
||||
with:
|
||||
path: ~/.m2
|
||||
key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}
|
||||
restore-keys: |
|
||||
${{ runner.os }}-maven-
|
||||
|
||||
- name: Setup Openmetadata Test Environment
|
||||
uses: ./.github/actions/setup-openmetadata-test-environment
|
||||
with:
|
||||
python-version: "3.10"
|
||||
args: "-d mysql"
|
||||
ingestion_dependency: "playwright"
|
||||
|
||||
- name: Install dependencies
|
||||
working-directory: openmetadata-ui/src/main/resources/ui/
|
||||
run: yarn --ignore-scripts --frozen-lockfile
|
||||
- name: Install Playwright Browsers
|
||||
run: npx playwright@1.57.0 install chromium --with-deps
|
||||
- name: Run Playwright tests
|
||||
working-directory: openmetadata-ui/src/main/resources/ui/
|
||||
run: |
|
||||
if [ "${{ matrix.shardIndex }}" -eq "1" ]; then
|
||||
echo "🔹 Running DataAssetRules-only tests on shard 1"
|
||||
|
||||
# The testMatch pattern ensures only DataAssetRules*.spec.ts files run
|
||||
npx playwright test \
|
||||
--project=setup \
|
||||
--project=DataAssetRulesEnabled \
|
||||
--project=DataAssetRulesDisabled
|
||||
|
||||
elif [ "${{ matrix.shardIndex }}" -eq "2" ]; then
|
||||
echo "🔹 Running search relevance tests on shard 2"
|
||||
|
||||
npx playwright test \
|
||||
--project=search-nightly \
|
||||
playwright/e2e/Search/SearchRelevance.spec.ts \
|
||||
--workers=1
|
||||
|
||||
else
|
||||
# Shards 3-7 handle chromium tests (5 shards total)
|
||||
CHROMIUM_SHARD=$(( ${{ matrix.shardIndex }} - 2 ))
|
||||
echo "🔹 Running all tests (excluding DataAssetRules) on chromium shard ${CHROMIUM_SHARD}/5"
|
||||
npx playwright test \
|
||||
--project=chromium \
|
||||
--grep-invert @dataAssetRules \
|
||||
--shard=${CHROMIUM_SHARD}/5
|
||||
fi
|
||||
|
||||
env:
|
||||
PLAYWRIGHT_IS_OSS: true
|
||||
PLAYWRIGHT_SNOWFLAKE_USERNAME: ${{ secrets.TEST_SNOWFLAKE_USERNAME }}
|
||||
PLAYWRIGHT_SNOWFLAKE_PASSWORD: ${{ secrets.TEST_SNOWFLAKE_PASSWORD }}
|
||||
PLAYWRIGHT_SNOWFLAKE_ACCOUNT: ${{ secrets.TEST_SNOWFLAKE_ACCOUNT }}
|
||||
PLAYWRIGHT_SNOWFLAKE_DATABASE: ${{ secrets.TEST_SNOWFLAKE_DATABASE }}
|
||||
PLAYWRIGHT_SNOWFLAKE_WAREHOUSE: ${{ secrets.TEST_SNOWFLAKE_WAREHOUSE }}
|
||||
PLAYWRIGHT_PROJECT_ID: ${{ steps.cypress-project-id.outputs.CYPRESS_PROJECT_ID }}
|
||||
PLAYWRIGHT_BQ_PRIVATE_KEY: ${{ secrets.TEST_BQ_PRIVATE_KEY }}
|
||||
PLAYWRIGHT_BQ_PROJECT_ID: ${{ secrets.PLAYWRIGHT_BQ_PROJECT_ID }}
|
||||
PLAYWRIGHT_BQ_PRIVATE_KEY_ID: ${{ secrets.TEST_BQ_PRIVATE_KEY_ID }}
|
||||
PLAYWRIGHT_BQ_PROJECT_ID_TAXONOMY: ${{ secrets.TEST_BQ_PROJECT_ID_TAXONOMY }}
|
||||
PLAYWRIGHT_BQ_CLIENT_EMAIL: ${{ secrets.TEST_BQ_CLIENT_EMAIL }}
|
||||
PLAYWRIGHT_BQ_CLIENT_ID: ${{ secrets.TEST_BQ_CLIENT_ID }}
|
||||
PLAYWRIGHT_REDSHIFT_HOST: ${{ secrets.E2E_REDSHIFT_HOST_PORT }}
|
||||
PLAYWRIGHT_REDSHIFT_USERNAME: ${{ secrets.E2E_REDSHIFT_USERNAME }}
|
||||
PLAYWRIGHT_REDSHIFT_PASSWORD: ${{ secrets.E2E_REDSHIFT_PASSWORD }}
|
||||
PLAYWRIGHT_REDSHIFT_DATABASE: ${{ secrets.TEST_REDSHIFT_DATABASE }}
|
||||
PLAYWRIGHT_METABASE_USERNAME: ${{ secrets.TEST_METABASE_USERNAME }}
|
||||
PLAYWRIGHT_METABASE_PASSWORD: ${{ secrets.TEST_METABASE_PASSWORD }}
|
||||
PLAYWRIGHT_METABASE_DB_SERVICE_NAME: ${{ secrets.TEST_METABASE_DB_SERVICE_NAME }}
|
||||
PLAYWRIGHT_METABASE_HOST_PORT: ${{ secrets.TEST_METABASE_HOST_PORT }}
|
||||
PLAYWRIGHT_SUPERSET_USERNAME: ${{ secrets.TEST_SUPERSET_USERNAME }}
|
||||
PLAYWRIGHT_SUPERSET_PASSWORD: ${{ secrets.TEST_SUPERSET_PASSWORD }}
|
||||
PLAYWRIGHT_SUPERSET_HOST_PORT: ${{ secrets.TEST_SUPERSET_HOST_PORT }}
|
||||
PLAYWRIGHT_KAFKA_BOOTSTRAP_SERVERS: ${{ secrets.TEST_KAFKA_BOOTSTRAP_SERVERS }}
|
||||
PLAYWRIGHT_KAFKA_SCHEMA_REGISTRY_URL: ${{ secrets.TEST_KAFKA_SCHEMA_REGISTRY_URL }}
|
||||
PLAYWRIGHT_GLUE_ACCESS_KEY: ${{ secrets.TEST_GLUE_ACCESS_KEY }}
|
||||
PLAYWRIGHT_GLUE_SECRET_KEY: ${{ secrets.TEST_GLUE_SECRET_KEY }}
|
||||
PLAYWRIGHT_GLUE_AWS_REGION: ${{ secrets.TEST_GLUE_AWS_REGION }}
|
||||
PLAYWRIGHT_GLUE_ENDPOINT: ${{ secrets.TEST_GLUE_ENDPOINT }}
|
||||
PLAYWRIGHT_GLUE_STORAGE_SERVICE: ${{ secrets.TEST_GLUE_STORAGE_SERVICE }}
|
||||
PLAYWRIGHT_MYSQL_USERNAME: ${{ secrets.TEST_MYSQL_USERNAME }}
|
||||
PLAYWRIGHT_MYSQL_PASSWORD: ${{ secrets.TEST_MYSQL_PASSWORD }}
|
||||
PLAYWRIGHT_MYSQL_HOST_PORT: ${{ secrets.TEST_MYSQL_HOST_PORT }}
|
||||
PLAYWRIGHT_MYSQL_DATABASE_SCHEMA: ${{ secrets.TEST_MYSQL_DATABASE_SCHEMA }}
|
||||
PLAYWRIGHT_POSTGRES_USERNAME: ${{ secrets.TEST_POSTGRES_USERNAME }}
|
||||
PLAYWRIGHT_POSTGRES_PASSWORD: ${{ secrets.TEST_POSTGRES_PASSWORD }}
|
||||
PLAYWRIGHT_POSTGRES_HOST_PORT: ${{ secrets.TEST_POSTGRES_HOST_PORT }}
|
||||
PLAYWRIGHT_POSTGRES_DATABASE: ${{ secrets.TEST_POSTGRES_DATABASE }}
|
||||
PLAYWRIGHT_AIRFLOW_HOST_PORT: ${{ secrets.TEST_AIRFLOW_HOST_PORT }}
|
||||
PLAYWRIGHT_ML_MODEL_TRACKING_URI: ${{ secrets.TEST_ML_MODEL_TRACKING_URI }}
|
||||
PLAYWRIGHT_ML_MODEL_REGISTRY_URI: ${{ secrets.TEST_ML_MODEL_REGISTRY_URI }}
|
||||
PLAYWRIGHT_S3_STORAGE_ACCESS_KEY_ID: ${{ secrets.TEST_S3_STORAGE_ACCESS_KEY_ID }}
|
||||
PLAYWRIGHT_S3_STORAGE_SECRET_ACCESS_KEY: ${{ secrets.TEST_S3_STORAGE_SECRET_ACCESS_KEY }}
|
||||
PLAYWRIGHT_S3_STORAGE_END_POINT_URL: ${{ secrets.TEST_S3_STORAGE_END_POINT_URL }}
|
||||
|
||||
# Recommended: pass the GitHub token lets this action correctly
|
||||
# determine the unique run id necessary to re-run the checks
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
- name: Upload blob report to GitHub Actions Artifacts
|
||||
if: ${{ !cancelled() }}
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: blob-report-${{ matrix.shardIndex }}
|
||||
path: openmetadata-ui/src/main/resources/ui/blob-report
|
||||
retention-days: 1
|
||||
|
||||
- name: Upload HTML report to GitHub Actions Artifacts
|
||||
if: ${{ !cancelled() }}
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: playwright-HTML-report-${{ matrix.shardIndex }}
|
||||
path: openmetadata-ui/src/main/resources/ui/playwright/output/playwright-report
|
||||
retention-days: 1
|
||||
compression-level: 9
|
||||
|
||||
- name: Clean Up
|
||||
run: |
|
||||
cd ./docker/development
|
||||
docker compose down --remove-orphans
|
||||
sudo rm -rf ${PWD}/docker-volume
|
||||
|
||||
merge-reports:
|
||||
needs: pw-ci-mysql-branch
|
||||
runs-on: ubuntu-latest
|
||||
if: ${{ !failure() || !cancelled() }}
|
||||
env:
|
||||
# Recommended: pass the GitHub token lets this action correctly
|
||||
# determine the unique run id necessary to re-run the checks
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
SLACK_BOT_USER_OAUTH_TOKEN: ${{ secrets.E2E_SLACK_BOT_OAUTH_TOKEN }}
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
ref: ${{ inputs.branch }}
|
||||
|
||||
- name: Setup Node.js
|
||||
uses: actions/setup-node@v4
|
||||
with:
|
||||
node-version-file: "openmetadata-ui/src/main/resources/ui/.nvmrc"
|
||||
|
||||
- name: Download blob reports from GitHub Actions Artifacts
|
||||
uses: actions/download-artifact@v4
|
||||
with:
|
||||
path: openmetadata-ui/src/main/resources/ui/blob-reports
|
||||
pattern: blob-report-*
|
||||
merge-multiple: true
|
||||
|
||||
- name: Merge Reports
|
||||
working-directory: openmetadata-ui/src/main/resources/ui/
|
||||
run: |
|
||||
npx playwright merge-reports --reporter json ./blob-reports > merged_tests_results.json
|
||||
|
||||
- name: Generate Slack Notification
|
||||
if: ${{ inputs.send_slack_notification }}
|
||||
working-directory: openmetadata-ui/src/main/resources/ui/
|
||||
env:
|
||||
RUN_TITLE: "MySQL Test for OSS Release: ${{ inputs.branch }}"
|
||||
RUN_URL: "${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"
|
||||
SLACK_BOT_USER_OAUTH_TOKEN: ${{ secrets.E2E_SLACK_BOT_OAUTH_TOKEN }}
|
||||
run: |
|
||||
npx playwright-slack-report -c playwright/slack-cli.config.json -j merged_tests_results.json > slack_report.json
|
||||
@@ -0,0 +1,190 @@
|
||||
# Copyright 2021 Collate
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
name: OpenMetadata Service Unit Tests
|
||||
|
||||
on:
|
||||
merge_group:
|
||||
workflow_dispatch:
|
||||
push:
|
||||
branches:
|
||||
- main
|
||||
pull_request:
|
||||
types: [labeled, opened, synchronize, reopened, ready_for_review]
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
checks: write
|
||||
pull-requests: write
|
||||
|
||||
concurrency:
|
||||
group: openmetadata-service-unit-tests-${{ github.head_ref || github.run_id }}
|
||||
cancel-in-progress: ${{ github.event_name != 'pull_request' || github.event.action != 'labeled' || github.event.label.name == 'safe to test' }}
|
||||
|
||||
jobs:
|
||||
# Detect whether relevant paths changed. When no Java service files
|
||||
# are modified the downstream jobs are skipped via their `if` condition.
|
||||
# A job skipped by `if` reports as "Success", so required checks still pass.
|
||||
# This replaces the old openmetadata-service-unit-tests-skip.yml companion workflow.
|
||||
changes:
|
||||
name: Detect Changes
|
||||
runs-on: ubuntu-latest
|
||||
if: ${{ (github.event_name != 'pull_request' || !github.event.pull_request.draft) && (github.event_name != 'pull_request' || github.event.action != 'labeled' || github.event.label.name == 'safe to test') }}
|
||||
outputs:
|
||||
java: ${{ github.event_name == 'workflow_dispatch' && 'true' || steps.filter.outputs.java }}
|
||||
k8s_operator: ${{ github.event_name == 'workflow_dispatch' && 'true' || steps.filter.outputs.k8s_operator }}
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v4
|
||||
if: ${{ github.event_name != 'workflow_dispatch' }}
|
||||
|
||||
- uses: dorny/paths-filter@v3
|
||||
id: filter
|
||||
if: ${{ github.event_name != 'workflow_dispatch' }}
|
||||
with:
|
||||
filters: |
|
||||
java:
|
||||
- '.github/workflows/openmetadata-service-unit-tests.yml'
|
||||
- 'openmetadata-service/**'
|
||||
- 'openmetadata-spec/**'
|
||||
- 'openmetadata-clients/**'
|
||||
- 'openmetadata-sdk/**'
|
||||
- 'common/**'
|
||||
- 'pom.xml'
|
||||
- 'Makefile'
|
||||
- 'bootstrap/**'
|
||||
k8s_operator:
|
||||
- 'openmetadata-k8s-operator/**'
|
||||
|
||||
# The openmetadata-service unit tests are pure JVM tests with no database
|
||||
# interaction (no testcontainers, no JDBC). The {mysql, postgresql} matrix used
|
||||
# to run the suite twice with different `-Pmysql` / `-Ppostgresql` profiles, but
|
||||
# those profiles are only defined in openmetadata-sdk/pom.xml and only affect
|
||||
# failsafe (integration) tests that aren't enabled in this workflow. Result:
|
||||
# both matrix jobs ran an identical surefire suite. DB-specific coverage
|
||||
# belongs in `openmetadata-integration-tests`, not here.
|
||||
openmetadata-service-unit-tests:
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 90
|
||||
needs: changes
|
||||
if: ${{ needs.changes.outputs.java == 'true' }}
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}
|
||||
|
||||
- name: Cache Maven dependencies
|
||||
uses: actions/cache@v4
|
||||
with:
|
||||
path: ~/.m2
|
||||
key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}
|
||||
restore-keys: |
|
||||
${{ runner.os }}-maven-
|
||||
|
||||
- name: Set up JDK 21
|
||||
uses: actions/setup-java@v4
|
||||
with:
|
||||
java-version: "21"
|
||||
distribution: "temurin"
|
||||
|
||||
- name: Install Ubuntu dependencies
|
||||
run: |
|
||||
sudo apt-get update
|
||||
sudo apt-get install -y unixodbc-dev python3-venv librdkafka-dev gcc libsasl2-dev build-essential libssl-dev libffi-dev \
|
||||
librdkafka-dev unixodbc-dev libevent-dev jq
|
||||
sudo make install_antlr_cli
|
||||
|
||||
- name: Run openmetadata-service unit tests
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
run: |
|
||||
mvn -B clean package -pl openmetadata-service -am \
|
||||
-Pstatic-code-analysis \
|
||||
-DfailIfNoTests=false \
|
||||
-Dsonar.skip=true
|
||||
|
||||
- name: Upload surefire reports
|
||||
if: ${{ failure() && hashFiles('openmetadata-service/target/surefire-reports/TEST-*.xml') != '' }}
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: openmetadata-service-surefire-reports
|
||||
path: openmetadata-service/target/surefire-reports/
|
||||
|
||||
- name: Publish Test Report
|
||||
if: ${{ always() && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository) && hashFiles('openmetadata-service/target/surefire-reports/TEST-*.xml') != '' }}
|
||||
uses: scacap/action-surefire-report@v1
|
||||
with:
|
||||
github_token: ${{ secrets.GITHUB_TOKEN }}
|
||||
fail_on_test_failures: true
|
||||
report_paths: "openmetadata-service/target/surefire-reports/TEST-*.xml"
|
||||
check_name: "Test Report"
|
||||
|
||||
k8s_operator-unit-tests:
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 30
|
||||
needs: changes
|
||||
if: ${{ needs.changes.outputs.k8s_operator == 'true' }}
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}
|
||||
|
||||
- name: Cache Maven dependencies
|
||||
uses: actions/cache@v4
|
||||
with:
|
||||
path: ~/.m2
|
||||
key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}
|
||||
restore-keys: |
|
||||
${{ runner.os }}-maven-
|
||||
|
||||
- name: Set up JDK 21
|
||||
uses: actions/setup-java@v4
|
||||
with:
|
||||
java-version: "21"
|
||||
distribution: "temurin"
|
||||
|
||||
- name: Build k8s-operator dependencies
|
||||
run: |
|
||||
mvn -B clean install -pl openmetadata-k8s-operator -am -DskipTests
|
||||
|
||||
- name: Run k8s-operator unit tests
|
||||
run: |
|
||||
mvn -B test -pl openmetadata-k8s-operator
|
||||
|
||||
- name: Upload surefire reports
|
||||
if: ${{ failure() && hashFiles('openmetadata-k8s-operator/target/surefire-reports/TEST-*.xml') != '' }}
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: k8s-operator-surefire-reports
|
||||
path: openmetadata-k8s-operator/target/surefire-reports/
|
||||
|
||||
- name: Publish Test Report
|
||||
if: ${{ always() && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository) && hashFiles('openmetadata-k8s-operator/target/surefire-reports/TEST-*.xml') != '' }}
|
||||
uses: scacap/action-surefire-report@v1
|
||||
with:
|
||||
github_token: ${{ secrets.GITHUB_TOKEN }}
|
||||
fail_on_test_failures: true
|
||||
report_paths: "openmetadata-k8s-operator/target/surefire-reports/TEST-*.xml"
|
||||
check_name: "K8s Operator Test Report"
|
||||
|
||||
# Single required-check gate for branch protection.
|
||||
# Skipped (= "Success") when all test jobs pass or are legitimately skipped.
|
||||
# Runs and exits 1 only when a test job fails or is cancelled.
|
||||
# Set "OpenMetadata Service Unit Tests / openmetadata-service-unit-tests-status" as the sole required check for this workflow.
|
||||
openmetadata-service-unit-tests-status:
|
||||
name: openmetadata-service-unit-tests-status
|
||||
needs: [changes, openmetadata-service-unit-tests, k8s_operator-unit-tests]
|
||||
if: ${{ failure() || cancelled() }}
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- run: exit 1
|
||||
@@ -0,0 +1,68 @@
|
||||
name: Verify Playwright Documentation
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
paths:
|
||||
- 'openmetadata-ui/src/main/resources/ui/playwright/e2e/**/*.spec.ts'
|
||||
- 'openmetadata-ui/src/main/resources/ui/playwright/doc-generator/**'
|
||||
branches:
|
||||
- main
|
||||
|
||||
permissions:
|
||||
contents: write
|
||||
pull-requests: write
|
||||
|
||||
jobs:
|
||||
verify-docs:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v3
|
||||
|
||||
- name: Setup Node.js
|
||||
uses: actions/setup-node@v3
|
||||
with:
|
||||
node-version-file: 'openmetadata-ui/src/main/resources/ui/.nvmrc'
|
||||
cache: 'yarn'
|
||||
cache-dependency-path: 'openmetadata-ui/src/main/resources/ui/yarn.lock'
|
||||
|
||||
- name: Install Dependencies
|
||||
working-directory: openmetadata-ui/src/main/resources/ui
|
||||
run: yarn install --frozen-lockfile --ignore-scripts
|
||||
|
||||
- name: Install Playwright Browsers
|
||||
# We need browsers installed for 'playwright test --list' to work in some versions,
|
||||
# although strictly speaking just listing shouldn't require binaries if configured right.
|
||||
# Adding just in case.
|
||||
working-directory: openmetadata-ui/src/main/resources/ui
|
||||
run: npx playwright install chromium --with-deps
|
||||
|
||||
- name: Generate Documentation
|
||||
working-directory: openmetadata-ui/src/main/resources/ui
|
||||
run: node playwright/doc-generator/generate.js
|
||||
|
||||
- name: Check for Changes
|
||||
id: git-check
|
||||
run: |
|
||||
git diff --exit-code --quiet openmetadata-ui/src/main/resources/ui/playwright/docs || echo "changes=true" >> $GITHUB_OUTPUT
|
||||
|
||||
- name: Commit and Push Changes
|
||||
if: steps.git-check.outputs.changes == 'true'
|
||||
run: |
|
||||
git config --global user.name 'github-actions[bot]'
|
||||
git config --global user.email 'github-actions[bot]@users.noreply.github.com'
|
||||
git add openmetadata-ui/src/main/resources/ui/playwright/docs
|
||||
git commit -m "docs: auto-generate playwright documentation"
|
||||
git push
|
||||
|
||||
- name: Comment on PR
|
||||
if: steps.git-check.outputs.changes == 'true'
|
||||
uses: peter-evans/create-or-update-comment@v4
|
||||
with:
|
||||
issue-number: ${{ github.event.pull_request.number }}
|
||||
body: |
|
||||
## 📝 Documentation Auto-Updated
|
||||
|
||||
The Playwright documentation has been automatically updated to match the changes in this PR.
|
||||
|
||||
* **Generated by**: `playwright-docs-check.yml`
|
||||
* **Status**: ✅ Updated and pushed to branch.
|
||||
@@ -0,0 +1,112 @@
|
||||
# Copyright 2021 Collate
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
# This workflow will build a package using Maven and then publish it to GitHub packages when a release is created
|
||||
# For more information see: https://github.com/actions/setup-java/blob/main/docs/advanced-usage.md#apache-maven-with-a-settings-path
|
||||
|
||||
name: MySQL Playwright Integration Tests
|
||||
on:
|
||||
pull_request_target:
|
||||
types: [labeled, opened, synchronize, reopened, ready_for_review]
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
|
||||
cancel-in-progress: ${{ github.event_name != 'pull_request_target' || github.event.action != 'labeled' || github.event.label.name == 'safe to test' }}
|
||||
|
||||
jobs:
|
||||
playwright-mysql:
|
||||
runs-on: ubuntu-latest
|
||||
if: ${{ !github.event.pull_request.draft && (github.event_name != 'pull_request_target' || github.event.action != 'labeled' || github.event.label.name == 'safe to test') }}
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
browser-type: ["chromium"]
|
||||
environment: test
|
||||
steps:
|
||||
- name: Free Disk Space (Ubuntu)
|
||||
uses: jlumbroso/free-disk-space@main
|
||||
with:
|
||||
tool-cache: false
|
||||
android: true
|
||||
dotnet: true
|
||||
haskell: true
|
||||
large-packages: false
|
||||
swap-storage: true
|
||||
docker-images: false
|
||||
- name: Wait for the labeler
|
||||
uses: lewagon/wait-on-check-action@v1.7.0
|
||||
if: ${{ github.event_name == 'pull_request_target' }}
|
||||
with:
|
||||
ref: ${{ github.event.pull_request.head.sha }}
|
||||
check-name: Team Label
|
||||
repo-token: ${{ secrets.GITHUB_TOKEN }}
|
||||
wait-interval: 90
|
||||
- name: Verify PR labels
|
||||
uses: jesusvasquez333/verify-pr-label-action@v1.4.0
|
||||
if: ${{ github.event_name == 'pull_request_target' }}
|
||||
with:
|
||||
github-token: "${{ secrets.GITHUB_TOKEN }}"
|
||||
valid-labels: "safe to test"
|
||||
pull-request-number: "${{ github.event.pull_request.number }}"
|
||||
disable-reviews: true # To not auto approve changes
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
ref: ${{ github.event.pull_request.head.sha }}
|
||||
- name: Cache Maven Dependencies
|
||||
id: cache-output
|
||||
uses: actions/cache@v4
|
||||
with:
|
||||
path: ~/.m2
|
||||
key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}
|
||||
restore-keys: |
|
||||
${{ runner.os }}-maven-
|
||||
- name: Get yarn cache directory path
|
||||
if: steps.cache-output.outputs.exit-code == 0
|
||||
id: yarn-cache-dir-path
|
||||
run: echo "dir=$(yarn cache dir)" >> $GITHUB_OUTPUT
|
||||
- uses: actions/cache@v4
|
||||
if: steps.yarn-cache-dir-path.outputs.exit-code == 0
|
||||
id: yarn-cache
|
||||
with:
|
||||
path: ${{ steps.yarn-cache-dir-path.outputs.dir }}
|
||||
key: ${{ runner.os }}-yarn-${{ hashFiles('**/yarn.lock') }}
|
||||
restore-keys: |
|
||||
${{ runner.os }}-yarn-
|
||||
- name: Setup Openmetadata Test Environment
|
||||
uses: ./.github/actions/setup-openmetadata-test-environment
|
||||
with:
|
||||
python-version: "3.10"
|
||||
args: "-d mysql"
|
||||
ingestion_dependency: "playwright"
|
||||
- name: Run Playwright Integration Tests with browser ${{ matrix.browser-type }}
|
||||
env:
|
||||
E2E_REDSHIFT_HOST_PORT: ${{ secrets.E2E_REDSHIFT_HOST_PORT }}
|
||||
E2E_REDSHIFT_USERNAME: ${{ secrets.E2E_REDSHIFT_USERNAME }}
|
||||
E2E_REDSHIFT_PASSWORD: ${{ secrets.E2E_REDSHIFT_PASSWORD }}
|
||||
E2E_REDSHIFT_DB: ${{ secrets.TEST_REDSHIFT_DATABASE }}
|
||||
E2E_DRUID_HOST_PORT: ${{ secrets.E2E_DRUID_HOST_PORT }}
|
||||
E2E_HIVE_HOST_PORT: ${{ secrets.E2E_HIVE_HOST_PORT }}
|
||||
run: |
|
||||
source env/bin/activate
|
||||
make install_e2e_tests
|
||||
make run_e2e_tests
|
||||
- uses: actions/upload-artifact@v4
|
||||
if: always()
|
||||
with:
|
||||
name: playwright-artifacts
|
||||
path: ingestion/tests/e2e/artifacts/
|
||||
retention-days: 1
|
||||
- name: Clean Up
|
||||
run: |
|
||||
cd ./docker/development
|
||||
docker compose down --remove-orphans
|
||||
sudo rm -rf ${PWD}/docker-volume
|
||||
@@ -0,0 +1,112 @@
|
||||
# Copyright 2021 Collate
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
# This workflow will build a package using Maven and then publish it to GitHub packages when a release is created
|
||||
# For more information see: https://github.com/actions/setup-java/blob/main/docs/advanced-usage.md#apache-maven-with-a-settings-path
|
||||
|
||||
name: Postgres Playwright Integration Tests
|
||||
on:
|
||||
pull_request_target:
|
||||
types: [labeled, opened, synchronize, reopened, ready_for_review]
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
|
||||
cancel-in-progress: ${{ github.event_name != 'pull_request_target' || github.event.action != 'labeled' || github.event.label.name == 'safe to test' }}
|
||||
|
||||
jobs:
|
||||
playwright-postgresql:
|
||||
runs-on: ubuntu-latest
|
||||
if: ${{ !github.event.pull_request.draft && (github.event_name != 'pull_request_target' || github.event.action != 'labeled' || github.event.label.name == 'safe to test') }}
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
browser-type: ["chromium"]
|
||||
environment: test
|
||||
steps:
|
||||
- name: Free Disk Space (Ubuntu)
|
||||
uses: jlumbroso/free-disk-space@main
|
||||
with:
|
||||
tool-cache: false
|
||||
android: true
|
||||
dotnet: true
|
||||
haskell: true
|
||||
large-packages: false
|
||||
swap-storage: true
|
||||
docker-images: false
|
||||
- name: Wait for the labeler
|
||||
uses: lewagon/wait-on-check-action@v1.7.0
|
||||
if: ${{ github.event_name == 'pull_request_target' }}
|
||||
with:
|
||||
ref: ${{ github.event.pull_request.head.sha }}
|
||||
check-name: Team Label
|
||||
repo-token: ${{ secrets.GITHUB_TOKEN }}
|
||||
wait-interval: 90
|
||||
- name: Verify PR labels
|
||||
uses: jesusvasquez333/verify-pr-label-action@v1.4.0
|
||||
if: ${{ github.event_name == 'pull_request_target' }}
|
||||
with:
|
||||
github-token: "${{ secrets.GITHUB_TOKEN }}"
|
||||
valid-labels: "safe to test"
|
||||
pull-request-number: "${{ github.event.pull_request.number }}"
|
||||
disable-reviews: true # To not auto approve changes
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
ref: ${{ github.event.pull_request.head.sha }}
|
||||
- name: Cache Maven Dependencies
|
||||
id: cache-output
|
||||
uses: actions/cache@v4
|
||||
with:
|
||||
path: ~/.m2
|
||||
key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}
|
||||
restore-keys: |
|
||||
${{ runner.os }}-maven-
|
||||
- name: Get yarn cache directory path
|
||||
if: steps.cache-output.outputs.exit-code == 0
|
||||
id: yarn-cache-dir-path
|
||||
run: echo "dir=$(yarn cache dir)" >> $GITHUB_OUTPUT
|
||||
- uses: actions/cache@v4
|
||||
if: steps.yarn-cache-dir-path.outputs.exit-code == 0
|
||||
id: yarn-cache
|
||||
with:
|
||||
path: ${{ steps.yarn-cache-dir-path.outputs.dir }}
|
||||
key: ${{ runner.os }}-yarn-${{ hashFiles('**/yarn.lock') }}
|
||||
restore-keys: |
|
||||
${{ runner.os }}-yarn-
|
||||
- name: Setup Openmetadata Test Environment
|
||||
uses: ./.github/actions/setup-openmetadata-test-environment
|
||||
with:
|
||||
python-version: "3.10"
|
||||
args: "-d postgresql"
|
||||
ingestion_dependency: "playwright"
|
||||
- name: Run Playwright Integration Tests with browser ${{ matrix.browser-type }}
|
||||
env:
|
||||
E2E_REDSHIFT_HOST_PORT: ${{ secrets.E2E_REDSHIFT_HOST_PORT }}
|
||||
E2E_REDSHIFT_USERNAME: ${{ secrets.E2E_REDSHIFT_USERNAME }}
|
||||
E2E_REDSHIFT_PASSWORD: ${{ secrets.E2E_REDSHIFT_PASSWORD }}
|
||||
E2E_REDSHIFT_DB: ${{ secrets.E2E_REDSHIFT_DATABASE }}
|
||||
E2E_DRUID_HOST_PORT: ${{ secrets.E2E_DRUID_HOST_PORT }}
|
||||
E2E_HIVE_HOST_PORT: ${{ secrets.E2E_HIVE_HOST_PORT }}
|
||||
run: |
|
||||
source env/bin/activate
|
||||
make install_e2e_tests
|
||||
make run_e2e_tests
|
||||
- uses: actions/upload-artifact@v4
|
||||
if: always()
|
||||
with:
|
||||
name: playwright-artifacts
|
||||
path: ingestion/tests/e2e/artifacts/
|
||||
retention-days: 1
|
||||
- name: Clean Up
|
||||
run: |
|
||||
cd ./docker/development
|
||||
docker compose down --remove-orphans
|
||||
sudo rm -rf ${PWD}/docker-volume
|
||||
@@ -0,0 +1,224 @@
|
||||
# Copyright 2021 Collate
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
name: Postgresql PR Knowledge Graph E2E Tests
|
||||
on:
|
||||
workflow_dispatch:
|
||||
pull_request:
|
||||
types:
|
||||
- opened
|
||||
- synchronize
|
||||
- reopened
|
||||
- ready_for_review
|
||||
paths:
|
||||
- ".github/actions/setup-openmetadata-test-environment/action.yml"
|
||||
- ".github/workflows/playwright-knowledge-graph-postgresql-e2e.yml"
|
||||
- "docker/run_local_docker.sh"
|
||||
- "docker/run_local_docker_common.sh"
|
||||
- "docker/run_local_docker_rdf.sh"
|
||||
- "docker/validate_compose.py"
|
||||
- "docker/development/docker-compose-fuseki.yml"
|
||||
- "docker/development/docker-compose-postgres-fuseki.yml"
|
||||
- "docs/rdf-local-development.md"
|
||||
- "openmetadata-service/src/main/java/org/openmetadata/service/apps/bundles/rdf/**"
|
||||
- "openmetadata-service/src/main/java/org/openmetadata/service/rdf/**"
|
||||
- "openmetadata-service/src/main/java/org/openmetadata/service/resources/rdf/**"
|
||||
- "openmetadata-service/src/test/java/org/openmetadata/service/apps/bundles/rdf/**"
|
||||
- "openmetadata-service/src/test/java/org/openmetadata/service/rdf/**"
|
||||
- "openmetadata-service/src/test/java/org/openmetadata/service/resources/rdf/**"
|
||||
- "openmetadata-spec/src/main/resources/rdf/**"
|
||||
- "openmetadata-ui/src/main/resources/ui/playwright/e2e/Features/KnowledgeGraph.spec.ts"
|
||||
- "openmetadata-ui/src/main/resources/ui/playwright.config.ts"
|
||||
- "openmetadata-ui/src/main/resources/ui/src/components/KnowledgeGraph3D/**"
|
||||
- "openmetadata-ui/src/main/resources/ui/src/components/OntologyExplorer/**"
|
||||
- "openmetadata-ui/src/main/resources/ui/src/rest/rdfAPI.ts"
|
||||
- "openmetadata-ui/src/main/resources/ui/src/types/knowledgeGraph.types.ts"
|
||||
- "openmetadata-ui/src/main/resources/ui/src/utils/TableUtils.tsx"
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
concurrency:
|
||||
group: playwright-knowledge-graph-pr-postgresql-${{ github.event.pull_request.number || github.run_id }}
|
||||
cancel-in-progress: true
|
||||
|
||||
jobs:
|
||||
build:
|
||||
runs-on: ubuntu-latest
|
||||
if: ${{ github.event_name != 'pull_request' || !github.event.pull_request.draft }}
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}
|
||||
|
||||
- name: Setup JDK 21
|
||||
uses: actions/setup-java@v4
|
||||
with:
|
||||
java-version: '21'
|
||||
distribution: 'temurin'
|
||||
|
||||
- name: Cache Maven Dependencies
|
||||
uses: actions/cache@v4
|
||||
with:
|
||||
path: ~/.m2
|
||||
key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}
|
||||
restore-keys: |
|
||||
${{ runner.os }}-maven-
|
||||
|
||||
- name: Install antlr cli
|
||||
run: sudo make install_antlr_cli
|
||||
|
||||
- name: Build with Maven
|
||||
run: mvn -DskipTests clean package
|
||||
|
||||
- name: Upload Maven build artifact
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: openmetadata-build
|
||||
path: openmetadata-dist/target/openmetadata-*.tar.gz
|
||||
retention-days: 1
|
||||
|
||||
playwright-knowledge-graph-postgresql:
|
||||
needs: [build]
|
||||
runs-on: ubuntu-latest
|
||||
if: ${{ !cancelled() && needs.build.result == 'success' && (github.event_name != 'pull_request' || !github.event.pull_request.head.repo.fork) }}
|
||||
environment: test
|
||||
steps:
|
||||
- name: Free Disk Space (Ubuntu)
|
||||
uses: jlumbroso/free-disk-space@main
|
||||
with:
|
||||
tool-cache: false
|
||||
android: true
|
||||
dotnet: true
|
||||
haskell: true
|
||||
large-packages: false
|
||||
swap-storage: true
|
||||
docker-images: false
|
||||
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}
|
||||
|
||||
- name: Prepare temporary directory for Maven build artifact
|
||||
run: mkdir -p "${{ runner.temp }}/openmetadata-build-artifact"
|
||||
|
||||
- name: Download Maven build artifact
|
||||
uses: actions/download-artifact@v4
|
||||
with:
|
||||
name: openmetadata-build
|
||||
path: ${{ runner.temp }}/openmetadata-build-artifact
|
||||
|
||||
- name: Copy Maven build artifact into workspace
|
||||
run: |
|
||||
mkdir -p openmetadata-dist/target
|
||||
cp -a "${{ runner.temp }}/openmetadata-build-artifact/." openmetadata-dist/target/
|
||||
|
||||
- name: Setup Openmetadata Test Environment
|
||||
uses: ./.github/actions/setup-openmetadata-test-environment
|
||||
with:
|
||||
python-version: "3.10"
|
||||
args: "-d postgresql -s true"
|
||||
startup-script: "./docker/run_local_docker_rdf.sh"
|
||||
ingestion_dependency: "all"
|
||||
|
||||
- name: Wait for Fuseki to be healthy
|
||||
run: |
|
||||
echo "Verifying Fuseki is healthy before running tests..."
|
||||
for i in $(seq 1 30); do
|
||||
if curl -sf "http://localhost:3030/\$/ping" > /dev/null 2>&1; then
|
||||
echo "Fuseki is healthy"
|
||||
exit 0
|
||||
fi
|
||||
echo "Waiting for Fuseki ($i/30)..."
|
||||
sleep 10
|
||||
done
|
||||
echo "Fuseki failed health check. Container logs:"
|
||||
docker logs openmetadata-fuseki --tail 100
|
||||
exit 1
|
||||
|
||||
- name: Setup Node.js
|
||||
uses: actions/setup-node@v4
|
||||
with:
|
||||
node-version-file: "openmetadata-ui/src/main/resources/ui/.nvmrc"
|
||||
|
||||
- name: Install dependencies
|
||||
working-directory: openmetadata-ui/src/main/resources/ui/
|
||||
run: yarn --ignore-scripts --frozen-lockfile
|
||||
|
||||
- name: Install Playwright Browsers
|
||||
run: npx playwright@1.57.0 install chromium --with-deps
|
||||
|
||||
- name: Run Knowledge Graph Playwright tests
|
||||
working-directory: openmetadata-ui/src/main/resources/ui/
|
||||
run: npx playwright test --project="Knowledge Graph"
|
||||
env:
|
||||
PLAYWRIGHT_IS_OSS: true
|
||||
PLAYWRIGHT_SNOWFLAKE_USERNAME: ${{ secrets.TEST_SNOWFLAKE_USERNAME }}
|
||||
PLAYWRIGHT_SNOWFLAKE_PASSWORD: ${{ secrets.TEST_SNOWFLAKE_PASSWORD }}
|
||||
PLAYWRIGHT_SNOWFLAKE_ACCOUNT: ${{ secrets.TEST_SNOWFLAKE_ACCOUNT }}
|
||||
PLAYWRIGHT_SNOWFLAKE_DATABASE: ${{ secrets.TEST_SNOWFLAKE_DATABASE }}
|
||||
PLAYWRIGHT_SNOWFLAKE_WAREHOUSE: ${{ secrets.TEST_SNOWFLAKE_WAREHOUSE }}
|
||||
PLAYWRIGHT_SNOWFLAKE_PASSPHRASE: ${{ secrets.TEST_SNOWFLAKE_PASSPHRASE }}
|
||||
PLAYWRIGHT_BQ_PRIVATE_KEY: ${{ secrets.TEST_BQ_PRIVATE_KEY }}
|
||||
PLAYWRIGHT_BQ_PROJECT_ID: ${{ secrets.PLAYWRIGHT_BQ_PROJECT_ID }}
|
||||
PLAYWRIGHT_BQ_PRIVATE_KEY_ID: ${{ secrets.TEST_BQ_PRIVATE_KEY_ID }}
|
||||
PLAYWRIGHT_BQ_PROJECT_ID_TAXONOMY: ${{ secrets.TEST_BQ_PROJECT_ID_TAXONOMY }}
|
||||
PLAYWRIGHT_BQ_CLIENT_EMAIL: ${{ secrets.TEST_BQ_CLIENT_EMAIL }}
|
||||
PLAYWRIGHT_BQ_CLIENT_ID: ${{ secrets.TEST_BQ_CLIENT_ID }}
|
||||
PLAYWRIGHT_REDSHIFT_HOST: ${{ secrets.E2E_REDSHIFT_HOST_PORT }}
|
||||
PLAYWRIGHT_REDSHIFT_USERNAME: ${{ secrets.E2E_REDSHIFT_USERNAME }}
|
||||
PLAYWRIGHT_REDSHIFT_PASSWORD: ${{ secrets.E2E_REDSHIFT_PASSWORD }}
|
||||
PLAYWRIGHT_REDSHIFT_DATABASE: ${{ secrets.TEST_REDSHIFT_DATABASE }}
|
||||
PLAYWRIGHT_METABASE_USERNAME: ${{ secrets.TEST_METABASE_USERNAME }}
|
||||
PLAYWRIGHT_METABASE_PASSWORD: ${{ secrets.TEST_METABASE_PASSWORD }}
|
||||
PLAYWRIGHT_METABASE_DB_SERVICE_NAME: ${{ secrets.TEST_METABASE_DB_SERVICE_NAME }}
|
||||
PLAYWRIGHT_METABASE_HOST_PORT: ${{ secrets.TEST_METABASE_HOST_PORT }}
|
||||
PLAYWRIGHT_SUPERSET_USERNAME: ${{ secrets.TEST_SUPERSET_USERNAME }}
|
||||
PLAYWRIGHT_SUPERSET_PASSWORD: ${{ secrets.TEST_SUPERSET_PASSWORD }}
|
||||
PLAYWRIGHT_SUPERSET_HOST_PORT: ${{ secrets.TEST_SUPERSET_HOST_PORT }}
|
||||
PLAYWRIGHT_KAFKA_BOOTSTRAP_SERVERS: ${{ secrets.TEST_KAFKA_BOOTSTRAP_SERVERS }}
|
||||
PLAYWRIGHT_KAFKA_SCHEMA_REGISTRY_URL: ${{ secrets.TEST_KAFKA_SCHEMA_REGISTRY_URL }}
|
||||
PLAYWRIGHT_GLUE_ACCESS_KEY: ${{ secrets.TEST_GLUE_ACCESS_KEY }}
|
||||
PLAYWRIGHT_GLUE_SECRET_KEY: ${{ secrets.TEST_GLUE_SECRET_KEY }}
|
||||
PLAYWRIGHT_GLUE_AWS_REGION: ${{ secrets.TEST_GLUE_AWS_REGION }}
|
||||
PLAYWRIGHT_GLUE_ENDPOINT: ${{ secrets.TEST_GLUE_ENDPOINT }}
|
||||
PLAYWRIGHT_GLUE_STORAGE_SERVICE: ${{ secrets.TEST_GLUE_STORAGE_SERVICE }}
|
||||
PLAYWRIGHT_MYSQL_USERNAME: ${{ secrets.TEST_MYSQL_USERNAME }}
|
||||
PLAYWRIGHT_MYSQL_PASSWORD: ${{ secrets.TEST_MYSQL_PASSWORD }}
|
||||
PLAYWRIGHT_MYSQL_HOST_PORT: ${{ secrets.TEST_MYSQL_HOST_PORT }}
|
||||
PLAYWRIGHT_MYSQL_DATABASE_SCHEMA: ${{ secrets.TEST_MYSQL_DATABASE_SCHEMA }}
|
||||
PLAYWRIGHT_POSTGRES_USERNAME: ${{ secrets.TEST_POSTGRES_USERNAME }}
|
||||
PLAYWRIGHT_POSTGRES_PASSWORD: ${{ secrets.TEST_POSTGRES_PASSWORD }}
|
||||
PLAYWRIGHT_POSTGRES_HOST_PORT: ${{ secrets.TEST_POSTGRES_HOST_PORT }}
|
||||
PLAYWRIGHT_POSTGRES_DATABASE: ${{ secrets.TEST_POSTGRES_DATABASE }}
|
||||
PLAYWRIGHT_AIRFLOW_HOST_PORT: ${{ secrets.TEST_AIRFLOW_HOST_PORT }}
|
||||
PLAYWRIGHT_ML_MODEL_TRACKING_URI: ${{ secrets.TEST_ML_MODEL_TRACKING_URI }}
|
||||
PLAYWRIGHT_ML_MODEL_REGISTRY_URI: ${{ secrets.TEST_ML_MODEL_REGISTRY_URI }}
|
||||
PLAYWRIGHT_S3_STORAGE_ACCESS_KEY_ID: ${{ secrets.TEST_S3_STORAGE_ACCESS_KEY_ID }}
|
||||
PLAYWRIGHT_S3_STORAGE_SECRET_ACCESS_KEY: ${{ secrets.TEST_S3_STORAGE_SECRET_ACCESS_KEY }}
|
||||
PLAYWRIGHT_S3_STORAGE_END_POINT_URL: ${{ secrets.TEST_S3_STORAGE_END_POINT_URL }}
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
- uses: actions/upload-artifact@v4
|
||||
if: ${{ !cancelled() }}
|
||||
with:
|
||||
name: playwright-knowledge-graph-report
|
||||
path: openmetadata-ui/src/main/resources/ui/playwright/output/playwright-report
|
||||
retention-days: 5
|
||||
|
||||
- name: Clean Up
|
||||
if: always()
|
||||
run: |
|
||||
docker compose -f docker/development/docker-compose-postgres.yml -f docker/development/docker-compose-fuseki.yml down --remove-orphans || true
|
||||
docker compose -f docker/development/docker-compose-postgres.yml down --remove-orphans || true
|
||||
sudo rm -rf ${PWD}/docker/development/docker-volume
|
||||
@@ -0,0 +1,42 @@
|
||||
# Copyright 2021 Collate
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
# Avoid running Playwright on each PR opened which does not modify Java or UI code
|
||||
# https://docs.github.com/en/repositories/configuring-branches-and-merges-in-your-repository/defining-the-mergeability-
|
||||
# of-pull-requests/troubleshooting-required-status-checks#handling-skipped-but-required-checks
|
||||
name: MySQL PR Playwright E2E Tests
|
||||
|
||||
on:
|
||||
pull_request_target:
|
||||
types:
|
||||
- labeled
|
||||
- opened
|
||||
- synchronize
|
||||
- reopened
|
||||
paths:
|
||||
- ".github/**"
|
||||
- "openmetadata-dist/**"
|
||||
- "docker/**"
|
||||
- "!docker/development/docker-compose.yml"
|
||||
- "!docker/development/docker-compose-postgres.yml"
|
||||
- "openmetadata-ui/src/main/resources/ui/playwright/doc-generator/**"
|
||||
- "openmetadata-ui/src/main/resources/ui/playwright/docs/**"
|
||||
|
||||
jobs:
|
||||
playwright-ci-mysql:
|
||||
runs-on: ubuntu-latest
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
shardIndex: [1, 2]
|
||||
shardTotal: [2]
|
||||
steps:
|
||||
- run: 'echo "Step is not required"'
|
||||
@@ -0,0 +1,191 @@
|
||||
# Copyright 2021 Collate
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
# This workflow executes end-to-end (e2e) tests using Playwright with MySQL as the database.
|
||||
# For more information see: https://github.com/actions/setup-java/blob/main/docs/advanced-usage.md#apache-maven-with-a-settings-path
|
||||
|
||||
name: MySQL PR Playwright E2E Tests
|
||||
on:
|
||||
workflow_dispatch:
|
||||
pull_request_target:
|
||||
types:
|
||||
- labeled
|
||||
- opened
|
||||
- synchronize
|
||||
- reopened
|
||||
- ready_for_review
|
||||
paths-ignore:
|
||||
- ".github/**"
|
||||
- "openmetadata-dist/**"
|
||||
- "docker/**"
|
||||
- "!docker/development/docker-compose.yml"
|
||||
- "!docker/development/docker-compose-postgres.yml"
|
||||
- "openmetadata-ui/src/main/resources/ui/playwright/doc-generator/**"
|
||||
- "openmetadata-ui/src/main/resources/ui/playwright/docs/**"
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
concurrency:
|
||||
group: playwright-ci-pr-mysql-${{ github.event.pull_request.number || github.run_id }}
|
||||
cancel-in-progress: ${{ github.event_name != 'pull_request_target' || github.event.action != 'labeled' || github.event.label.name == 'safe to test' }}
|
||||
|
||||
jobs:
|
||||
playwright-ci-mysql:
|
||||
runs-on: ubuntu-latest
|
||||
if: ${{ !github.event.pull_request.draft && (github.event_name != 'pull_request_target' || github.event.action != 'labeled' || github.event.label.name == 'safe to test') }}
|
||||
environment: test
|
||||
permissions:
|
||||
contents: read
|
||||
id-token: write # Required for GitHub OIDC (Flakiness.io reporter)
|
||||
env:
|
||||
# Playwright logs the admin user in many times (performAdminLogin per test, parallel
|
||||
# workers, retries). The production default of 5 active sessions per user would evict the
|
||||
# long-lived storageState session the page fixtures rely on and 401 every request. Raise
|
||||
# the cap for E2E only; docker compose reads this for ${AUTHENTICATION_MAX_ACTIVE_SESSIONS_PER_USER:-5}.
|
||||
AUTHENTICATION_MAX_ACTIVE_SESSIONS_PER_USER: "10000"
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
shardIndex: [1, 2]
|
||||
shardTotal: [2]
|
||||
steps:
|
||||
- name: Free Disk Space (Ubuntu)
|
||||
uses: jlumbroso/free-disk-space@main
|
||||
with:
|
||||
tool-cache: false
|
||||
android: true
|
||||
dotnet: true
|
||||
haskell: true
|
||||
large-packages: false
|
||||
swap-storage: true
|
||||
docker-images: false
|
||||
- name: Wait for the labeler
|
||||
uses: lewagon/wait-on-check-action@v1.7.0
|
||||
if: ${{ github.event_name == 'pull_request_target' }}
|
||||
with:
|
||||
ref: ${{ github.event.pull_request.head.sha }}
|
||||
check-name: Team Label
|
||||
repo-token: ${{ secrets.GITHUB_TOKEN }}
|
||||
wait-interval: 90
|
||||
|
||||
- name: Verify PR labels
|
||||
uses: jesusvasquez333/verify-pr-label-action@v1.4.0
|
||||
if: ${{ github.event_name == 'pull_request_target' }}
|
||||
with:
|
||||
github-token: "${{ secrets.GITHUB_TOKEN }}"
|
||||
valid-labels: "safe to test"
|
||||
pull-request-number: "${{ github.event.pull_request.number }}"
|
||||
disable-reviews: true # To not auto approve changes
|
||||
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
ref: ${{ github.event.pull_request.head.sha }}
|
||||
|
||||
- name: Cache Maven Dependencies
|
||||
id: cache-output
|
||||
uses: actions/cache@v4
|
||||
with:
|
||||
path: ~/.m2
|
||||
key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}
|
||||
restore-keys: |
|
||||
${{ runner.os }}-maven-
|
||||
|
||||
- name: Setup Openmetadata Test Environment
|
||||
uses: ./.github/actions/setup-openmetadata-test-environment
|
||||
with:
|
||||
python-version: "3.10"
|
||||
args: "-d mysql"
|
||||
ingestion_dependency: "all"
|
||||
|
||||
- name: Setup Node.js
|
||||
uses: actions/setup-node@v4
|
||||
with:
|
||||
node-version-file: 'openmetadata-ui/src/main/resources/ui/.nvmrc'
|
||||
|
||||
- name: Install dependencies
|
||||
working-directory: openmetadata-ui/src/main/resources/ui/
|
||||
run: yarn --ignore-scripts --frozen-lockfile
|
||||
- name: Install Playwright Browsers
|
||||
run: npx playwright@1.57.0 install chromium --with-deps
|
||||
- name: Run Playwright tests
|
||||
working-directory: openmetadata-ui/src/main/resources/ui/
|
||||
run: |
|
||||
if [[ "${{ contains(github.event.pull_request.changed_files, 'ingestion/') }}" == "true" ]]; then
|
||||
npx playwright test --grep @ingestion --shard=${{ matrix.shardIndex }}/${{ matrix.shardTotal }}
|
||||
else
|
||||
npx playwright test --shard=${{ matrix.shardIndex }}/${{ matrix.shardTotal }}
|
||||
fi
|
||||
env:
|
||||
PLAYWRIGHT_IS_OSS: true
|
||||
PLAYWRIGHT_SNOWFLAKE_USERNAME: ${{ secrets.TEST_SNOWFLAKE_USERNAME }}
|
||||
PLAYWRIGHT_SNOWFLAKE_PASSWORD: ${{ secrets.TEST_SNOWFLAKE_PASSWORD }}
|
||||
PLAYWRIGHT_SNOWFLAKE_ACCOUNT: ${{ secrets.TEST_SNOWFLAKE_ACCOUNT }}
|
||||
PLAYWRIGHT_SNOWFLAKE_DATABASE: ${{ secrets.TEST_SNOWFLAKE_DATABASE }}
|
||||
PLAYWRIGHT_SNOWFLAKE_WAREHOUSE: ${{ secrets.TEST_SNOWFLAKE_WAREHOUSE }}
|
||||
PLAYWRIGHT_SNOWFLAKE_PASSPHRASE: ${{ secrets.TEST_SNOWFLAKE_PASSPHRASE }}
|
||||
PLAYWRIGHT_PROJECT_ID: ${{ steps.cypress-project-id.outputs.CYPRESS_PROJECT_ID }}
|
||||
PLAYWRIGHT_BQ_PRIVATE_KEY: ${{ secrets.TEST_BQ_PRIVATE_KEY }}
|
||||
PLAYWRIGHT_BQ_PROJECT_ID: ${{ secrets.PLAYWRIGHT_BQ_PROJECT_ID }}
|
||||
PLAYWRIGHT_BQ_PRIVATE_KEY_ID: ${{ secrets.TEST_BQ_PRIVATE_KEY_ID }}
|
||||
PLAYWRIGHT_BQ_PROJECT_ID_TAXONOMY: ${{ secrets.TEST_BQ_PROJECT_ID_TAXONOMY }}
|
||||
PLAYWRIGHT_BQ_CLIENT_EMAIL: ${{ secrets.TEST_BQ_CLIENT_EMAIL }}
|
||||
PLAYWRIGHT_BQ_CLIENT_ID: ${{ secrets.TEST_BQ_CLIENT_ID }}
|
||||
PLAYWRIGHT_REDSHIFT_HOST: ${{ secrets.E2E_REDSHIFT_HOST_PORT }}
|
||||
PLAYWRIGHT_REDSHIFT_USERNAME: ${{ secrets.E2E_REDSHIFT_USERNAME }}
|
||||
PLAYWRIGHT_REDSHIFT_PASSWORD: ${{ secrets.E2E_REDSHIFT_PASSWORD }}
|
||||
PLAYWRIGHT_REDSHIFT_DATABASE: ${{ secrets.TEST_REDSHIFT_DATABASE }}
|
||||
PLAYWRIGHT_METABASE_USERNAME: ${{ secrets.TEST_METABASE_USERNAME }}
|
||||
PLAYWRIGHT_METABASE_PASSWORD: ${{ secrets.TEST_METABASE_PASSWORD }}
|
||||
PLAYWRIGHT_METABASE_DB_SERVICE_NAME: ${{ secrets.TEST_METABASE_DB_SERVICE_NAME }}
|
||||
PLAYWRIGHT_METABASE_HOST_PORT: ${{ secrets.TEST_METABASE_HOST_PORT }}
|
||||
PLAYWRIGHT_SUPERSET_USERNAME: ${{ secrets.TEST_SUPERSET_USERNAME }}
|
||||
PLAYWRIGHT_SUPERSET_PASSWORD: ${{ secrets.TEST_SUPERSET_PASSWORD }}
|
||||
PLAYWRIGHT_SUPERSET_HOST_PORT: ${{ secrets.TEST_SUPERSET_HOST_PORT }}
|
||||
PLAYWRIGHT_KAFKA_BOOTSTRAP_SERVERS: ${{ secrets.TEST_KAFKA_BOOTSTRAP_SERVERS }}
|
||||
PLAYWRIGHT_KAFKA_SCHEMA_REGISTRY_URL: ${{ secrets.TEST_KAFKA_SCHEMA_REGISTRY_URL }}
|
||||
PLAYWRIGHT_GLUE_ACCESS_KEY: ${{ secrets.TEST_GLUE_ACCESS_KEY }}
|
||||
PLAYWRIGHT_GLUE_SECRET_KEY: ${{ secrets.TEST_GLUE_SECRET_KEY }}
|
||||
PLAYWRIGHT_GLUE_AWS_REGION: ${{ secrets.TEST_GLUE_AWS_REGION }}
|
||||
PLAYWRIGHT_GLUE_ENDPOINT: ${{ secrets.TEST_GLUE_ENDPOINT }}
|
||||
PLAYWRIGHT_GLUE_STORAGE_SERVICE: ${{ secrets.TEST_GLUE_STORAGE_SERVICE }}
|
||||
PLAYWRIGHT_MYSQL_USERNAME: ${{ secrets.TEST_MYSQL_USERNAME }}
|
||||
PLAYWRIGHT_MYSQL_PASSWORD: ${{ secrets.TEST_MYSQL_PASSWORD }}
|
||||
PLAYWRIGHT_MYSQL_HOST_PORT: ${{ secrets.TEST_MYSQL_HOST_PORT }}
|
||||
PLAYWRIGHT_MYSQL_DATABASE_SCHEMA: ${{ secrets.TEST_MYSQL_DATABASE_SCHEMA }}
|
||||
PLAYWRIGHT_POSTGRES_USERNAME: ${{ secrets.TEST_POSTGRES_USERNAME }}
|
||||
PLAYWRIGHT_POSTGRES_PASSWORD: ${{ secrets.TEST_POSTGRES_PASSWORD }}
|
||||
PLAYWRIGHT_POSTGRES_HOST_PORT: ${{ secrets.TEST_POSTGRES_HOST_PORT }}
|
||||
PLAYWRIGHT_POSTGRES_DATABASE: ${{ secrets.TEST_POSTGRES_DATABASE }}
|
||||
PLAYWRIGHT_AIRFLOW_HOST_PORT: ${{ secrets.TEST_AIRFLOW_HOST_PORT }}
|
||||
PLAYWRIGHT_ML_MODEL_TRACKING_URI: ${{ secrets.TEST_ML_MODEL_TRACKING_URI }}
|
||||
PLAYWRIGHT_ML_MODEL_REGISTRY_URI: ${{ secrets.TEST_ML_MODEL_REGISTRY_URI }}
|
||||
PLAYWRIGHT_S3_STORAGE_ACCESS_KEY_ID: ${{ secrets.TEST_S3_STORAGE_ACCESS_KEY_ID }}
|
||||
PLAYWRIGHT_S3_STORAGE_SECRET_ACCESS_KEY: ${{ secrets.TEST_S3_STORAGE_SECRET_ACCESS_KEY }}
|
||||
PLAYWRIGHT_S3_STORAGE_END_POINT_URL: ${{ secrets.TEST_S3_STORAGE_END_POINT_URL }}
|
||||
|
||||
# Recommended: pass the GitHub token lets this action correctly
|
||||
# determine the unique run id necessary to re-run the checks
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
- uses: actions/upload-artifact@v4
|
||||
if: ${{ !cancelled() }}
|
||||
with:
|
||||
name: playwright-report--${{ matrix.shardIndex }}
|
||||
path: openmetadata-ui/src/main/resources/ui/playwright/output/playwright-report
|
||||
retention-days: 5
|
||||
|
||||
- name: Clean Up
|
||||
run: |
|
||||
cd ./docker/development
|
||||
docker compose down --remove-orphans
|
||||
sudo rm -rf ${PWD}/docker-volume
|
||||
@@ -0,0 +1,175 @@
|
||||
# Copyright 2021 Collate
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
name: Postgresql PR Ontology RDF E2E Tests
|
||||
on:
|
||||
workflow_dispatch:
|
||||
pull_request:
|
||||
types:
|
||||
- opened
|
||||
- synchronize
|
||||
- reopened
|
||||
- ready_for_review
|
||||
paths:
|
||||
- ".github/actions/setup-openmetadata-test-environment/action.yml"
|
||||
- ".github/workflows/playwright-ontology-rdf-postgresql-e2e.yml"
|
||||
- "docker/run_local_docker.sh"
|
||||
- "docker/run_local_docker_common.sh"
|
||||
- "docker/run_local_docker_rdf.sh"
|
||||
- "docker/development/docker-compose-fuseki.yml"
|
||||
- "docker/development/docker-compose-postgres-fuseki.yml"
|
||||
- "openmetadata-service/src/main/java/org/openmetadata/service/rdf/**"
|
||||
- "openmetadata-service/src/main/java/org/openmetadata/service/resources/glossary/**"
|
||||
- "openmetadata-spec/src/main/resources/json/schema/api/data/createGlossaryTerm.json"
|
||||
- "openmetadata-spec/src/main/resources/json/schema/configuration/glossaryTermRelationSettings.json"
|
||||
- "openmetadata-spec/src/main/resources/json/schema/entity/data/glossary.json"
|
||||
- "openmetadata-spec/src/main/resources/json/schema/entity/data/glossaryTerm.json"
|
||||
- "openmetadata-ui/src/main/resources/ui/playwright/e2e/Features/OntologyImportRdf.spec.ts"
|
||||
- "openmetadata-ui/src/main/resources/ui/playwright.config.ts"
|
||||
- "openmetadata-ui/src/main/resources/ui/src/components/Glossary/GlossaryHeader/**"
|
||||
- "openmetadata-ui/src/main/resources/ui/src/components/Glossary/ImportOntologyModal/**"
|
||||
- "openmetadata-ui/src/main/resources/ui/src/rest/importExportAPI.ts"
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
concurrency:
|
||||
group: playwright-ontology-rdf-pr-postgresql-${{ github.event.pull_request.number || github.run_id }}
|
||||
cancel-in-progress: true
|
||||
|
||||
jobs:
|
||||
build:
|
||||
runs-on: ubuntu-latest
|
||||
if: ${{ github.event_name != 'pull_request' || !github.event.pull_request.draft }}
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}
|
||||
|
||||
- name: Setup JDK 21
|
||||
uses: actions/setup-java@v4
|
||||
with:
|
||||
java-version: '21'
|
||||
distribution: 'temurin'
|
||||
|
||||
- name: Cache Maven Dependencies
|
||||
uses: actions/cache@v4
|
||||
with:
|
||||
path: ~/.m2
|
||||
key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}
|
||||
restore-keys: |
|
||||
${{ runner.os }}-maven-
|
||||
|
||||
- name: Install antlr cli
|
||||
run: sudo make install_antlr_cli
|
||||
|
||||
- name: Build with Maven
|
||||
run: mvn -DskipTests clean package
|
||||
|
||||
- name: Upload Maven build artifact
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: openmetadata-build
|
||||
path: openmetadata-dist/target/openmetadata-*.tar.gz
|
||||
retention-days: 1
|
||||
|
||||
playwright-ontology-rdf-postgresql:
|
||||
needs: [build]
|
||||
runs-on: ubuntu-latest
|
||||
if: ${{ !cancelled() && needs.build.result == 'success' && (github.event_name != 'pull_request' || !github.event.pull_request.head.repo.fork) }}
|
||||
environment: test
|
||||
steps:
|
||||
- name: Free Disk Space (Ubuntu)
|
||||
uses: jlumbroso/free-disk-space@main
|
||||
with:
|
||||
tool-cache: false
|
||||
android: true
|
||||
dotnet: true
|
||||
haskell: true
|
||||
large-packages: false
|
||||
swap-storage: true
|
||||
docker-images: false
|
||||
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}
|
||||
|
||||
- name: Prepare temporary directory for Maven build artifact
|
||||
run: mkdir -p "${{ runner.temp }}/openmetadata-build-artifact"
|
||||
|
||||
- name: Download Maven build artifact
|
||||
uses: actions/download-artifact@v4
|
||||
with:
|
||||
name: openmetadata-build
|
||||
path: ${{ runner.temp }}/openmetadata-build-artifact
|
||||
|
||||
- name: Copy Maven build artifact into workspace
|
||||
run: |
|
||||
mkdir -p openmetadata-dist/target
|
||||
cp -a "${{ runner.temp }}/openmetadata-build-artifact/." openmetadata-dist/target/
|
||||
|
||||
- name: Setup Openmetadata Test Environment
|
||||
uses: ./.github/actions/setup-openmetadata-test-environment
|
||||
with:
|
||||
python-version: "3.10"
|
||||
args: "-d postgresql -s true"
|
||||
startup-script: "./docker/run_local_docker_rdf.sh"
|
||||
ingestion_dependency: "all"
|
||||
|
||||
- name: Wait for Fuseki to be healthy
|
||||
run: |
|
||||
echo "Verifying Fuseki is healthy before running tests..."
|
||||
for i in $(seq 1 30); do
|
||||
if curl -sf "http://localhost:3030/\$/ping" > /dev/null 2>&1; then
|
||||
echo "Fuseki is healthy"
|
||||
exit 0
|
||||
fi
|
||||
echo "Waiting for Fuseki ($i/30)..."
|
||||
sleep 10
|
||||
done
|
||||
echo "Fuseki failed health check. Container logs:"
|
||||
docker logs openmetadata-fuseki --tail 100
|
||||
exit 1
|
||||
|
||||
- name: Setup Node.js
|
||||
uses: actions/setup-node@v4
|
||||
with:
|
||||
node-version-file: "openmetadata-ui/src/main/resources/ui/.nvmrc"
|
||||
|
||||
- name: Install dependencies
|
||||
working-directory: openmetadata-ui/src/main/resources/ui/
|
||||
run: yarn --ignore-scripts --frozen-lockfile
|
||||
|
||||
- name: Install Playwright Browsers
|
||||
run: npx playwright@1.57.0 install chromium --with-deps
|
||||
|
||||
- name: Run Ontology RDF Playwright tests
|
||||
working-directory: openmetadata-ui/src/main/resources/ui/
|
||||
run: npx playwright test --project="Ontology RDF"
|
||||
env:
|
||||
PLAYWRIGHT_IS_OSS: true
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
- uses: actions/upload-artifact@v4
|
||||
if: ${{ !cancelled() }}
|
||||
with:
|
||||
name: playwright-ontology-rdf-report
|
||||
path: openmetadata-ui/src/main/resources/ui/playwright/output/playwright-report
|
||||
retention-days: 5
|
||||
|
||||
- name: Clean Up
|
||||
if: always()
|
||||
run: |
|
||||
docker compose -f docker/development/docker-compose-postgres.yml -f docker/development/docker-compose-fuseki.yml down --remove-orphans || true
|
||||
docker compose -f docker/development/docker-compose-postgres.yml down --remove-orphans || true
|
||||
sudo rm -rf ${PWD}/docker/development/docker-volume
|
||||
@@ -0,0 +1,620 @@
|
||||
# Copyright 2021 Collate
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
# This workflow executes end-to-end (e2e) tests using Playwright with PostgreSQL as the database.
|
||||
# For more information see: https://github.com/actions/setup-java/blob/main/docs/advanced-usage.md#apache-maven-with-a-settings-path
|
||||
|
||||
name: Postgresql PR Playwright E2E Tests
|
||||
on:
|
||||
merge_group:
|
||||
workflow_dispatch:
|
||||
# Note: paths-ignore removed — the workflow always triggers so that the
|
||||
# required status check (playwright-summary) always completes. Path filtering
|
||||
# is handled inside the workflow via dorny/paths-filter so PRs without
|
||||
# relevant changes skip the expensive jobs while still reporting a passing status.
|
||||
pull_request_target:
|
||||
types:
|
||||
- labeled
|
||||
- opened
|
||||
- synchronize
|
||||
- reopened
|
||||
- ready_for_review
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
concurrency:
|
||||
group: playwright-ci-pr-postgresql-${{ github.event.pull_request.number || github.run_id }}
|
||||
cancel-in-progress: ${{ github.event_name != 'pull_request_target' || github.event.action != 'labeled' || github.event.label.name == 'safe to test' }}
|
||||
|
||||
jobs:
|
||||
check-changes:
|
||||
runs-on: ubuntu-latest
|
||||
outputs:
|
||||
e2e: ${{ steps.filter.outputs.e2e }}
|
||||
docker-compose: ${{ steps.filter.outputs.docker-compose }}
|
||||
steps:
|
||||
- uses: dorny/paths-filter@v4
|
||||
id: filter
|
||||
with:
|
||||
filters: |
|
||||
e2e:
|
||||
- 'openmetadata-service/**'
|
||||
- 'openmetadata-ui/**'
|
||||
- 'openmetadata-spec/**'
|
||||
- 'openmetadata-integration-tests/**'
|
||||
- 'ingestion/**'
|
||||
- 'bootstrap/**'
|
||||
- 'conf/**'
|
||||
- 'docker/development/**'
|
||||
- 'openmetadata-clients/**'
|
||||
- 'openmetadata-airflow-apis/**'
|
||||
- 'openmetadata-mcp/**'
|
||||
- 'openmetadata-sdk/**'
|
||||
- 'openmetadata-shaded-deps/**'
|
||||
- 'openmetadata-ui-core-components/**'
|
||||
- 'openmetadata-k8s-operator/**'
|
||||
- 'common/**'
|
||||
- 'openspec/**'
|
||||
- 'pom.xml'
|
||||
- 'Makefile'
|
||||
docker-compose:
|
||||
- 'docker/development/docker-compose.yml'
|
||||
- 'docker/development/docker-compose-postgres.yml'
|
||||
|
||||
build:
|
||||
needs: check-changes
|
||||
runs-on: ubuntu-latest
|
||||
if: |
|
||||
!github.event.pull_request.draft &&
|
||||
(github.event_name == 'merge_group' || github.event_name == 'workflow_dispatch' ||
|
||||
needs.check-changes.outputs.e2e == 'true' || needs.check-changes.outputs.docker-compose == 'true') &&
|
||||
(github.event_name != 'pull_request_target' || github.event.action != 'labeled' || github.event.label.name == 'safe to test')
|
||||
steps:
|
||||
- name: Wait for the labeler
|
||||
uses: lewagon/wait-on-check-action@v1.7.0
|
||||
if: ${{ github.event_name == 'pull_request_target' }}
|
||||
with:
|
||||
ref: ${{ github.event.pull_request.head.sha }}
|
||||
check-name: Team Label
|
||||
repo-token: ${{ secrets.GITHUB_TOKEN }}
|
||||
wait-interval: 90
|
||||
|
||||
- name: Verify PR labels
|
||||
uses: jesusvasquez333/verify-pr-label-action@v1.4.0
|
||||
if: ${{ github.event_name == 'pull_request_target' }}
|
||||
with:
|
||||
github-token: "${{ secrets.GITHUB_TOKEN }}"
|
||||
valid-labels: "safe to test"
|
||||
pull-request-number: "${{ github.event.pull_request.number }}"
|
||||
disable-reviews: true
|
||||
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
ref: ${{ github.event_name == 'merge_group' && github.sha || github.event.pull_request.head.sha }}
|
||||
|
||||
- name: Setup JDK 21
|
||||
uses: actions/setup-java@v4
|
||||
with:
|
||||
java-version: '21'
|
||||
distribution: 'temurin'
|
||||
|
||||
- name: Cache Maven Dependencies
|
||||
uses: actions/cache@v4
|
||||
with:
|
||||
path: ~/.m2
|
||||
key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}
|
||||
restore-keys: |
|
||||
${{ runner.os }}-maven-
|
||||
|
||||
- name: Install antlr cli
|
||||
run: sudo make install_antlr_cli
|
||||
|
||||
- name: Build with Maven
|
||||
run: mvn -DskipTests clean package
|
||||
|
||||
- name: Upload Maven build artifact
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: openmetadata-build
|
||||
path: openmetadata-dist/target/openmetadata-*.tar.gz
|
||||
retention-days: 1
|
||||
|
||||
detect-changes:
|
||||
needs: check-changes
|
||||
runs-on: ubuntu-latest
|
||||
if: |
|
||||
!github.event.pull_request.draft &&
|
||||
(github.event_name == 'merge_group' || github.event_name == 'workflow_dispatch' ||
|
||||
needs.check-changes.outputs.e2e == 'true' || needs.check-changes.outputs.docker-compose == 'true') &&
|
||||
(github.event_name != 'pull_request_target' || github.event.action != 'labeled' || github.event.label.name == 'safe to test')
|
||||
outputs:
|
||||
spec_only_mode: ${{ steps.compute.outputs.spec_only_mode }}
|
||||
changed_specs: ${{ steps.compute.outputs.changed_specs }}
|
||||
matrix: ${{ steps.compute.outputs.matrix }}
|
||||
steps:
|
||||
- name: Checkout
|
||||
if: ${{ github.event_name == 'pull_request_target' }}
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
ref: ${{ github.event.pull_request.head.sha }}
|
||||
fetch-depth: 0
|
||||
filter: blob:none
|
||||
|
||||
- name: Get all changed files
|
||||
id: all-changes
|
||||
if: ${{ github.event_name == 'pull_request_target' }}
|
||||
uses: tj-actions/changed-files@9426d40962ed5378910ee2e21d5f8c6fcbf2dd96 # v47.0.6
|
||||
|
||||
- name: Get changed runnable spec files
|
||||
id: changed-specs
|
||||
if: ${{ github.event_name == 'pull_request_target' }}
|
||||
uses: tj-actions/changed-files@9426d40962ed5378910ee2e21d5f8c6fcbf2dd96 # v47.0.6
|
||||
with:
|
||||
path: openmetadata-ui/src/main/resources/ui
|
||||
files: |
|
||||
playwright/e2e/**/*.spec.ts
|
||||
files_ignore: |
|
||||
playwright/e2e/**/SystemCertificationTags.spec.ts
|
||||
playwright/e2e/**/IntakeForm.spec.ts
|
||||
|
||||
- name: Compute spec-only mode and matrix
|
||||
id: compute
|
||||
env:
|
||||
ALL_FILES: ${{ steps.all-changes.outputs.all_changed_files }}
|
||||
SPEC_FILES: ${{ steps.changed-specs.outputs.all_changed_files }}
|
||||
run: |
|
||||
FULL_MATRIX='{"shardIndex":[1,2,3,4,5,6,7],"shardTotal":[7]}'
|
||||
SPEC_MATRIX='{"shardIndex":[1],"shardTotal":[1]}'
|
||||
|
||||
ALL_COUNT=$(echo "$ALL_FILES" | wc -w)
|
||||
SPEC_COUNT=$(echo "$SPEC_FILES" | wc -w)
|
||||
|
||||
if [ "$SPEC_COUNT" -gt 0 ] && [ "$ALL_COUNT" -eq "$SPEC_COUNT" ]; then
|
||||
echo "spec_only_mode=true" >> "$GITHUB_OUTPUT"
|
||||
echo "changed_specs=$SPEC_FILES" >> "$GITHUB_OUTPUT"
|
||||
echo "matrix=$SPEC_MATRIX" >> "$GITHUB_OUTPUT"
|
||||
echo "Spec-only mode: ${SPEC_COUNT} spec(s) changed"
|
||||
else
|
||||
echo "spec_only_mode=false" >> "$GITHUB_OUTPUT"
|
||||
echo "changed_specs=" >> "$GITHUB_OUTPUT"
|
||||
echo "matrix=$FULL_MATRIX" >> "$GITHUB_OUTPUT"
|
||||
echo "Full suite mode (total=${ALL_COUNT}, runnable-specs=${SPEC_COUNT})"
|
||||
fi
|
||||
|
||||
playwright-ci-postgresql:
|
||||
needs: [build, detect-changes]
|
||||
runs-on: ubuntu-latest
|
||||
if: ${{ !cancelled() && needs.build.result == 'success' && needs.detect-changes.result == 'success' }}
|
||||
environment: test
|
||||
permissions:
|
||||
contents: read
|
||||
id-token: write # Required for GitHub OIDC (Flakiness.io reporter)
|
||||
env:
|
||||
# Playwright logs the admin user in many times (performAdminLogin per test, parallel
|
||||
# workers, retries). The production default of 5 active sessions per user would evict the
|
||||
# long-lived storageState session the page fixtures rely on and 401 every request. Raise
|
||||
# the cap for E2E only; docker compose reads this for ${AUTHENTICATION_MAX_ACTIVE_SESSIONS_PER_USER:-5}.
|
||||
AUTHENTICATION_MAX_ACTIVE_SESSIONS_PER_USER: "10000"
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix: ${{ fromJSON(needs.detect-changes.outputs.matrix) }}
|
||||
steps:
|
||||
- name: Free Disk Space (Ubuntu)
|
||||
uses: jlumbroso/free-disk-space@main
|
||||
with:
|
||||
tool-cache: false
|
||||
android: true
|
||||
dotnet: true
|
||||
haskell: true
|
||||
large-packages: false
|
||||
swap-storage: true
|
||||
docker-images: false
|
||||
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
ref: ${{ github.event_name == 'merge_group' && github.sha || github.event.pull_request.head.sha }}
|
||||
|
||||
- name: Download Maven build artifact
|
||||
uses: actions/download-artifact@v4
|
||||
with:
|
||||
name: openmetadata-build
|
||||
path: openmetadata-dist/target
|
||||
|
||||
- name: Setup Openmetadata Test Environment
|
||||
uses: ./.github/actions/setup-openmetadata-test-environment
|
||||
with:
|
||||
python-version: "3.10"
|
||||
args: "-d postgresql -s true"
|
||||
ingestion_dependency: "all"
|
||||
|
||||
- name: Setup Node.js
|
||||
uses: actions/setup-node@v4
|
||||
with:
|
||||
node-version-file: "openmetadata-ui/src/main/resources/ui/.nvmrc"
|
||||
|
||||
- name: Install dependencies
|
||||
working-directory: openmetadata-ui/src/main/resources/ui/
|
||||
run: yarn --ignore-scripts --frozen-lockfile
|
||||
- name: Install Playwright Browsers
|
||||
run: npx playwright@1.57.0 install chromium --with-deps
|
||||
- name: Run Playwright tests
|
||||
id: run-tests
|
||||
working-directory: openmetadata-ui/src/main/resources/ui/
|
||||
run: |
|
||||
if [ "$SPEC_ONLY" = "true" ]; then
|
||||
echo "🔹 Spec-only mode: running changed specs → ${CHANGED_SPECS}"
|
||||
|
||||
PROJECT_ARGS=""
|
||||
|
||||
# DataAssetRulesEnabled runs standalone; its deps (setup) are resolved automatically
|
||||
if echo "$CHANGED_SPECS" | tr ' ' '\n' | grep -q "DataAssetRulesEnabled\.spec\.ts"; then
|
||||
PROJECT_ARGS="$PROJECT_ARGS --project=DataAssetRulesEnabled"
|
||||
fi
|
||||
# DataAssetRulesDisabled depends on DataAssetRulesEnabled — Playwright resolves this automatically
|
||||
if echo "$CHANGED_SPECS" | tr ' ' '\n' | grep -q "DataAssetRulesDisabled\.spec\.ts"; then
|
||||
PROJECT_ARGS="$PROJECT_ARGS --project=DataAssetRulesDisabled"
|
||||
fi
|
||||
# SearchRBAC depends on the full DataAssetRules chain — Playwright resolves this automatically
|
||||
if echo "$CHANGED_SPECS" | tr ' ' '\n' | grep -q "SearchRBAC\.spec\.ts"; then
|
||||
PROJECT_ARGS="$PROJECT_ARGS --project=SearchRBAC"
|
||||
fi
|
||||
# DomainIsolation uses serial workers and its own testMatch
|
||||
if echo "$CHANGED_SPECS" | tr ' ' '\n' | grep -q "DomainIsolation/"; then
|
||||
PROJECT_ARGS="$PROJECT_ARGS --project=DomainIsolation"
|
||||
fi
|
||||
# Search relevance specs use the dedicated search-nightly project.
|
||||
if echo "$CHANGED_SPECS" | tr ' ' '\n' | grep -q "playwright/e2e/Search/"; then
|
||||
PROJECT_ARGS="$PROJECT_ARGS --project=search-nightly"
|
||||
fi
|
||||
# Regular chromium specs: anything not in a special-project category.
|
||||
# Also add Basic because chromium has grepInvert:[@basic], so @basic-tagged tests
|
||||
# in the changed files would be silently skipped without it.
|
||||
if echo "$CHANGED_SPECS" | tr ' ' '\n' | grep -qvE "(DataAssetRulesEnabled\.spec\.ts|DataAssetRulesDisabled\.spec\.ts|SearchRBAC\.spec\.ts|DomainIsolation/|playwright/e2e/Search/)"; then
|
||||
PROJECT_ARGS="$PROJECT_ARGS --project=chromium --project=Basic"
|
||||
fi
|
||||
|
||||
[ -z "$PROJECT_ARGS" ] && PROJECT_ARGS="--project=chromium --project=Basic"
|
||||
|
||||
npx playwright test $PROJECT_ARGS $CHANGED_SPECS
|
||||
|
||||
elif [ "${{ matrix.shardIndex }}" -eq "1" ]; then
|
||||
echo "🔹 Running DataAssetRules-only tests on shard 1"
|
||||
|
||||
# The testMatch pattern ensures only DataAssetRules*.spec.ts files run
|
||||
npx playwright test \
|
||||
--project=setup \
|
||||
--project=DataAssetRulesEnabled \
|
||||
--project=DataAssetRulesDisabled \
|
||||
--project=Basic \
|
||||
--project=SearchRBAC \
|
||||
--project=DomainIsolation \
|
||||
|
||||
elif [ "${{ matrix.shardIndex }}" -eq "2" ]; then
|
||||
echo "🔹 Running search relevance tests on shard 2"
|
||||
|
||||
npx playwright test \
|
||||
--project=search-nightly \
|
||||
playwright/e2e/Search/SearchRelevance.spec.ts \
|
||||
--workers=1
|
||||
|
||||
else
|
||||
# Shards 3-7 run common chromium tests equally distributed (5-way sharding)
|
||||
CHROMIUM_SHARD=$(( ${{ matrix.shardIndex }} - 2 ))
|
||||
echo "🔹 Running common chromium tests on shard ${CHROMIUM_SHARD}/5"
|
||||
npx playwright test \
|
||||
--project=chromium \
|
||||
--grep-invert @dataAssetRules \
|
||||
--shard=${CHROMIUM_SHARD}/5
|
||||
fi
|
||||
|
||||
env:
|
||||
SPEC_ONLY: ${{ needs.detect-changes.outputs.spec_only_mode }}
|
||||
CHANGED_SPECS: ${{ needs.detect-changes.outputs.changed_specs }}
|
||||
PLAYWRIGHT_IS_OSS: true
|
||||
PLAYWRIGHT_SNOWFLAKE_USERNAME: ${{ secrets.TEST_SNOWFLAKE_USERNAME }}
|
||||
PLAYWRIGHT_SNOWFLAKE_PASSWORD: ${{ secrets.TEST_SNOWFLAKE_PASSWORD }}
|
||||
PLAYWRIGHT_SNOWFLAKE_ACCOUNT: ${{ secrets.TEST_SNOWFLAKE_ACCOUNT }}
|
||||
PLAYWRIGHT_SNOWFLAKE_DATABASE: ${{ secrets.TEST_SNOWFLAKE_DATABASE }}
|
||||
PLAYWRIGHT_SNOWFLAKE_WAREHOUSE: ${{ secrets.TEST_SNOWFLAKE_WAREHOUSE }}
|
||||
PLAYWRIGHT_SNOWFLAKE_PASSPHRASE: ${{ secrets.TEST_SNOWFLAKE_PASSPHRASE }}
|
||||
PLAYWRIGHT_PROJECT_ID: ${{ steps.cypress-project-id.outputs.CYPRESS_PROJECT_ID }}
|
||||
PLAYWRIGHT_BQ_PRIVATE_KEY: ${{ secrets.TEST_BQ_PRIVATE_KEY }}
|
||||
PLAYWRIGHT_BQ_PROJECT_ID: ${{ secrets.PLAYWRIGHT_BQ_PROJECT_ID }}
|
||||
PLAYWRIGHT_BQ_PRIVATE_KEY_ID: ${{ secrets.TEST_BQ_PRIVATE_KEY_ID }}
|
||||
PLAYWRIGHT_BQ_PROJECT_ID_TAXONOMY: ${{ secrets.TEST_BQ_PROJECT_ID_TAXONOMY }}
|
||||
PLAYWRIGHT_BQ_CLIENT_EMAIL: ${{ secrets.TEST_BQ_CLIENT_EMAIL }}
|
||||
PLAYWRIGHT_BQ_CLIENT_ID: ${{ secrets.TEST_BQ_CLIENT_ID }}
|
||||
PLAYWRIGHT_REDSHIFT_HOST: ${{ secrets.E2E_REDSHIFT_HOST_PORT }}
|
||||
PLAYWRIGHT_REDSHIFT_USERNAME: ${{ secrets.E2E_REDSHIFT_USERNAME }}
|
||||
PLAYWRIGHT_REDSHIFT_PASSWORD: ${{ secrets.E2E_REDSHIFT_PASSWORD }}
|
||||
PLAYWRIGHT_REDSHIFT_DATABASE: ${{ secrets.TEST_REDSHIFT_DATABASE }}
|
||||
PLAYWRIGHT_METABASE_USERNAME: ${{ secrets.TEST_METABASE_USERNAME }}
|
||||
PLAYWRIGHT_METABASE_PASSWORD: ${{ secrets.TEST_METABASE_PASSWORD }}
|
||||
PLAYWRIGHT_METABASE_DB_SERVICE_NAME: ${{ secrets.TEST_METABASE_DB_SERVICE_NAME }}
|
||||
PLAYWRIGHT_METABASE_HOST_PORT: ${{ secrets.TEST_METABASE_HOST_PORT }}
|
||||
PLAYWRIGHT_SUPERSET_USERNAME: ${{ secrets.TEST_SUPERSET_USERNAME }}
|
||||
PLAYWRIGHT_SUPERSET_PASSWORD: ${{ secrets.TEST_SUPERSET_PASSWORD }}
|
||||
PLAYWRIGHT_SUPERSET_HOST_PORT: ${{ secrets.TEST_SUPERSET_HOST_PORT }}
|
||||
PLAYWRIGHT_KAFKA_BOOTSTRAP_SERVERS: ${{ secrets.TEST_KAFKA_BOOTSTRAP_SERVERS }}
|
||||
PLAYWRIGHT_KAFKA_SCHEMA_REGISTRY_URL: ${{ secrets.TEST_KAFKA_SCHEMA_REGISTRY_URL }}
|
||||
PLAYWRIGHT_GLUE_ACCESS_KEY: ${{ secrets.TEST_GLUE_ACCESS_KEY }}
|
||||
PLAYWRIGHT_GLUE_SECRET_KEY: ${{ secrets.TEST_GLUE_SECRET_KEY }}
|
||||
PLAYWRIGHT_GLUE_AWS_REGION: ${{ secrets.TEST_GLUE_AWS_REGION }}
|
||||
PLAYWRIGHT_GLUE_ENDPOINT: ${{ secrets.TEST_GLUE_ENDPOINT }}
|
||||
PLAYWRIGHT_GLUE_STORAGE_SERVICE: ${{ secrets.TEST_GLUE_STORAGE_SERVICE }}
|
||||
PLAYWRIGHT_MYSQL_USERNAME: ${{ secrets.TEST_MYSQL_USERNAME }}
|
||||
PLAYWRIGHT_MYSQL_PASSWORD: ${{ secrets.TEST_MYSQL_PASSWORD }}
|
||||
PLAYWRIGHT_MYSQL_HOST_PORT: ${{ secrets.TEST_MYSQL_HOST_PORT }}
|
||||
PLAYWRIGHT_MYSQL_DATABASE_SCHEMA: ${{ secrets.TEST_MYSQL_DATABASE_SCHEMA }}
|
||||
PLAYWRIGHT_POSTGRES_USERNAME: ${{ secrets.TEST_POSTGRES_USERNAME }}
|
||||
PLAYWRIGHT_POSTGRES_PASSWORD: ${{ secrets.TEST_POSTGRES_PASSWORD }}
|
||||
PLAYWRIGHT_POSTGRES_HOST_PORT: ${{ secrets.TEST_POSTGRES_HOST_PORT }}
|
||||
PLAYWRIGHT_POSTGRES_DATABASE: ${{ secrets.TEST_POSTGRES_DATABASE }}
|
||||
PLAYWRIGHT_AIRFLOW_HOST_PORT: ${{ secrets.TEST_AIRFLOW_HOST_PORT }}
|
||||
PLAYWRIGHT_ML_MODEL_TRACKING_URI: ${{ secrets.TEST_ML_MODEL_TRACKING_URI }}
|
||||
PLAYWRIGHT_ML_MODEL_REGISTRY_URI: ${{ secrets.TEST_ML_MODEL_REGISTRY_URI }}
|
||||
PLAYWRIGHT_S3_STORAGE_ACCESS_KEY_ID: ${{ secrets.TEST_S3_STORAGE_ACCESS_KEY_ID }}
|
||||
PLAYWRIGHT_S3_STORAGE_SECRET_ACCESS_KEY: ${{ secrets.TEST_S3_STORAGE_SECRET_ACCESS_KEY }}
|
||||
PLAYWRIGHT_S3_STORAGE_END_POINT_URL: ${{ secrets.TEST_S3_STORAGE_END_POINT_URL }}
|
||||
|
||||
# Recommended: pass the GitHub token lets this action correctly
|
||||
# determine the unique run id necessary to re-run the checks
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
- uses: actions/upload-artifact@v4
|
||||
if: ${{ !cancelled() }}
|
||||
with:
|
||||
name: playwright-report-${{ matrix.shardIndex }}
|
||||
path: openmetadata-ui/src/main/resources/ui/playwright/output/playwright-report
|
||||
retention-days: 5
|
||||
|
||||
- name: Upload test results (screenshots, traces)
|
||||
uses: actions/upload-artifact@v4
|
||||
if: ${{ !cancelled() }}
|
||||
with:
|
||||
name: playwright-test-results-${{ matrix.shardIndex }}
|
||||
path: openmetadata-ui/src/main/resources/ui/playwright/output/test-results
|
||||
retention-days: 5
|
||||
|
||||
- name: Upload results JSON for summary
|
||||
uses: actions/upload-artifact@v4
|
||||
if: ${{ !cancelled() }}
|
||||
with:
|
||||
name: playwright-results-json-${{ matrix.shardIndex }}
|
||||
path: openmetadata-ui/src/main/resources/ui/playwright/output/results.json
|
||||
retention-days: 1
|
||||
if-no-files-found: ignore
|
||||
|
||||
- name: Clean Up
|
||||
run: |
|
||||
cd ./docker/development
|
||||
docker compose down --remove-orphans
|
||||
sudo rm -rf ${PWD}/docker-volume
|
||||
|
||||
playwright-summary:
|
||||
if: always()
|
||||
needs: playwright-ci-postgresql
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
contents: read
|
||||
pull-requests: write # Posts the consolidated PR comment
|
||||
steps:
|
||||
- name: Download all results JSON
|
||||
if: needs.playwright-ci-postgresql.result != 'skipped'
|
||||
uses: actions/download-artifact@v4
|
||||
with:
|
||||
pattern: playwright-results-json-*
|
||||
path: results
|
||||
|
||||
- name: Post consolidated PR comment and gate on results
|
||||
uses: actions/github-script@v7
|
||||
with:
|
||||
github-token: ${{ secrets.GITHUB_TOKEN }}
|
||||
script: |
|
||||
const upstreamResult = '${{ needs.playwright-ci-postgresql.result }}';
|
||||
if (upstreamResult === 'skipped') {
|
||||
// Distinguish the two reasons build/playwright can be skipped:
|
||||
// 1. A non-"safe to test" label was added — build skips intentionally for fork security.
|
||||
// Tests will run once "safe to test" is applied or code is pushed.
|
||||
// 2. No relevant paths changed — nothing to test.
|
||||
const eventAction = context.payload.action;
|
||||
const labelName = context.payload.label?.name ?? '';
|
||||
if (eventAction === 'labeled' && labelName !== 'safe to test') {
|
||||
console.log(`Playwright E2E tests pending — label "${labelName}" added but only "safe to test" triggers E2E. Tests will run when "safe to test" is applied or code is pushed.`);
|
||||
} else {
|
||||
console.log('Playwright E2E tests not required for this PR (no relevant paths changed).');
|
||||
}
|
||||
return;
|
||||
}
|
||||
if (upstreamResult === 'cancelled') {
|
||||
core.setFailed('Playwright E2E tests were cancelled.');
|
||||
return;
|
||||
}
|
||||
|
||||
const fs = require('fs');
|
||||
const path = require('path');
|
||||
|
||||
const runId = '${{ github.run_id }}';
|
||||
const repo = context.repo;
|
||||
const prNumber = context.payload.pull_request?.number;
|
||||
|
||||
const artifactUrl = `https://github.com/${repo.owner}/${repo.repo}/actions/runs/${runId}`;
|
||||
const commentMarker = '<!-- playwright-summary -->';
|
||||
|
||||
// Collect results from all shards
|
||||
const shardResults = [];
|
||||
const resultsDir = 'results';
|
||||
if (fs.existsSync(resultsDir)) {
|
||||
for (const dir of fs.readdirSync(resultsDir).sort()) {
|
||||
const jsonPath = path.join(resultsDir, dir, 'results.json');
|
||||
if (!fs.existsSync(jsonPath)) continue;
|
||||
|
||||
const shardNum = dir.replace('playwright-results-json-', '');
|
||||
const report = JSON.parse(fs.readFileSync(jsonPath, 'utf8'));
|
||||
|
||||
const allTests = [];
|
||||
function collectTests(suite, filePath) {
|
||||
const file = suite.file || filePath || '';
|
||||
for (const spec of (suite.specs || [])) {
|
||||
for (const test of (spec.tests || [])) {
|
||||
const results = test.results || [];
|
||||
const lastResult = results[results.length - 1] || {};
|
||||
const firstResult = results[0] || {};
|
||||
allTests.push({
|
||||
title: spec.title,
|
||||
file: file,
|
||||
status: test.status,
|
||||
retries: results.length - 1,
|
||||
error: lastResult.error?.message || firstResult.error?.message || '',
|
||||
});
|
||||
}
|
||||
}
|
||||
for (const child of (suite.suites || [])) {
|
||||
collectTests(child, file);
|
||||
}
|
||||
}
|
||||
for (const suite of (report.suites || [])) {
|
||||
collectTests(suite, '');
|
||||
}
|
||||
|
||||
shardResults.push({
|
||||
shard: shardNum,
|
||||
genuine: allTests.filter(t => t.status === 'unexpected'),
|
||||
flaky: allTests.filter(t => t.status === 'flaky'),
|
||||
passed: allTests.filter(t => t.status === 'expected'),
|
||||
skipped: allTests.filter(t => t.status === 'skipped'),
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
if (shardResults.length === 0) {
|
||||
core.setFailed('No Playwright results found — shards may have been cancelled or failed to upload artifacts.');
|
||||
return;
|
||||
}
|
||||
|
||||
// Aggregate totals
|
||||
const totalPassed = shardResults.reduce((s, r) => s + r.passed.length, 0);
|
||||
const totalFailed = shardResults.reduce((s, r) => s + r.genuine.length, 0);
|
||||
const totalFlaky = shardResults.reduce((s, r) => s + r.flaky.length, 0);
|
||||
const totalSkipped = shardResults.reduce((s, r) => s + r.skipped.length, 0);
|
||||
|
||||
const lines = [commentMarker];
|
||||
|
||||
if (totalFailed > 0) {
|
||||
lines.push(`## 🔴 Playwright Results — ${totalFailed} failure(s)${totalFlaky > 0 ? `, ${totalFlaky} flaky` : ''}`);
|
||||
} else if (totalFlaky > 0) {
|
||||
lines.push(`## 🟡 Playwright Results — all passed (${totalFlaky} flaky)`);
|
||||
} else {
|
||||
lines.push(`## ✅ Playwright Results — all ${totalPassed} tests passed`);
|
||||
}
|
||||
lines.push('');
|
||||
lines.push(`✅ ${totalPassed} passed · ❌ ${totalFailed} failed · 🟡 ${totalFlaky} flaky · ⏭️ ${totalSkipped} skipped`);
|
||||
lines.push('');
|
||||
|
||||
// Per-shard summary table
|
||||
lines.push('| Shard | Passed | Failed | Flaky | Skipped |');
|
||||
lines.push('|-------|--------|--------|-------|---------|');
|
||||
for (const r of shardResults) {
|
||||
const status = r.genuine.length > 0 ? '🔴' : r.flaky.length > 0 ? '🟡' : '✅';
|
||||
lines.push(`| ${status} Shard ${r.shard} | ${r.passed.length} | ${r.genuine.length} | ${r.flaky.length} | ${r.skipped.length} |`);
|
||||
}
|
||||
lines.push('');
|
||||
|
||||
// Genuine failures detail
|
||||
const allGenuine = shardResults.flatMap(r => r.genuine.map(t => ({ ...t, shard: r.shard })));
|
||||
if (allGenuine.length > 0) {
|
||||
lines.push('### Genuine Failures (failed on all attempts)');
|
||||
lines.push('');
|
||||
for (const t of allGenuine.slice(0, 30)) {
|
||||
const shortFile = t.file.replace(/.*playwright\/e2e\//, '');
|
||||
lines.push(`<details><summary>❌ <code>${shortFile}</code> › ${t.title} (shard ${t.shard})</summary>`);
|
||||
lines.push('');
|
||||
lines.push('```');
|
||||
lines.push(t.error.substring(0, 1000));
|
||||
lines.push('```');
|
||||
lines.push('</details>');
|
||||
lines.push('');
|
||||
}
|
||||
if (allGenuine.length > 30) {
|
||||
lines.push(`... and ${allGenuine.length - 30} more failures`);
|
||||
lines.push('');
|
||||
}
|
||||
}
|
||||
|
||||
// Flaky tests
|
||||
const allFlaky = shardResults.flatMap(r => r.flaky.map(t => ({ ...t, shard: r.shard })));
|
||||
if (allFlaky.length > 0) {
|
||||
lines.push(`<details><summary>🟡 ${allFlaky.length} flaky test(s) (passed on retry)</summary>`);
|
||||
lines.push('');
|
||||
for (const t of allFlaky.slice(0, 30)) {
|
||||
const shortFile = t.file.replace(/.*playwright\/e2e\//, '');
|
||||
lines.push(`- \`${shortFile}\` › ${t.title} (shard ${t.shard}, ${t.retries} ${t.retries === 1 ? 'retry' : 'retries'})`);
|
||||
}
|
||||
if (allFlaky.length > 30) {
|
||||
lines.push(`- ... and ${allFlaky.length - 30} more`);
|
||||
}
|
||||
lines.push('');
|
||||
lines.push('</details>');
|
||||
lines.push('');
|
||||
}
|
||||
|
||||
lines.push(`📦 [Download artifacts](${artifactUrl})`);
|
||||
lines.push('');
|
||||
lines.push('<details><summary>How to debug locally</summary>');
|
||||
lines.push('');
|
||||
lines.push('```bash');
|
||||
lines.push('# Download playwright-test-results-<shard> artifact and unzip');
|
||||
lines.push('npx playwright show-trace path/to/trace.zip # view trace');
|
||||
lines.push('```');
|
||||
lines.push('</details>');
|
||||
|
||||
const body = lines.join('\n');
|
||||
|
||||
// Post PR comment only for pull_request_target events
|
||||
if (prNumber) {
|
||||
let existingComment = null;
|
||||
for await (const response of github.paginate.iterator(
|
||||
github.rest.issues.listComments, { ...repo, issue_number: prNumber, per_page: 100 }
|
||||
)) {
|
||||
const found = response.data.find(c =>
|
||||
c.user?.login === 'github-actions[bot]' && c.body?.includes(commentMarker)
|
||||
);
|
||||
if (found) { existingComment = found; break; }
|
||||
}
|
||||
|
||||
// Also clean up any old per-shard comments from previous runs
|
||||
for await (const response of github.paginate.iterator(
|
||||
github.rest.issues.listComments, { ...repo, issue_number: prNumber, per_page: 100 }
|
||||
)) {
|
||||
for (const c of response.data) {
|
||||
if (c.user?.login === 'github-actions[bot]' && /Playwright Shard \d/.test(c.body || '') && !c.body?.includes(commentMarker)) {
|
||||
await github.rest.issues.deleteComment({ ...repo, comment_id: c.id });
|
||||
console.log(`Deleted old per-shard comment ${c.id}`);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (existingComment) {
|
||||
await github.rest.issues.updateComment({ ...repo, comment_id: existingComment.id, body });
|
||||
} else {
|
||||
await github.rest.issues.createComment({ ...repo, issue_number: prNumber, body });
|
||||
}
|
||||
}
|
||||
|
||||
// Gate: fail the step (and therefore the job) when genuine test failures exist.
|
||||
// This is the single source of truth — the same parsed results that drive the
|
||||
// PR comment also determine whether playwright-summary passes or fails.
|
||||
if (totalFailed > 0) {
|
||||
core.setFailed(`${totalFailed} Playwright test(s) failed.`);
|
||||
}
|
||||
@@ -0,0 +1,104 @@
|
||||
# Copyright 2026 Collate
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
name: playwright-search-nightly
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
concurrency:
|
||||
group: playwright-search-nightly-${{ github.ref }}
|
||||
cancel-in-progress: true
|
||||
|
||||
jobs:
|
||||
playwright-search-nightly:
|
||||
runs-on: ubuntu-latest
|
||||
environment: test
|
||||
timeout-minutes: 45
|
||||
steps:
|
||||
- name: Free Disk Space (Ubuntu)
|
||||
uses: jlumbroso/free-disk-space@main
|
||||
with:
|
||||
tool-cache: false
|
||||
android: true
|
||||
dotnet: true
|
||||
haskell: true
|
||||
large-packages: false
|
||||
swap-storage: true
|
||||
docker-images: false
|
||||
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Cache Maven Dependencies
|
||||
uses: actions/cache@v4
|
||||
with:
|
||||
path: ~/.m2
|
||||
key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}
|
||||
restore-keys: |
|
||||
${{ runner.os }}-maven-
|
||||
|
||||
- name: Setup OpenMetadata Test Environment
|
||||
uses: ./.github/actions/setup-openmetadata-test-environment
|
||||
with:
|
||||
python-version: '3.10'
|
||||
args: '-d postgresql -i false'
|
||||
ingestion_dependency: 'all'
|
||||
|
||||
- name: Setup Node.js
|
||||
uses: actions/setup-node@v4
|
||||
with:
|
||||
node-version-file: 'openmetadata-ui/src/main/resources/ui/.nvmrc'
|
||||
|
||||
- name: Install dependencies
|
||||
working-directory: openmetadata-ui/src/main/resources/ui/
|
||||
run: yarn --ignore-scripts --frozen-lockfile
|
||||
|
||||
- name: Install Playwright Browsers
|
||||
run: npx playwright@1.57.0 install chromium --with-deps
|
||||
|
||||
- name: Run Search Nightly
|
||||
working-directory: openmetadata-ui/src/main/resources/ui
|
||||
env:
|
||||
PLAYWRIGHT_IS_OSS: true
|
||||
run: |
|
||||
# All search tests live in playwright/e2e/Search/. The search-nightly
|
||||
# project in playwright.config.ts maps testMatch to **/Search/** so only
|
||||
# that folder is picked up. Add new search specs to that folder.
|
||||
npx playwright test --project=search-nightly --workers=1
|
||||
|
||||
- name: Upload HTML report
|
||||
if: always()
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: search-nightly-html-report
|
||||
path: openmetadata-ui/src/main/resources/ui/playwright/output/playwright-report
|
||||
retention-days: 5
|
||||
|
||||
- name: Send Slack Notification
|
||||
if: always()
|
||||
working-directory: openmetadata-ui/src/main/resources/ui
|
||||
env:
|
||||
RUN_TITLE: "Playwright Search Nightly (${{ github.ref_name }})"
|
||||
RUN_URL: "${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"
|
||||
SLACK_BOT_USER_OAUTH_TOKEN: ${{ secrets.E2E_SLACK_BOT_OAUTH_TOKEN }}
|
||||
run: |
|
||||
npx playwright-slack-report -c playwright/slack-cli.config.json -j playwright/output/results.json > slack_report.json
|
||||
|
||||
- name: Clean Up
|
||||
if: always()
|
||||
run: |
|
||||
cd ./docker/development
|
||||
docker compose down --remove-orphans
|
||||
sudo rm -rf ${PWD}/docker-volume
|
||||
@@ -0,0 +1,155 @@
|
||||
# Copyright 2025 Collate
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
name: SSO Login Nightly
|
||||
|
||||
on:
|
||||
schedule:
|
||||
- cron: '0 3 * * *'
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
sso_provider:
|
||||
description: 'SSO provider (or "all")'
|
||||
required: true
|
||||
default: okta
|
||||
type: choice
|
||||
options:
|
||||
- okta
|
||||
- keycloak-azure-saml
|
||||
- keycloak-azure-saml-crosssite
|
||||
- all
|
||||
send_slack_notification:
|
||||
description: "Send Slack notification with test results"
|
||||
required: false
|
||||
type: boolean
|
||||
default: false
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
concurrency:
|
||||
group: sso-login-nightly-${{ github.event.inputs.sso_provider || 'scheduled' }}
|
||||
cancel-in-progress: true
|
||||
|
||||
jobs:
|
||||
# To onboard a new provider:
|
||||
# 1. Add a matrix entry below (`name` is the lowercase provider id used by
|
||||
# the Playwright helper; `env_prefix` is the uppercase/underscore form
|
||||
# used to look up credentials). Also add `name` to the dispatch
|
||||
# `options:` list above.
|
||||
# 2. Add <ENV_PREFIX>_SSO_USERNAME (variable) and <ENV_PREFIX>_SSO_PASSWORD
|
||||
# (variable) to the `test` environment. Use a secret instead of a
|
||||
# variable for the password if the provider uses a real (non-fixture)
|
||||
# credential.
|
||||
# 3. Register the helper in playwright/utils/sso-providers/index.ts.
|
||||
sso-login:
|
||||
runs-on: ubuntu-latest
|
||||
environment: test
|
||||
timeout-minutes: 45
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
provider:
|
||||
${{ (github.event_name == 'schedule' || github.event.inputs.sso_provider == 'all')
|
||||
&& fromJSON('[{"name":"okta","env_prefix":"OKTA"},{"name":"keycloak-azure-saml","env_prefix":"KEYCLOAK_AZURE_SAML"},{"name":"keycloak-azure-saml-crosssite","env_prefix":"KEYCLOAK_AZURE_SAML"}]')
|
||||
|| (github.event.inputs.sso_provider == 'keycloak-azure-saml'
|
||||
&& fromJSON('[{"name":"keycloak-azure-saml","env_prefix":"KEYCLOAK_AZURE_SAML"}]')
|
||||
|| (github.event.inputs.sso_provider == 'keycloak-azure-saml-crosssite'
|
||||
&& fromJSON('[{"name":"keycloak-azure-saml-crosssite","env_prefix":"KEYCLOAK_AZURE_SAML"}]')
|
||||
|| fromJSON('[{"name":"okta","env_prefix":"OKTA"}]'))) }}
|
||||
steps:
|
||||
- name: Free Disk Space (Ubuntu)
|
||||
uses: jlumbroso/free-disk-space@main
|
||||
with:
|
||||
tool-cache: false
|
||||
android: true
|
||||
dotnet: true
|
||||
haskell: true
|
||||
large-packages: false
|
||||
swap-storage: true
|
||||
docker-images: false
|
||||
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Cache Maven Dependencies
|
||||
uses: actions/cache@v4
|
||||
with:
|
||||
path: ~/.m2
|
||||
key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}
|
||||
restore-keys: |
|
||||
${{ runner.os }}-maven-
|
||||
|
||||
- name: Setup OpenMetadata Test Environment
|
||||
uses: ./.github/actions/setup-openmetadata-test-environment
|
||||
with:
|
||||
python-version: '3.10'
|
||||
args: '-d postgresql -i false'
|
||||
ingestion_dependency: 'all'
|
||||
|
||||
- name: Setup Node.js
|
||||
uses: actions/setup-node@v4
|
||||
with:
|
||||
node-version-file: 'openmetadata-ui/src/main/resources/ui/.nvmrc'
|
||||
|
||||
- name: Install dependencies
|
||||
working-directory: openmetadata-ui/src/main/resources/ui/
|
||||
run: yarn --ignore-scripts --frozen-lockfile
|
||||
|
||||
- name: Install Playwright Browsers
|
||||
run: npx playwright@1.57.0 install chromium --with-deps
|
||||
|
||||
- name: Start Keycloak SAML IdP
|
||||
if: startsWith(matrix.provider.name, 'keycloak-')
|
||||
run: |
|
||||
docker compose -f docker/local-sso/keycloak-saml/docker-compose.yml up -d
|
||||
timeout 180 bash -c 'until curl -fsS http://localhost:8080/realms/om-azure-saml >/dev/null; do sleep 2; done'
|
||||
|
||||
- name: Run SSO Login Spec
|
||||
working-directory: openmetadata-ui/src/main/resources/ui
|
||||
env:
|
||||
SSO_PROVIDER_TYPE: ${{ matrix.provider.name }}
|
||||
SSO_USERNAME: ${{ vars[format('{0}_SSO_USERNAME', matrix.provider.env_prefix)] }}
|
||||
SSO_PASSWORD: ${{ vars[format('{0}_SSO_PASSWORD', matrix.provider.env_prefix)] || secrets[format('{0}_SSO_PASSWORD', matrix.provider.env_prefix)] }}
|
||||
# The '-crosssite' variant fronts the IdP on 127.0.0.1 while OM stays on localhost. The two
|
||||
# are a same registrable-domain pair but a *different site* (SameSite ignores port; 127.0.0.1
|
||||
# and localhost are distinct sites), so the SAML callback POST is cross-site and the browser
|
||||
# drops the SameSite=Lax OM_SESSION cookie. This reproduces the production "No pending session"
|
||||
# failure that the localhost-only job cannot, guarding the SAML RelayState fix.
|
||||
KEYCLOAK_SAML_BASE_URL: ${{ matrix.provider.name == 'keycloak-azure-saml-crosssite' && 'http://127.0.0.1:8080' || 'http://localhost:8080' }}
|
||||
PLAYWRIGHT_IS_OSS: true
|
||||
run: npx playwright test --project=sso-auth --workers=1
|
||||
|
||||
- name: Upload HTML report
|
||||
if: always()
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: sso-login-html-report-${{ matrix.provider.name }}
|
||||
path: openmetadata-ui/src/main/resources/ui/playwright/output/playwright-report
|
||||
retention-days: 5
|
||||
|
||||
- name: Send Slack Notification
|
||||
if: ${{ always() && (github.event_name == 'schedule' || inputs.send_slack_notification == true) }}
|
||||
working-directory: openmetadata-ui/src/main/resources/ui
|
||||
env:
|
||||
RUN_TITLE: "SSO Login Nightly: ${{ matrix.provider.name }} (${{ github.ref_name }})"
|
||||
RUN_URL: "${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"
|
||||
SLACK_BOT_USER_OAUTH_TOKEN: ${{ secrets.E2E_SLACK_BOT_OAUTH_TOKEN }}
|
||||
run: |
|
||||
npx playwright-slack-report -c playwright/slack-cli.config.json -j playwright/output/results.json > slack_report.json
|
||||
|
||||
- name: Clean Up
|
||||
if: always()
|
||||
run: |
|
||||
docker compose -f docker/local-sso/keycloak-saml/docker-compose.yml down --remove-orphans || true
|
||||
cd ./docker/development
|
||||
docker compose down --remove-orphans
|
||||
sudo rm -rf ${PWD}/docker-volume
|
||||
@@ -0,0 +1,357 @@
|
||||
# Copyright 2025 Collate
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
# This workflow executes SSO-specific end-to-end (e2e) tests using Playwright with PostgreSQL as the database.
|
||||
#
|
||||
# Purpose:
|
||||
# - Run SSO token renewal E2E tests using a mock OIDC provider on PRs
|
||||
# - Run SSO configuration tests for various providers (Google, Azure AD, Okta, etc.) via manual trigger
|
||||
# - Tests run in isolation from the regular sharded Playwright E2E tests
|
||||
#
|
||||
# Triggers:
|
||||
# - Pull requests with "safe to test" label (mock-oidc only, Chromium + WebKit)
|
||||
# - Manual trigger via workflow_dispatch (any SSO provider, Chromium + WebKit)
|
||||
#
|
||||
# Test Location:
|
||||
# - openmetadata-ui/src/main/resources/ui/playwright/e2e/Auth/SSOAuthentication.spec.ts (mock-oidc)
|
||||
|
||||
name: SSO Playwright E2E Tests
|
||||
|
||||
on:
|
||||
pull_request_target:
|
||||
types:
|
||||
- labeled
|
||||
- opened
|
||||
- synchronize
|
||||
- reopened
|
||||
- ready_for_review
|
||||
paths:
|
||||
- "openmetadata-ui/src/main/resources/ui/src/components/Auth/**"
|
||||
- "openmetadata-ui/src/main/resources/ui/src/rest/auth-API*"
|
||||
- "openmetadata-ui/src/main/resources/ui/src/utils/AuthProvider*"
|
||||
- "openmetadata-ui/src/main/resources/ui/src/utils/TokenServiceUtil*"
|
||||
- "openmetadata-ui/src/main/resources/ui/src/utils/UserDataUtils*"
|
||||
- "openmetadata-ui/src/main/resources/ui/src/hooks/useSignIn*"
|
||||
- "openmetadata-ui/src/main/resources/ui/playwright/e2e/Auth/**"
|
||||
- "openmetadata-ui/src/main/resources/ui/playwright/utils/mockOidc*"
|
||||
- "openmetadata-ui/src/main/resources/ui/playwright.sso.config.ts"
|
||||
- "docker/development/mock-oidc-provider/**"
|
||||
- "docker/development/docker-compose.yml"
|
||||
- "docker/development/.env.sso-test"
|
||||
- ".github/workflows/playwright-sso-tests.yml"
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
sso_provider:
|
||||
description: "SSO Provider to test"
|
||||
required: true
|
||||
default: "mock-oidc"
|
||||
type: choice
|
||||
options:
|
||||
- mock-oidc
|
||||
- google
|
||||
- okta
|
||||
- azure
|
||||
- auth0
|
||||
- saml
|
||||
- cognito
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
pull-requests: write
|
||||
|
||||
concurrency:
|
||||
group: sso-playwright-${{ github.event.pull_request.number || github.run_id }}
|
||||
cancel-in-progress: ${{ github.event_name != 'pull_request_target' || github.event.action != 'labeled' || github.event.label.name == 'safe to test' }}
|
||||
|
||||
jobs:
|
||||
sso-auth-tests:
|
||||
runs-on: ubuntu-latest
|
||||
if: ${{ !github.event.pull_request.draft && (github.event_name != 'pull_request_target' || github.event.action != 'labeled' || github.event.label.name == 'safe to test') }}
|
||||
environment: test
|
||||
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
provider: ${{ github.event_name == 'pull_request_target' && fromJSON('["mock-oidc"]') || github.event.inputs.sso_provider && fromJSON(format('["{0}"]', github.event.inputs.sso_provider)) || fromJSON('["mock-oidc"]') }}
|
||||
|
||||
steps:
|
||||
- name: Free Disk Space (Ubuntu)
|
||||
uses: jlumbroso/free-disk-space@main
|
||||
with:
|
||||
tool-cache: false
|
||||
android: true
|
||||
dotnet: true
|
||||
haskell: true
|
||||
large-packages: false
|
||||
swap-storage: true
|
||||
docker-images: false
|
||||
|
||||
- name: Wait for the labeler
|
||||
uses: lewagon/wait-on-check-action@v1.7.0
|
||||
if: ${{ github.event_name == 'pull_request_target' }}
|
||||
with:
|
||||
ref: ${{ github.event.pull_request.head.sha }}
|
||||
check-name: Team Label
|
||||
repo-token: ${{ secrets.GITHUB_TOKEN }}
|
||||
wait-interval: 90
|
||||
|
||||
- name: Verify PR labels
|
||||
uses: jesusvasquez333/verify-pr-label-action@v1.4.0
|
||||
if: ${{ github.event_name == 'pull_request_target' }}
|
||||
with:
|
||||
github-token: "${{ secrets.GITHUB_TOKEN }}"
|
||||
valid-labels: "safe to test"
|
||||
pull-request-number: "${{ github.event.pull_request.number }}"
|
||||
disable-reviews: true
|
||||
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
ref: ${{ github.event.pull_request.head.sha }}
|
||||
|
||||
- name: Cache Maven Dependencies
|
||||
id: cache-output
|
||||
uses: actions/cache@v4
|
||||
with:
|
||||
path: ~/.m2
|
||||
key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}
|
||||
restore-keys: |
|
||||
${{ runner.os }}-maven-
|
||||
|
||||
- name: Start Mock OIDC Provider
|
||||
if: ${{ matrix.provider == 'mock-oidc' }}
|
||||
run: |
|
||||
cd docker/development
|
||||
docker compose --profile sso-test build mock-oidc-provider
|
||||
docker compose --profile sso-test up -d mock-oidc-provider
|
||||
echo "Waiting for mock OIDC provider to be healthy..."
|
||||
for i in $(seq 1 30); do
|
||||
if curl -sf http://localhost:9090/health > /dev/null 2>&1; then
|
||||
echo "Mock OIDC provider is ready"
|
||||
break
|
||||
fi
|
||||
echo "Waiting... ($i/30)"
|
||||
sleep 2
|
||||
done
|
||||
curl -sf http://localhost:9090/.well-known/openid-configuration || echo "WARNING: Discovery endpoint not ready"
|
||||
|
||||
- name: Setup Openmetadata Test Environment
|
||||
uses: ./.github/actions/setup-openmetadata-test-environment
|
||||
with:
|
||||
python-version: "3.10"
|
||||
args: "-d postgresql -i false"
|
||||
ingestion_dependency: "all"
|
||||
env:
|
||||
AUTHENTICATION_PROVIDER: ${{ matrix.provider == 'mock-oidc' && 'custom-oidc' || 'basic' }}
|
||||
CUSTOM_OIDC_AUTHENTICATION_PROVIDER_NAME: ${{ matrix.provider == 'mock-oidc' && 'mock-oidc' || '' }}
|
||||
AUTHENTICATION_PUBLIC_KEYS: ${{ matrix.provider == 'mock-oidc' && '[http://localhost:9090/jwks,http://localhost:8585/api/v1/system/config/jwks]' || '[http://localhost:8585/api/v1/system/config/jwks]' }}
|
||||
AUTHENTICATION_AUTHORITY: ${{ matrix.provider == 'mock-oidc' && 'http://localhost:9090' || 'https://accounts.google.com' }}
|
||||
AUTHENTICATION_CLIENT_ID: ${{ matrix.provider == 'mock-oidc' && 'openmetadata-test' || '' }}
|
||||
AUTHENTICATION_CALLBACK_URL: ${{ matrix.provider == 'mock-oidc' && 'http://localhost:8585/callback' || '' }}
|
||||
AUTHENTICATION_JWT_PRINCIPAL_CLAIMS_MAPPING: ${{ matrix.provider == 'mock-oidc' && '[username:sub,email:email]' || '' }}
|
||||
OIDC_CLIENT_ID: ${{ matrix.provider == 'mock-oidc' && 'openmetadata-test' || '' }}
|
||||
OIDC_TYPE: ${{ matrix.provider == 'mock-oidc' && 'custom-oidc' || '' }}
|
||||
OIDC_CLIENT_SECRET: ${{ matrix.provider == 'mock-oidc' && 'openmetadata-test-secret' || '' }}
|
||||
OIDC_DISCOVERY_URI: ${{ matrix.provider == 'mock-oidc' && 'http://localhost:9090/.well-known/openid-configuration' || '' }}
|
||||
OIDC_CALLBACK: ${{ matrix.provider == 'mock-oidc' && 'http://localhost:8585/callback' || 'http://localhost:8585/callback' }}
|
||||
OIDC_SERVER_URL: ${{ matrix.provider == 'mock-oidc' && 'http://localhost:8585' || 'http://localhost:8585' }}
|
||||
AUTHENTICATION_CLIENT_TYPE: ${{ matrix.provider == 'mock-oidc' && 'confidential' || 'public' }}
|
||||
|
||||
- name: Setup Node.js
|
||||
uses: actions/setup-node@v4
|
||||
with:
|
||||
node-version-file: 'openmetadata-ui/src/main/resources/ui/.nvmrc'
|
||||
|
||||
- name: Install dependencies
|
||||
working-directory: openmetadata-ui/src/main/resources/ui/
|
||||
run: yarn --ignore-scripts --frozen-lockfile
|
||||
|
||||
- name: Install Playwright Browsers
|
||||
run: npx playwright@1.51.1 install chromium webkit --with-deps
|
||||
|
||||
- name: Run Mock OIDC Authentication Tests
|
||||
if: ${{ matrix.provider == 'mock-oidc' }}
|
||||
working-directory: openmetadata-ui/src/main/resources/ui
|
||||
run: npx playwright test --config=playwright.sso.config.ts --workers=1
|
||||
env:
|
||||
MOCK_OIDC_URL: http://localhost:9090
|
||||
PLAYWRIGHT_IS_OSS: true
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
timeout-minutes: 60
|
||||
|
||||
- name: Run SSO Authentication Tests
|
||||
if: ${{ matrix.provider != 'mock-oidc' }}
|
||||
working-directory: openmetadata-ui/src/main/resources/ui
|
||||
run: npx playwright test --config=playwright.sso.config.ts --workers=1
|
||||
env:
|
||||
SSO_PROVIDER_TYPE: ${{ matrix.provider }}
|
||||
SSO_USERNAME: ${{ secrets[format('{0}_SSO_USERNAME', upper(matrix.provider))] }}
|
||||
SSO_PASSWORD: ${{ secrets[format('{0}_SSO_PASSWORD', upper(matrix.provider))] }}
|
||||
PLAYWRIGHT_IS_OSS: true
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
timeout-minutes: 60
|
||||
|
||||
- name: Upload test results
|
||||
if: always()
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: sso-auth-test-results-${{ matrix.provider }}
|
||||
path: |
|
||||
openmetadata-ui/src/main/resources/ui/playwright/output/sso-playwright-report
|
||||
openmetadata-ui/src/main/resources/ui/playwright/output/sso-test-results
|
||||
retention-days: 5
|
||||
|
||||
- name: Upload results JSON for summary
|
||||
uses: actions/upload-artifact@v4
|
||||
if: ${{ !cancelled() }}
|
||||
with:
|
||||
name: sso-results-json-${{ matrix.provider }}
|
||||
path: openmetadata-ui/src/main/resources/ui/playwright/output/sso-results.json
|
||||
retention-days: 1
|
||||
if-no-files-found: ignore
|
||||
|
||||
- name: Clean Up
|
||||
run: |
|
||||
cd ./docker/development
|
||||
docker compose --profile sso-test down --remove-orphans 2>/dev/null || true
|
||||
docker compose down --remove-orphans
|
||||
sudo rm -rf ${PWD}/docker-volume
|
||||
|
||||
sso-summary:
|
||||
if: ${{ !cancelled() && github.event_name == 'pull_request_target' && (github.event.action != 'labeled' || github.event.label.name == 'safe to test') }}
|
||||
needs: sso-auth-tests
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Download results JSON
|
||||
uses: actions/download-artifact@v4
|
||||
with:
|
||||
pattern: sso-results-json-*
|
||||
path: results
|
||||
|
||||
- name: Post SSO test summary as PR comment
|
||||
uses: actions/github-script@v7
|
||||
with:
|
||||
github-token: ${{ secrets.GITHUB_TOKEN }}
|
||||
script: |
|
||||
const fs = require('fs');
|
||||
const path = require('path');
|
||||
|
||||
const runId = '${{ github.run_id }}';
|
||||
const repo = context.repo;
|
||||
const prNumber = context.payload.pull_request?.number;
|
||||
if (!prNumber) return;
|
||||
|
||||
const artifactUrl = `https://github.com/${repo.owner}/${repo.repo}/actions/runs/${runId}`;
|
||||
const commentMarker = '<!-- sso-playwright-summary -->';
|
||||
|
||||
const allTests = [];
|
||||
const resultsDir = 'results';
|
||||
if (fs.existsSync(resultsDir)) {
|
||||
for (const dir of fs.readdirSync(resultsDir).sort()) {
|
||||
const jsonPath = path.join(resultsDir, dir, 'sso-results.json');
|
||||
if (!fs.existsSync(jsonPath)) continue;
|
||||
|
||||
const report = JSON.parse(fs.readFileSync(jsonPath, 'utf8'));
|
||||
|
||||
function collectTests(suite, filePath) {
|
||||
const file = suite.file || filePath || '';
|
||||
for (const spec of (suite.specs || [])) {
|
||||
for (const test of (spec.tests || [])) {
|
||||
const results = test.results || [];
|
||||
const lastResult = results[results.length - 1] || {};
|
||||
const firstResult = results[0] || {};
|
||||
allTests.push({
|
||||
title: spec.title,
|
||||
project: test.projectName || '',
|
||||
file: file,
|
||||
status: test.status,
|
||||
retries: results.length - 1,
|
||||
error: lastResult.error?.message || firstResult.error?.message || '',
|
||||
});
|
||||
}
|
||||
}
|
||||
for (const child of (suite.suites || [])) {
|
||||
collectTests(child, file);
|
||||
}
|
||||
}
|
||||
for (const suite of (report.suites || [])) {
|
||||
collectTests(suite, '');
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (allTests.length === 0) {
|
||||
console.log('No SSO test results found');
|
||||
return;
|
||||
}
|
||||
|
||||
const passed = allTests.filter(t => t.status === 'expected');
|
||||
const failed = allTests.filter(t => t.status === 'unexpected');
|
||||
const flaky = allTests.filter(t => t.status === 'flaky');
|
||||
const skipped = allTests.filter(t => t.status === 'skipped');
|
||||
|
||||
const lines = [commentMarker];
|
||||
|
||||
if (failed.length > 0) {
|
||||
lines.push(`## 🔴 SSO Playwright Results — ${failed.length} failure(s)${flaky.length > 0 ? `, ${flaky.length} flaky` : ''}`);
|
||||
} else if (flaky.length > 0) {
|
||||
lines.push(`## 🟡 SSO Playwright Results — all passed (${flaky.length} flaky)`);
|
||||
} else {
|
||||
lines.push(`## ✅ SSO Playwright Results — all ${passed.length} tests passed`);
|
||||
}
|
||||
lines.push('');
|
||||
lines.push(`✅ ${passed.length} passed · ❌ ${failed.length} failed · 🟡 ${flaky.length} flaky · ⏭️ ${skipped.length} skipped`);
|
||||
lines.push('');
|
||||
|
||||
if (failed.length > 0) {
|
||||
lines.push('### Failures');
|
||||
lines.push('');
|
||||
for (const t of failed.slice(0, 20)) {
|
||||
lines.push(`<details><summary>❌ ${t.project} › ${t.title}</summary>`);
|
||||
lines.push('');
|
||||
lines.push('```');
|
||||
lines.push(t.error.substring(0, 1000));
|
||||
lines.push('```');
|
||||
lines.push('</details>');
|
||||
lines.push('');
|
||||
}
|
||||
}
|
||||
|
||||
if (flaky.length > 0) {
|
||||
lines.push(`<details><summary>🟡 ${flaky.length} flaky test(s)</summary>`);
|
||||
lines.push('');
|
||||
for (const t of flaky.slice(0, 20)) {
|
||||
lines.push(`- ${t.project} › ${t.title} (${t.retries} ${t.retries === 1 ? 'retry' : 'retries'})`);
|
||||
}
|
||||
lines.push('');
|
||||
lines.push('</details>');
|
||||
lines.push('');
|
||||
}
|
||||
|
||||
lines.push(`📦 [View run details](${artifactUrl})`);
|
||||
|
||||
const body = lines.join('\n');
|
||||
|
||||
let existingComment = null;
|
||||
for await (const response of github.paginate.iterator(
|
||||
github.rest.issues.listComments, { ...repo, issue_number: prNumber, per_page: 100 }
|
||||
)) {
|
||||
const found = response.data.find(c =>
|
||||
c.user?.login === 'github-actions[bot]' && c.body?.includes(commentMarker)
|
||||
);
|
||||
if (found) { existingComment = found; break; }
|
||||
}
|
||||
|
||||
if (existingComment) {
|
||||
await github.rest.issues.updateComment({ ...repo, comment_id: existingComment.id, body });
|
||||
} else {
|
||||
await github.rest.issues.createComment({ ...repo, issue_number: prNumber, body });
|
||||
}
|
||||
@@ -0,0 +1,234 @@
|
||||
# Copyright 2021 Collate
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
# This workflow executes end-to-end (e2e) tests using Playwright with PostgreSQL as the database.
|
||||
# For more information see: https://github.com/actions/setup-java/blob/main/docs/advanced-usage.md#apache-maven-with-a-settings-path
|
||||
|
||||
name: Postgresql Playwright E2E Tests
|
||||
on:
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
branch:
|
||||
description: "Branch to run tests on"
|
||||
required: true
|
||||
type: string
|
||||
default: "main"
|
||||
send_slack_notification:
|
||||
description: "Send Slack notification with test results"
|
||||
required: false
|
||||
type: boolean
|
||||
default: false
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
concurrency:
|
||||
group: pw-ci-postgresql-branch-${{ inputs.branch }}
|
||||
cancel-in-progress: true
|
||||
|
||||
jobs:
|
||||
pw-ci-postgresql-branch:
|
||||
runs-on: ubuntu-latest
|
||||
environment: test
|
||||
env:
|
||||
# Playwright logs the admin user in many times (performAdminLogin per test, parallel
|
||||
# workers, retries). The production default of 5 active sessions per user would evict the
|
||||
# long-lived storageState session the page fixtures rely on and 401 every request. Raise
|
||||
# the cap for E2E only; docker compose reads this for ${AUTHENTICATION_MAX_ACTIVE_SESSIONS_PER_USER:-5}.
|
||||
AUTHENTICATION_MAX_ACTIVE_SESSIONS_PER_USER: "10000"
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
shardIndex: [1, 2, 3, 4, 5, 6, 7]
|
||||
shardTotal: [7]
|
||||
steps:
|
||||
- name: Free Disk Space (Ubuntu)
|
||||
uses: jlumbroso/free-disk-space@main
|
||||
with:
|
||||
tool-cache: false
|
||||
android: true
|
||||
dotnet: true
|
||||
haskell: true
|
||||
large-packages: false
|
||||
swap-storage: true
|
||||
docker-images: false
|
||||
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
ref: ${{ inputs.branch }}
|
||||
|
||||
- name: Cache Maven Dependencies
|
||||
id: cache-output
|
||||
uses: actions/cache@v4
|
||||
with:
|
||||
path: ~/.m2
|
||||
key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}
|
||||
restore-keys: |
|
||||
${{ runner.os }}-maven-
|
||||
|
||||
- name: Setup Openmetadata Test Environment
|
||||
uses: ./.github/actions/setup-openmetadata-test-environment
|
||||
with:
|
||||
python-version: "3.10"
|
||||
args: "-d postgresql"
|
||||
ingestion_dependency: "playwright"
|
||||
|
||||
- name: Install dependencies
|
||||
working-directory: openmetadata-ui/src/main/resources/ui/
|
||||
run: yarn --ignore-scripts --frozen-lockfile
|
||||
- name: Install Playwright Browsers
|
||||
run: npx playwright@1.57.0 install chromium --with-deps
|
||||
- name: Run Playwright tests
|
||||
working-directory: openmetadata-ui/src/main/resources/ui/
|
||||
run: |
|
||||
if [ "${{ matrix.shardIndex }}" -eq "1" ]; then
|
||||
echo "🔹 Running DataAssetRules-only tests on shard 1"
|
||||
|
||||
# The testMatch pattern ensures only DataAssetRules*.spec.ts files run
|
||||
npx playwright test \
|
||||
--project=setup \
|
||||
--project=DataAssetRulesEnabled \
|
||||
--project=DataAssetRulesDisabled
|
||||
|
||||
elif [ "${{ matrix.shardIndex }}" -eq "2" ]; then
|
||||
echo "🔹 Running search relevance tests on shard 2"
|
||||
|
||||
npx playwright test \
|
||||
--project=search-nightly \
|
||||
playwright/e2e/Search/SearchRelevance.spec.ts \
|
||||
--workers=1
|
||||
|
||||
else
|
||||
# Shards 3-7 handle chromium tests (5 shards total)
|
||||
CHROMIUM_SHARD=$(( ${{ matrix.shardIndex }} - 2 ))
|
||||
echo "🔹 Running all tests (excluding DataAssetRules) on chromium shard ${CHROMIUM_SHARD}/5"
|
||||
npx playwright test \
|
||||
--project=chromium \
|
||||
--grep-invert @dataAssetRules \
|
||||
--shard=${CHROMIUM_SHARD}/5
|
||||
fi
|
||||
|
||||
env:
|
||||
PLAYWRIGHT_IS_OSS: true
|
||||
PLAYWRIGHT_SNOWFLAKE_USERNAME: ${{ secrets.TEST_SNOWFLAKE_USERNAME }}
|
||||
PLAYWRIGHT_SNOWFLAKE_PASSWORD: ${{ secrets.TEST_SNOWFLAKE_PASSWORD }}
|
||||
PLAYWRIGHT_SNOWFLAKE_ACCOUNT: ${{ secrets.TEST_SNOWFLAKE_ACCOUNT }}
|
||||
PLAYWRIGHT_SNOWFLAKE_DATABASE: ${{ secrets.TEST_SNOWFLAKE_DATABASE }}
|
||||
PLAYWRIGHT_SNOWFLAKE_WAREHOUSE: ${{ secrets.TEST_SNOWFLAKE_WAREHOUSE }}
|
||||
PLAYWRIGHT_PROJECT_ID: ${{ steps.cypress-project-id.outputs.CYPRESS_PROJECT_ID }}
|
||||
PLAYWRIGHT_BQ_PRIVATE_KEY: ${{ secrets.TEST_BQ_PRIVATE_KEY }}
|
||||
PLAYWRIGHT_BQ_PROJECT_ID: ${{ secrets.PLAYWRIGHT_BQ_PROJECT_ID }}
|
||||
PLAYWRIGHT_BQ_PRIVATE_KEY_ID: ${{ secrets.TEST_BQ_PRIVATE_KEY_ID }}
|
||||
PLAYWRIGHT_BQ_PROJECT_ID_TAXONOMY: ${{ secrets.TEST_BQ_PROJECT_ID_TAXONOMY }}
|
||||
PLAYWRIGHT_BQ_CLIENT_EMAIL: ${{ secrets.TEST_BQ_CLIENT_EMAIL }}
|
||||
PLAYWRIGHT_BQ_CLIENT_ID: ${{ secrets.TEST_BQ_CLIENT_ID }}
|
||||
PLAYWRIGHT_REDSHIFT_HOST: ${{ secrets.E2E_REDSHIFT_HOST_PORT }}
|
||||
PLAYWRIGHT_REDSHIFT_USERNAME: ${{ secrets.E2E_REDSHIFT_USERNAME }}
|
||||
PLAYWRIGHT_REDSHIFT_PASSWORD: ${{ secrets.E2E_REDSHIFT_PASSWORD }}
|
||||
PLAYWRIGHT_REDSHIFT_DATABASE: ${{ secrets.TEST_REDSHIFT_DATABASE }}
|
||||
PLAYWRIGHT_METABASE_USERNAME: ${{ secrets.TEST_METABASE_USERNAME }}
|
||||
PLAYWRIGHT_METABASE_PASSWORD: ${{ secrets.TEST_METABASE_PASSWORD }}
|
||||
PLAYWRIGHT_METABASE_DB_SERVICE_NAME: ${{ secrets.TEST_METABASE_DB_SERVICE_NAME }}
|
||||
PLAYWRIGHT_METABASE_HOST_PORT: ${{ secrets.TEST_METABASE_HOST_PORT }}
|
||||
PLAYWRIGHT_SUPERSET_USERNAME: ${{ secrets.TEST_SUPERSET_USERNAME }}
|
||||
PLAYWRIGHT_SUPERSET_PASSWORD: ${{ secrets.TEST_SUPERSET_PASSWORD }}
|
||||
PLAYWRIGHT_SUPERSET_HOST_PORT: ${{ secrets.TEST_SUPERSET_HOST_PORT }}
|
||||
PLAYWRIGHT_KAFKA_BOOTSTRAP_SERVERS: ${{ secrets.TEST_KAFKA_BOOTSTRAP_SERVERS }}
|
||||
PLAYWRIGHT_KAFKA_SCHEMA_REGISTRY_URL: ${{ secrets.TEST_KAFKA_SCHEMA_REGISTRY_URL }}
|
||||
PLAYWRIGHT_GLUE_ACCESS_KEY: ${{ secrets.TEST_GLUE_ACCESS_KEY }}
|
||||
PLAYWRIGHT_GLUE_SECRET_KEY: ${{ secrets.TEST_GLUE_SECRET_KEY }}
|
||||
PLAYWRIGHT_GLUE_AWS_REGION: ${{ secrets.TEST_GLUE_AWS_REGION }}
|
||||
PLAYWRIGHT_GLUE_ENDPOINT: ${{ secrets.TEST_GLUE_ENDPOINT }}
|
||||
PLAYWRIGHT_GLUE_STORAGE_SERVICE: ${{ secrets.TEST_GLUE_STORAGE_SERVICE }}
|
||||
PLAYWRIGHT_MYSQL_USERNAME: ${{ secrets.TEST_MYSQL_USERNAME }}
|
||||
PLAYWRIGHT_MYSQL_PASSWORD: ${{ secrets.TEST_MYSQL_PASSWORD }}
|
||||
PLAYWRIGHT_MYSQL_HOST_PORT: ${{ secrets.TEST_MYSQL_HOST_PORT }}
|
||||
PLAYWRIGHT_MYSQL_DATABASE_SCHEMA: ${{ secrets.TEST_MYSQL_DATABASE_SCHEMA }}
|
||||
PLAYWRIGHT_POSTGRES_USERNAME: ${{ secrets.TEST_POSTGRES_USERNAME }}
|
||||
PLAYWRIGHT_POSTGRES_PASSWORD: ${{ secrets.TEST_POSTGRES_PASSWORD }}
|
||||
PLAYWRIGHT_POSTGRES_HOST_PORT: ${{ secrets.TEST_POSTGRES_HOST_PORT }}
|
||||
PLAYWRIGHT_POSTGRES_DATABASE: ${{ secrets.TEST_POSTGRES_DATABASE }}
|
||||
PLAYWRIGHT_AIRFLOW_HOST_PORT: ${{ secrets.TEST_AIRFLOW_HOST_PORT }}
|
||||
PLAYWRIGHT_ML_MODEL_TRACKING_URI: ${{ secrets.TEST_ML_MODEL_TRACKING_URI }}
|
||||
PLAYWRIGHT_ML_MODEL_REGISTRY_URI: ${{ secrets.TEST_ML_MODEL_REGISTRY_URI }}
|
||||
PLAYWRIGHT_S3_STORAGE_ACCESS_KEY_ID: ${{ secrets.TEST_S3_STORAGE_ACCESS_KEY_ID }}
|
||||
PLAYWRIGHT_S3_STORAGE_SECRET_ACCESS_KEY: ${{ secrets.TEST_S3_STORAGE_SECRET_ACCESS_KEY }}
|
||||
PLAYWRIGHT_S3_STORAGE_END_POINT_URL: ${{ secrets.TEST_S3_STORAGE_END_POINT_URL }}
|
||||
|
||||
# Recommended: pass the GitHub token lets this action correctly
|
||||
# determine the unique run id necessary to re-run the checks
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
- name: Upload blob report to GitHub Actions Artifacts
|
||||
if: ${{ !cancelled() }}
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: blob-report-${{ matrix.shardIndex }}
|
||||
path: openmetadata-ui/src/main/resources/ui/blob-report
|
||||
retention-days: 1
|
||||
|
||||
- name: Upload HTML report to GitHub Actions Artifacts
|
||||
if: ${{ !cancelled() }}
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: playwright-HTML-report-${{ matrix.shardIndex }}
|
||||
path: openmetadata-ui/src/main/resources/ui/playwright/output/playwright-report
|
||||
retention-days: 1
|
||||
compression-level: 9
|
||||
|
||||
- name: Clean Up
|
||||
run: |
|
||||
cd ./docker/development
|
||||
docker compose down --remove-orphans
|
||||
sudo rm -rf ${PWD}/docker-volume
|
||||
|
||||
merge-reports:
|
||||
needs: pw-ci-postgresql-branch
|
||||
runs-on: ubuntu-latest
|
||||
if: ${{ !failure() || !cancelled() }}
|
||||
env:
|
||||
# Recommended: pass the GitHub token lets this action correctly
|
||||
# determine the unique run id necessary to re-run the checks
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
SLACK_BOT_USER_OAUTH_TOKEN: ${{ secrets.E2E_SLACK_BOT_OAUTH_TOKEN }}
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
ref: ${{ inputs.branch }}
|
||||
|
||||
- name: Setup Node.js
|
||||
uses: actions/setup-node@v4
|
||||
with:
|
||||
node-version-file: 'openmetadata-ui/src/main/resources/ui/.nvmrc'
|
||||
|
||||
- name: Download blob reports from GitHub Actions Artifacts
|
||||
uses: actions/download-artifact@v4
|
||||
with:
|
||||
path: openmetadata-ui/src/main/resources/ui/blob-reports
|
||||
pattern: blob-report-*
|
||||
merge-multiple: true
|
||||
|
||||
- name: Merge Reports
|
||||
working-directory: openmetadata-ui/src/main/resources/ui/
|
||||
run: |
|
||||
npx playwright merge-reports --reporter json ./blob-reports > merged_tests_results.json
|
||||
|
||||
- name: Generate Slack Notification
|
||||
if: ${{ inputs.send_slack_notification }}
|
||||
working-directory: openmetadata-ui/src/main/resources/ui/
|
||||
env:
|
||||
RUN_TITLE: "PostgreSQL Test for OSS Release: ${{ inputs.branch }}"
|
||||
RUN_URL: "${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"
|
||||
SLACK_BOT_USER_OAUTH_TOKEN: ${{ secrets.E2E_SLACK_BOT_OAUTH_TOKEN }}
|
||||
run: |
|
||||
npx playwright-slack-report -c playwright/slack-cli.config.json -j merged_tests_results.json > slack_report.json
|
||||
@@ -0,0 +1,424 @@
|
||||
# Copyright 2026 Collate
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
name: PR Metadata Validation
|
||||
|
||||
# Fails the PR check unless it links a GitHub issue that (a) has a real
|
||||
# description and (b) is tracked in the "Shipping" project with the Status,
|
||||
# Source, Priority, Domain and Release fields set.
|
||||
#
|
||||
# Linked issues are resolved from two sources:
|
||||
# * Same-repo: GitHub's native `closingIssuesReferences` (the PR "Development"
|
||||
# section + same-repo closing keywords). This is authoritative and also
|
||||
# catches issues linked via the sidebar without a body keyword.
|
||||
# * Cross-repo (same org only): parsed from the PR body, because GitHub cannot
|
||||
# represent a cross-repo link in `closingIssuesReferences`. This path is
|
||||
# restricted to trusted authors and an explicit repo allowlist so the
|
||||
# privileged PROJECTS_TOKEN is never pointed at attacker-chosen repositories
|
||||
# under pull_request_target, and the public PR comment never echoes private
|
||||
# issue content (only numbers and field names).
|
||||
#
|
||||
# Projects v2 data is only available via GraphQL and the default GITHUB_TOKEN
|
||||
# cannot read org-level projects (nor private same-org repos), so a PROJECTS_TOKEN
|
||||
# secret (a PAT or GitHub App token with `read:project` and read access to the
|
||||
# allowlisted repos) is required. Only PR / issue / project metadata is read
|
||||
# through the API — no untrusted checkout — so pull_request_target is safe here.
|
||||
on:
|
||||
pull_request_target:
|
||||
types: [opened, edited, reopened, synchronize, ready_for_review]
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
pr_number:
|
||||
description: "PR number to validate"
|
||||
required: true
|
||||
type: number
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
issues: read
|
||||
pull-requests: write
|
||||
|
||||
concurrency:
|
||||
# Serialize runs per PR (avoids duplicate sticky comments) but never cancel:
|
||||
# this check gates merges, so every event must reach a definitive conclusion.
|
||||
group: pr-metadata-validation-${{ github.event.pull_request.number || github.event.inputs.pr_number || github.run_id }}
|
||||
cancel-in-progress: false
|
||||
|
||||
jobs:
|
||||
validate-pr-metadata:
|
||||
name: Validate PR Metadata
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Check linked issue and Shipping project fields
|
||||
uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1
|
||||
env:
|
||||
PROJECTS_TOKEN: ${{ secrets.PROJECTS_TOKEN }}
|
||||
with:
|
||||
script: |
|
||||
const CHECK_MARKER = '<!-- pr-metadata-check -->';
|
||||
const BYPASS_LABEL = 'skip-pr-checks';
|
||||
|
||||
const { owner, repo } = context.repo;
|
||||
|
||||
// ---- Policy --------------------------------------------------------
|
||||
const REQUIRE_LINKED_ISSUE = true;
|
||||
const MIN_DESCRIPTION_WORDS = 25;
|
||||
const PROJECT_NAME = 'Shipping';
|
||||
const REQUIRED_PROJECT_FIELDS = ['Status', 'Source', 'Priority', 'Domain', 'Release'];
|
||||
const FAIL_WHEN_PROJECT_UNVERIFIABLE = true;
|
||||
|
||||
// ---- Cross-repo policy (security-sensitive) ------------------------
|
||||
// closingIssuesReferences is same-repo only, so a cross-repo link can
|
||||
// only live in the PR body. Honor it ONLY for trusted authors and ONLY
|
||||
// for these same-org repos, so untrusted fork PRs can never drive the
|
||||
// privileged token at arbitrary repositories or probe private issues.
|
||||
const CLOSING_KEYWORDS = '(?:close[sd]?|fix(?:e[sd])?|resolve[sd]?)';
|
||||
const ALLOWED_CROSS_REPOS = new Set(['openmetadata-collate']);
|
||||
const TRUSTED_ASSOCIATIONS = new Set(['OWNER', 'MEMBER', 'COLLABORATOR']);
|
||||
|
||||
const PR_LINKS_QUERY = `
|
||||
query($owner: String!, $repo: String!, $number: Int!) {
|
||||
repository(owner: $owner, name: $repo) {
|
||||
pullRequest(number: $number) {
|
||||
closingIssuesReferences(first: 25) {
|
||||
nodes { number repository { name owner { login } } }
|
||||
}
|
||||
}
|
||||
}
|
||||
}`;
|
||||
|
||||
const ISSUE_PROJECT_QUERY = `
|
||||
query($owner: String!, $repo: String!, $number: Int!) {
|
||||
repository(owner: $owner, name: $repo) {
|
||||
issue(number: $number) {
|
||||
projectItems(first: 25) {
|
||||
nodes {
|
||||
project { title }
|
||||
fieldValues(first: 50) {
|
||||
nodes {
|
||||
__typename
|
||||
... on ProjectV2ItemFieldSingleSelectValue { name field { ... on ProjectV2FieldCommon { name } } }
|
||||
... on ProjectV2ItemFieldTextValue { text field { ... on ProjectV2FieldCommon { name } } }
|
||||
... on ProjectV2ItemFieldNumberValue { number field { ... on ProjectV2FieldCommon { name } } }
|
||||
... on ProjectV2ItemFieldIterationValue { title field { ... on ProjectV2FieldCommon { name } } }
|
||||
... on ProjectV2ItemFieldDateValue { date field { ... on ProjectV2FieldCommon { name } } }
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}`;
|
||||
|
||||
function elevatedClient() {
|
||||
const token = process.env.PROJECTS_TOKEN;
|
||||
return token ? new github.constructor({ auth: token }) : null;
|
||||
}
|
||||
|
||||
const isSameRepo = (ref) => ref.owner === owner && ref.repo === repo;
|
||||
const refKey = (ref) => `${ref.owner}/${ref.repo}#${ref.number}`;
|
||||
const refLabel = (ref) => (isSameRepo(ref) ? `#${ref.number}` : refKey(ref));
|
||||
const dedupeRefs = (refs) => [...new Map(refs.map((r) => [refKey(r), r])).values()];
|
||||
|
||||
async function loadPullRequest() {
|
||||
if (context.payload.pull_request) {
|
||||
return context.payload.pull_request;
|
||||
}
|
||||
const pull_number = Number(context.payload.inputs.pr_number);
|
||||
const { data } = await github.rest.pulls.get({ owner, repo, pull_number });
|
||||
return data;
|
||||
}
|
||||
|
||||
// Same-repo links: authoritative, includes sidebar-linked issues.
|
||||
async function nativeLinkedRefs(prNumber) {
|
||||
const data = await github.graphql(PR_LINKS_QUERY, { owner, repo, number: prNumber });
|
||||
const nodes =
|
||||
(data.repository &&
|
||||
data.repository.pullRequest &&
|
||||
data.repository.pullRequest.closingIssuesReferences.nodes) ||
|
||||
[];
|
||||
return nodes.map((node) => ({
|
||||
owner: node.repository.owner.login,
|
||||
repo: node.repository.name,
|
||||
number: node.number,
|
||||
}));
|
||||
}
|
||||
|
||||
// Trust the author for cross-repo refs if their PR association is already a trusted
|
||||
// value, or — since the default GITHUB_TOKEN cannot see *private* org membership and
|
||||
// reports such authors as CONTRIBUTOR — by confirming org membership with the elevated
|
||||
// token. Fork PRs from non-members still resolve to untrusted.
|
||||
async function isTrustedAuthor(pr) {
|
||||
let trusted = TRUSTED_ASSOCIATIONS.has(pr.author_association);
|
||||
const client = elevatedClient();
|
||||
if (!trusted && client && pr.user && pr.user.login) {
|
||||
try {
|
||||
await client.rest.orgs.checkMembershipForUser({ org: owner, username: pr.user.login });
|
||||
trusted = true; // 204 => org member (including private membership)
|
||||
} catch (error) {
|
||||
trusted = false; // 404 => not a member
|
||||
}
|
||||
}
|
||||
return trusted;
|
||||
}
|
||||
|
||||
// Cross-repo links: body-parsed, allowlisted repos only (caller gates on author trust).
|
||||
function crossRepoRefsFromBody(text) {
|
||||
const refs = [];
|
||||
const add = (refOwner, refRepo, number) => {
|
||||
if (refOwner === owner && ALLOWED_CROSS_REPOS.has(refRepo)) {
|
||||
refs.push({ owner: refOwner, repo: refRepo, number: Number(number) });
|
||||
}
|
||||
};
|
||||
const shorthand = new RegExp(
|
||||
`\\b${CLOSING_KEYWORDS}\\b\\s*:?\\s*([\\w.-]+)/([\\w.-]+)#(\\d+)`,
|
||||
'gi'
|
||||
);
|
||||
const url = new RegExp(
|
||||
`\\b${CLOSING_KEYWORDS}\\b\\s*:?\\s*https?://github\\.com/([\\w.-]+)/([\\w.-]+)/issues/(\\d+)`,
|
||||
'gi'
|
||||
);
|
||||
let match;
|
||||
while ((match = shorthand.exec(text)) !== null) {
|
||||
add(match[1], match[2], match[3]);
|
||||
}
|
||||
while ((match = url.exec(text)) !== null) {
|
||||
add(match[1], match[2], match[3]);
|
||||
}
|
||||
return refs;
|
||||
}
|
||||
|
||||
function hasProperDescription(body) {
|
||||
if (!body) {
|
||||
return false;
|
||||
}
|
||||
const meaningful = body.replace(/<!--[\s\S]*?-->/g, ' ').replace(/\s+/g, ' ').trim();
|
||||
const wordCount = meaningful ? meaningful.split(' ').filter(Boolean).length : 0;
|
||||
return wordCount >= MIN_DESCRIPTION_WORDS;
|
||||
}
|
||||
|
||||
async function loadIssue(ref) {
|
||||
const client = isSameRepo(ref) ? github : elevatedClient();
|
||||
if (!client) {
|
||||
return { error: 'missing-token' };
|
||||
}
|
||||
try {
|
||||
const { data } = await client.rest.issues.get({
|
||||
owner: ref.owner,
|
||||
repo: ref.repo,
|
||||
issue_number: ref.number,
|
||||
});
|
||||
return { issue: data };
|
||||
} catch (error) {
|
||||
return { error: error.status === 404 ? 'not-found' : error.message };
|
||||
}
|
||||
}
|
||||
|
||||
async function loadIssueProjectItems(ref) {
|
||||
const client = elevatedClient();
|
||||
if (!client) {
|
||||
return { error: 'missing-token' };
|
||||
}
|
||||
try {
|
||||
const data = await client.graphql(ISSUE_PROJECT_QUERY, {
|
||||
owner: ref.owner,
|
||||
repo: ref.repo,
|
||||
number: ref.number,
|
||||
});
|
||||
return { items: data.repository.issue.projectItems.nodes };
|
||||
} catch (error) {
|
||||
return { error: error.message };
|
||||
}
|
||||
}
|
||||
|
||||
function resolveFieldValue(fieldValue) {
|
||||
const candidates = [fieldValue.name, fieldValue.text, fieldValue.title, fieldValue.date];
|
||||
const text = candidates.find((value) => value != null && String(value).trim() !== '');
|
||||
if (text != null) {
|
||||
return text;
|
||||
}
|
||||
return fieldValue.number != null ? String(fieldValue.number) : null;
|
||||
}
|
||||
|
||||
function collectSetFields(item) {
|
||||
const setFields = new Map();
|
||||
for (const fieldValue of item.fieldValues.nodes) {
|
||||
const fieldName = fieldValue.field && fieldValue.field.name;
|
||||
const resolved = resolveFieldValue(fieldValue);
|
||||
if (fieldName && resolved != null) {
|
||||
setFields.set(fieldName, resolved);
|
||||
}
|
||||
}
|
||||
return setFields;
|
||||
}
|
||||
|
||||
async function collectProjectProblems(ref) {
|
||||
const result = await loadIssueProjectItems(ref);
|
||||
if (result.error) {
|
||||
if (!FAIL_WHEN_PROJECT_UNVERIFIABLE) {
|
||||
core.warning(`Skipping project validation for ${refLabel(ref)}: ${result.error}`);
|
||||
return [];
|
||||
}
|
||||
core.warning(`Project verification failed for ${refLabel(ref)}: ${result.error}`);
|
||||
const detail = result.error === 'missing-token'
|
||||
? 'the `PROJECTS_TOKEN` secret (a token with `read:project` and access to the linked repo) is not configured'
|
||||
: 'the project query could not be completed (see the workflow run logs)';
|
||||
return [`Could not verify the **${PROJECT_NAME}** project on issue ${refLabel(ref)}: ${detail}.`];
|
||||
}
|
||||
const projectItem = (result.items || []).find(
|
||||
(item) => item.project && item.project.title === PROJECT_NAME
|
||||
);
|
||||
if (!projectItem) {
|
||||
return [`Linked issue ${refLabel(ref)} is not added to the **${PROJECT_NAME}** project.`];
|
||||
}
|
||||
const setFields = collectSetFields(projectItem);
|
||||
const problems = [];
|
||||
for (const field of REQUIRED_PROJECT_FIELDS) {
|
||||
if (!setFields.has(field)) {
|
||||
problems.push(`Issue ${refLabel(ref)} is missing **${PROJECT_NAME} → ${field}**.`);
|
||||
}
|
||||
}
|
||||
return problems;
|
||||
}
|
||||
|
||||
async function validateIssue(ref) {
|
||||
const loaded = await loadIssue(ref);
|
||||
if (loaded.error === 'not-found') {
|
||||
return { ref, problems: [`Linked issue ${refLabel(ref)} does not exist or is not accessible.`] };
|
||||
}
|
||||
if (loaded.error) {
|
||||
const detail = loaded.error === 'missing-token'
|
||||
? 'the `PROJECTS_TOKEN` secret is not configured'
|
||||
: 'the issue lookup could not be completed (see the workflow run logs)';
|
||||
core.warning(`Issue lookup failed for ${refLabel(ref)}: ${loaded.error}`);
|
||||
return { ref, problems: [`Could not read linked issue ${refLabel(ref)}: ${detail}.`] };
|
||||
}
|
||||
const issue = loaded.issue;
|
||||
if (issue.pull_request) {
|
||||
return { ref, problems: [`${refLabel(ref)} is a pull request, not an issue.`] };
|
||||
}
|
||||
const problems = [];
|
||||
if (!hasProperDescription(issue.body)) {
|
||||
problems.push(
|
||||
`Linked issue ${refLabel(ref)} needs a real description ` +
|
||||
`(at least ${MIN_DESCRIPTION_WORDS} words covering the problem, repro, and expected behavior).`
|
||||
);
|
||||
}
|
||||
problems.push(...(await collectProjectProblems(ref)));
|
||||
return { ref, problems };
|
||||
}
|
||||
|
||||
async function collectIssueProblems(pr) {
|
||||
const native = await nativeLinkedRefs(pr.number);
|
||||
const cross = (await isTrustedAuthor(pr))
|
||||
? crossRepoRefsFromBody(`${pr.title || ''}\n${pr.body || ''}`)
|
||||
: [];
|
||||
const refs = dedupeRefs([...native, ...cross]);
|
||||
if (refs.length === 0) {
|
||||
return [
|
||||
'No GitHub issue is linked. Link an issue in the **Development** section of the PR ' +
|
||||
'(or add `Fixes #12345` to the description). For a same-org cross-repo issue, add ' +
|
||||
'`Fixes open-metadata/<repo>#123` to the description.',
|
||||
];
|
||||
}
|
||||
const validations = await Promise.all(refs.map(validateIssue));
|
||||
const validIssues = validations.filter((validation) => validation.problems.length === 0);
|
||||
const problems = [];
|
||||
if (validIssues.length === 0) {
|
||||
for (const validation of validations) {
|
||||
problems.push(...validation.problems);
|
||||
}
|
||||
}
|
||||
return problems;
|
||||
}
|
||||
|
||||
function buildFailureComment(problems) {
|
||||
const lines = [
|
||||
CHECK_MARKER,
|
||||
'### ❌ PR checklist incomplete',
|
||||
'',
|
||||
'This PR cannot be merged until the following are addressed on its linked issue:',
|
||||
'',
|
||||
];
|
||||
for (const problem of problems) {
|
||||
lines.push(`- ${problem}`);
|
||||
}
|
||||
lines.push('');
|
||||
lines.push(
|
||||
`The fields live on the **linked issue** in the **${PROJECT_NAME}** project ` +
|
||||
'(open the issue → right sidebar → **Projects**). After you set them, ' +
|
||||
'**re-run this check** (or push a commit) — issue/project changes do not re-trigger it automatically.'
|
||||
);
|
||||
lines.push('');
|
||||
lines.push(`_Maintainers can bypass this check by adding the \`${BYPASS_LABEL}\` label._`);
|
||||
return lines.join('\n');
|
||||
}
|
||||
|
||||
function buildSuccessComment() {
|
||||
return [
|
||||
CHECK_MARKER,
|
||||
'### ✅ PR checks passed',
|
||||
'',
|
||||
`The linked issue has a description and all required **${PROJECT_NAME}** project fields set. Thanks!`,
|
||||
].join('\n');
|
||||
}
|
||||
|
||||
async function syncStickyComment(prNumber, body, hadFailure) {
|
||||
const comments = await github.paginate(github.rest.issues.listComments, {
|
||||
owner,
|
||||
repo,
|
||||
issue_number: prNumber,
|
||||
per_page: 100,
|
||||
});
|
||||
const existing = comments.find((comment) => comment.body && comment.body.includes(CHECK_MARKER));
|
||||
if (hadFailure) {
|
||||
if (existing) {
|
||||
await github.rest.issues.updateComment({ owner, repo, comment_id: existing.id, body });
|
||||
} else {
|
||||
await github.rest.issues.createComment({ owner, repo, issue_number: prNumber, body });
|
||||
}
|
||||
} else if (existing) {
|
||||
await github.rest.issues.updateComment({ owner, repo, comment_id: existing.id, body });
|
||||
}
|
||||
}
|
||||
|
||||
const pr = await loadPullRequest();
|
||||
|
||||
if (pr.draft) {
|
||||
core.info('Draft PR — skipping metadata validation.');
|
||||
return;
|
||||
}
|
||||
if (pr.user && pr.user.type === 'Bot') {
|
||||
core.info(`PR opened by bot ${pr.user.login} — skipping metadata validation.`);
|
||||
return;
|
||||
}
|
||||
const labels = (pr.labels || []).map((label) => label.name);
|
||||
if (labels.includes(BYPASS_LABEL)) {
|
||||
core.info(`'${BYPASS_LABEL}' label present — skipping metadata validation.`);
|
||||
return;
|
||||
}
|
||||
|
||||
const problems = [];
|
||||
if (REQUIRE_LINKED_ISSUE) {
|
||||
problems.push(...(await collectIssueProblems(pr)));
|
||||
}
|
||||
|
||||
const hadFailure = problems.length > 0;
|
||||
const body = hadFailure ? buildFailureComment(problems) : buildSuccessComment();
|
||||
await syncStickyComment(pr.number, body, hadFailure);
|
||||
|
||||
if (hadFailure) {
|
||||
core.setFailed(
|
||||
`PR metadata checklist incomplete: ${problems.length} item(s) need attention. See the PR comment.`
|
||||
);
|
||||
} else {
|
||||
core.info('All PR metadata checks passed.');
|
||||
}
|
||||
@@ -0,0 +1,84 @@
|
||||
# Copyright 2021 Collate
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
name: Publish Package to Maven Central Repository
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
|
||||
push:
|
||||
branches:
|
||||
- main
|
||||
paths:
|
||||
- "openmetadata-service/**"
|
||||
- "openmetadata-spec/src/main/resources/json/schema/**"
|
||||
- "openmetadata-clients/**"
|
||||
- "pom.xml"
|
||||
- ".github/workflows/publish-maven-package.yml"
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
publish-maven-packages:
|
||||
runs-on: ubuntu-latest
|
||||
environment: publish
|
||||
steps:
|
||||
- name: Free Disk Space (Ubuntu)
|
||||
uses: jlumbroso/free-disk-space@main
|
||||
with:
|
||||
tool-cache: false
|
||||
android: true
|
||||
dotnet: true
|
||||
haskell: true
|
||||
large-packages: false
|
||||
swap-storage: true
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- name: Set up JDK 21
|
||||
uses: actions/setup-java@v4
|
||||
with:
|
||||
java-version: 21
|
||||
distribution: "temurin"
|
||||
server-id: central
|
||||
server-username: MAVEN_USERNAME
|
||||
server-password: MAVEN_PASSWORD
|
||||
gpg-private-key: ${{ secrets.OSSRH_GPG_SECRET_KEY }}
|
||||
gpg-passphrase: MAVEN_GPG_PASSPHRASE
|
||||
|
||||
- name: Install Ubuntu dependencies
|
||||
run: |
|
||||
sudo apt-get update
|
||||
sudo apt-get install -y unixodbc-dev python3-venv librdkafka-dev gcc libsasl2-dev build-essential libssl-dev libffi-dev \
|
||||
librdkafka-dev unixodbc-dev libevent-dev
|
||||
- name: Setup Test Containers Properties
|
||||
run: |
|
||||
sudo make install_antlr_cli
|
||||
echo 'testcontainers.reuse.enable=true' >> $HOME/.testcontainers.properties
|
||||
- name: Setup settings-security.xml
|
||||
run: |
|
||||
rm -rf ~/.m2/settings-security.xml
|
||||
echo "<settingsSecurity><master>${{ secrets.MAVEN_MASTER_PASSWORD }}</master></settingsSecurity>" > ~/.m2/settings-security.xml
|
||||
- name: Install gpg secret key
|
||||
run: |
|
||||
cat <(echo -e "${{ secrets.OSSRH_GPG_SECRET_KEY }}") | gpg --batch --import
|
||||
gpg --list-secret-keys --keyid-format LONG
|
||||
- name: Publish to Central Repository
|
||||
env:
|
||||
MAVEN_USERNAME: ${{ secrets.OSSRH_USERNAME }}
|
||||
MAVEN_PASSWORD: ${{ secrets.OSSRH_TOKEN }}
|
||||
MAVEN_GPG_PASSPHRASE: ${{ secrets.OSSRH_GPG_SECRET_KEY_PASSWORD }}
|
||||
run: |
|
||||
mvn \
|
||||
--no-transfer-progress \
|
||||
--batch-mode \
|
||||
-Dgpg.passphrase=${{ secrets.OSSRH_GPG_SECRET_KEY_PASSWORD }} \
|
||||
-DskipTests -Prelease clean deploy
|
||||
@@ -0,0 +1,103 @@
|
||||
# Copyright 2021 Collate
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
name: Python Checkstyle
|
||||
|
||||
# read-write repo token
|
||||
# access to secrets
|
||||
on:
|
||||
merge_group:
|
||||
pull_request_target:
|
||||
types: [labeled, opened, synchronize, reopened, ready_for_review]
|
||||
paths-ignore:
|
||||
- "openmetadata-ui/src/main/resources/ui/playwright/doc-generator/**"
|
||||
- "openmetadata-ui/src/main/resources/ui/playwright/docs/**"
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
concurrency:
|
||||
group: py-checkstyle-${{ github.event.pull_request.number || github.run_id }}
|
||||
cancel-in-progress: ${{ github.event_name != 'pull_request_target' || github.event.action != 'labeled' || github.event.label.name == 'safe to test' }}
|
||||
|
||||
jobs:
|
||||
py-checkstyle:
|
||||
runs-on: ubuntu-latest
|
||||
if: ${{ !github.event.pull_request.draft && (github.event_name != 'pull_request_target' || github.event.action != 'labeled' || github.event.label.name == 'safe to test') }}
|
||||
permissions:
|
||||
pull-requests: write
|
||||
|
||||
steps:
|
||||
- name: Wait for the labeler
|
||||
uses: lewagon/wait-on-check-action@v1.7.0
|
||||
if: ${{ github.event_name == 'pull_request_target' }}
|
||||
with:
|
||||
ref: ${{ github.event.pull_request.head.sha }}
|
||||
check-name: Team Label
|
||||
repo-token: ${{ secrets.GITHUB_TOKEN }}
|
||||
wait-interval: 90
|
||||
|
||||
- name: Verify PR labels
|
||||
uses: jesusvasquez333/verify-pr-label-action@v1.4.0
|
||||
if: ${{ github.event_name == 'pull_request_target' }}
|
||||
with:
|
||||
github-token: '${{ secrets.GITHUB_TOKEN }}'
|
||||
valid-labels: 'safe to test'
|
||||
pull-request-number: '${{ github.event.pull_request.number }}'
|
||||
disable-reviews: true # To not auto approve changes
|
||||
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
ref: ${{ github.event_name == 'merge_group' && github.sha || github.event.pull_request.head.sha }}
|
||||
|
||||
- name: Set up Python 3.10
|
||||
uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: '3.10'
|
||||
|
||||
- name: Install Ubuntu related dependencies
|
||||
run: |
|
||||
sudo apt-get update && sudo apt-get install -y libsasl2-dev unixodbc-dev python3-venv
|
||||
|
||||
- name: Install Python & Openmetadata related dependencies
|
||||
run: |
|
||||
python3 -m venv env
|
||||
source env/bin/activate
|
||||
sudo make install_antlr_cli
|
||||
make install install_test install_dev
|
||||
|
||||
# Add back linting once we have 10/10 on main
|
||||
- name: Code style check
|
||||
id: style
|
||||
continue-on-error: true
|
||||
run: |
|
||||
source env/bin/activate
|
||||
make generate
|
||||
make py_format_check
|
||||
|
||||
- name: Create a comment in the PR with the instructions
|
||||
if: ${{ steps.style.outcome != 'success' && github.event_name == 'pull_request_target' }}
|
||||
uses: peter-evans/create-or-update-comment@v1
|
||||
with:
|
||||
issue-number: ${{ github.event.pull_request.number }}
|
||||
body: |
|
||||
**The Python checkstyle failed.**
|
||||
|
||||
Please run `make py_format` and `py_format_check` in the root of your repository and commit the changes to this PR.
|
||||
You can also use [pre-commit](https://pre-commit.com/) to automate the Python code formatting.
|
||||
|
||||
You can install the pre-commit hooks with `make install_test precommit_install`.
|
||||
|
||||
- name: Python checkstyle failed, check the comment in the PR
|
||||
if: steps.style.outcome != 'success'
|
||||
run: |
|
||||
exit 1
|
||||
@@ -0,0 +1,306 @@
|
||||
# Copyright 2021 Collate
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
name: py-cli-e2e-tests
|
||||
on:
|
||||
schedule:
|
||||
- cron: '0 0 * * *'
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
e2e-tests:
|
||||
description: "E2E Tests to run"
|
||||
required: True
|
||||
default: '["bigquery", "dbt_redshift", "metabase", "mssql", "mysql", "redash", "snowflake", "tableau", "python-unittests", "python-integration", "redshift", "quicksight", "datalake_s3", "postgres", "oracle", "athena", "bigquery_multiple_project", "exasol"]'
|
||||
debug:
|
||||
description: "If Debugging the Pipeline, Slack and Sonar events won't be triggered [default, true or false]. Default will trigger only on main branch."
|
||||
required: False
|
||||
default: "default"
|
||||
|
||||
env:
|
||||
DEBUG: "${{ (inputs.debug == 'default' && github.ref == 'refs/heads/main' && 'false') || (inputs.debug == 'default' && github.ref != 'refs/heads/main' && 'true') || inputs.debug || 'false' }}"
|
||||
|
||||
permissions:
|
||||
id-token: write
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
# Needed since env is not available on the job context: https://github.com/actions/runner/issues/2372
|
||||
check-debug:
|
||||
runs-on: ubuntu-latest
|
||||
outputs:
|
||||
DEBUG: ${{ env.DEBUG }}
|
||||
steps:
|
||||
- run: echo "INPUTS_DEBUG=${{ inputs.debug }}, GITHUB_REF=${{ github.ref }}, DEBUG=$DEBUG"
|
||||
|
||||
py-cli-e2e-tests:
|
||||
runs-on: ubuntu-latest
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
e2e-test: ${{ fromJSON(inputs.e2e-tests || '["bigquery", "dbt_redshift", "metabase", "mssql", "mysql", "redash", "snowflake", "tableau", "python-unittests", "python-integration", "redshift", "quicksight", "datalake_s3", "postgres", "oracle", "athena", "bigquery_multiple_project", "exasol"]') }}
|
||||
environment: test
|
||||
|
||||
steps:
|
||||
- name: Free Disk Space (Ubuntu)
|
||||
if: matrix.e2e-test != 'python-unittests'
|
||||
uses: jlumbroso/free-disk-space@main
|
||||
with:
|
||||
tool-cache: false
|
||||
android: true
|
||||
dotnet: true
|
||||
haskell: true
|
||||
large-packages: false
|
||||
swap-storage: true
|
||||
docker-images: false
|
||||
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: configure aws credentials
|
||||
if: contains('quicksight', matrix.e2e-test) || contains('datalake_s3', matrix.e2e-test) || contains('athena', matrix.e2e-test)
|
||||
uses: aws-actions/configure-aws-credentials@v4
|
||||
with:
|
||||
role-to-assume: ${{ secrets.E2E_AWS_IAM_ROLE_ARN }}
|
||||
role-session-name: github-ci-aws-e2e-tests
|
||||
aws-region: ${{ secrets.E2E_AWS_REGION }}
|
||||
|
||||
- name: Setup Openmetadata Test Environment (without server)
|
||||
if: matrix.e2e-test == 'python-unittests'
|
||||
uses: ./.github/actions/setup-openmetadata-test-environment
|
||||
with:
|
||||
python-version: '3.10'
|
||||
install-server: 'false'
|
||||
|
||||
- name: Setup Openmetadata Test Environment
|
||||
if: matrix.e2e-test != 'python-unittests'
|
||||
uses: ./.github/actions/setup-openmetadata-test-environment
|
||||
with:
|
||||
python-version: '3.10'
|
||||
|
||||
- name: Run Python Unit Tests
|
||||
if: matrix.e2e-test == 'python-unittests'
|
||||
id: python-unittest
|
||||
continue-on-error: true
|
||||
run: |
|
||||
source env/bin/activate
|
||||
cd ingestion
|
||||
nox --no-venv -s unit-tests
|
||||
shell: bash
|
||||
|
||||
- name: Rename coverage file for Python unit tests
|
||||
if: matrix.e2e-test == 'python-unittests' && steps.python-unittest.outcome == 'success' && env.DEBUG == 'false'
|
||||
run: mv ingestion/.coverage .coverage.python-unittests
|
||||
|
||||
- name: Run Python Integration Tests
|
||||
if: matrix.e2e-test == 'python-integration'
|
||||
id: python-integration-test
|
||||
continue-on-error: true
|
||||
run: |
|
||||
source env/bin/activate
|
||||
cd ingestion
|
||||
nox --no-venv -s integration-tests -- --standalone --durations=5
|
||||
env:
|
||||
TESTCONTAINERS_RYUK_DISABLED: true
|
||||
shell: bash
|
||||
|
||||
- name: Run CLI E2E Python Tests & record coverage
|
||||
if: matrix.e2e-test != 'python-unittests' && matrix.e2e-test != 'python-integration'
|
||||
id: e2e-test
|
||||
continue-on-error: true
|
||||
env:
|
||||
E2E_TEST: ${{ matrix.e2e-test }}
|
||||
E2E_BQ_PROJECT_ID_TAXONOMY: ${{ secrets.TEST_BQ_PROJECT_ID_TAXONOMY }}
|
||||
E2E_BQ_PRIVATE_KEY: ${{ secrets.TEST_BQ_PRIVATE_KEY_E2E }}
|
||||
E2E_BQ_PROJECT_ID: ${{ secrets.TEST_BQ_PROJECT_ID }}
|
||||
E2E_BQ_PROJECT_ID2: ${{ secrets.TEST_BQ_PROJECT_ID2 }}
|
||||
E2E_BQ_PRIVATE_KEY_ID: ${{ secrets.TEST_BQ_PRIVATE_KEY_ID }}
|
||||
E2E_BQ_CLIENT_EMAIL: ${{ secrets.TEST_BQ_CLIENT_EMAIL }}
|
||||
E2E_BQ_CLIENT_ID: ${{ secrets.TEST_BQ_CLIENT_ID }}
|
||||
E2E_SNOWFLAKE_PASSWORD: ${{ secrets.TEST_SNOWFLAKE_PASSWORD_YAML }}
|
||||
E2E_SNOWFLAKE_PASSPHRASE: ${{ secrets.TEST_SNOWFLAKE_PASSPHRASE }}
|
||||
E2E_SNOWFLAKE_USERNAME: ${{ secrets.TEST_SNOWFLAKE_USERNAME }}
|
||||
E2E_SNOWFLAKE_ACCOUNT: ${{ secrets.TEST_SNOWFLAKE_ACCOUNT }}
|
||||
E2E_SNOWFLAKE_DATABASE: ${{ secrets.TEST_SNOWFLAKE_DATABASE_E2E }}
|
||||
E2E_SNOWFLAKE_WAREHOUSE: ${{ secrets.TEST_SNOWFLAKE_WAREHOUSE }}
|
||||
E2E_REDSHIFT_DBT_CATALOG_HTTP_FILE_PATH: ${{ secrets.E2E_REDSHIFT_DBT_CATALOG_HTTP_FILE_PATH }}
|
||||
E2E_REDSHIFT_DBT_MANIFEST_HTTP_FILE_PATH: ${{ secrets.E2E_REDSHIFT_DBT_MANIFEST_HTTP_FILE_PATH }}
|
||||
E2E_REDSHIFT_DBT_RUN_RESULTS_HTTP_FILE_PATH: ${{ secrets.E2E_REDSHIFT_DBT_RUN_RESULTS_HTTP_FILE_PATH }}
|
||||
E2E_REDSHIFT_HOST_PORT: ${{ secrets.E2E_REDSHIFT_HOST_PORT }}
|
||||
E2E_REDSHIFT_USERNAME: ${{ secrets.E2E_REDSHIFT_USERNAME }}
|
||||
E2E_REDSHIFT_PASSWORD: ${{ secrets.E2E_REDSHIFT_PASSWORD }}
|
||||
E2E_REDSHIFT_DATABASE: ${{ secrets.E2E_REDSHIFT_DATABASE }}
|
||||
E2E_REDSHIFT_DB: ${{ secrets.E2E_REDSHIFT_DB }}
|
||||
E2E_MSSQL_USERNAME: ${{ secrets.E2E_MSSQL_USERNAME }}
|
||||
E2E_MSSQL_PASSWORD: ${{ secrets.E2E_MSSQL_PASSWORD }}
|
||||
E2E_MSSQL_HOST: ${{ secrets.E2E_MSSQL_HOST }}
|
||||
E2E_MSSQL_DATABASE: ${{ secrets.E2E_MSSQL_DATABASE }}
|
||||
E2E_VERTICA_USERNAME: ${{ secrets.E2E_VERTICA_USERNAME }}
|
||||
E2E_VERTICA_PASSWORD: ${{ secrets.E2E_VERTICA_PASSWORD }}
|
||||
E2E_VERTICA_HOST_PORT: ${{ secrets.E2E_VERTICA_HOST_PORT }}
|
||||
E2E_TABLEAU_PAT_NAME: ${{ secrets.E2E_TABLEAU_PAT_NAME }}
|
||||
E2E_TABLEAU_PAT_SECRET: ${{ secrets.E2E_TABLEAU_PAT_SECRET }}
|
||||
E2E_TABLEAU_HOST_PORT: ${{ secrets.E2E_TABLEAU_HOST_PORT }}
|
||||
E2E_TABLEAU_SITE: ${{ secrets.E2E_TABLEAU_SITE }}
|
||||
E2E_REDASH_HOST_PORT: ${{ secrets.E2E_REDASH_HOST_PORT }}
|
||||
E2E_REDASH_USERNAME: ${{ secrets.E2E_REDASH_USERNAME }}
|
||||
E2E_REDASH_API_KEY: ${{ secrets.E2E_REDASH_API_KEY }}
|
||||
E2E_POWERBI_CLIENT_SECRET: ${{ secrets.E2E_POWERBI_CLIENT_SECRET }}
|
||||
E2E_POWERBI_CLIENT_ID: ${{ secrets.E2E_POWERBI_CLIENT_ID }}
|
||||
E2E_POWERBI_TENANT_ID: ${{ secrets.E2E_POWERBI_TENANT_ID }}
|
||||
E2E_METABASE_HOST_PORT: ${{ secrets.TEST_METABASE_HOST_PORT }}
|
||||
E2E_METABASE_PASSWORD: ${{ secrets.TEST_METABASE_PASSWORD }}
|
||||
E2E_METABASE_USERNAME: ${{ secrets.TEST_METABASE_USERNAME }}
|
||||
E2E_HIVE_HOST_PORT: ${{ secrets.E2E_HIVE_HOST_PORT }}
|
||||
E2E_HIVE_AUTH: ${{ secrets.E2E_HIVE_AUTH }}
|
||||
E2E_QUICKSIGHT_AWS_ACCOUNTID: ${{ secrets.E2E_QUICKSIGHT_AWS_ACCOUNTID }}
|
||||
E2E_QUICKSIGHT_REGION: ${{ secrets.E2E_QUICKSIGHT_REGION }}
|
||||
E2E_DATALAKE_S3_BUCKET_NAME: ${{ secrets.E2E_DATALAKE_S3_BUCKET_NAME }}
|
||||
E2E_DATALAKE_S3_PREFIX: ${{ secrets.E2E_DATALAKE_S3_PREFIX }}
|
||||
E2E_DATALAKE_S3_REGION: ${{ secrets.E2E_AWS_REGION }}
|
||||
E2E_POSTGRES_USERNAME: ${{ secrets.E2E_POSTGRES_USERNAME }}
|
||||
E2E_POSTGRES_PASSWORD: ${{ secrets.E2E_POSTGRES_PASSWORD }}
|
||||
E2E_POSTGRES_HOSTPORT: ${{ secrets.TEST_POSTGRES_HOST_PORT }}
|
||||
E2E_POSTGRES_DATABASE: ${{ secrets.E2E_POSTGRES_DATABASE }}
|
||||
E2E_ORACLE_HOST_PORT: ${{ secrets.E2E_ORACLE_HOST_PORT }}
|
||||
E2E_ORACLE_USERNAME: ${{ secrets.E2E_ORACLE_USERNAME }}
|
||||
E2E_ORACLE_PASSWORD: ${{ secrets.E2E_ORACLE_PASSWORD }}
|
||||
E2E_ORACLE_SERVICE_NAME: ${{ secrets.E2E_ORACLE_SERVICE_NAME }}
|
||||
E2E_ATHENA_REGION: ${{ secrets.E2E_AWS_REGION }}
|
||||
E2E_ATHENA_S3STAGINGDIR: ${{ secrets.E2E_ATHENA_S3STAGINGDIR }}
|
||||
E2E_ATHENA_WORKGROUP: ${{ secrets.E2E_ATHENA_WORKGROUP }}
|
||||
run: |
|
||||
source env/bin/activate
|
||||
export SITE_CUSTOMIZE_PATH=$(python -c "import site; import os; from pathlib import Path; print(os.path.relpath(site.getsitepackages()[0], str(Path.cwd())))")/sitecustomize.py
|
||||
echo "import os" >> $SITE_CUSTOMIZE_PATH
|
||||
echo "try:" >> $SITE_CUSTOMIZE_PATH
|
||||
echo " import coverage" >> $SITE_CUSTOMIZE_PATH
|
||||
echo " os.environ['COVERAGE_PROCESS_START'] = os.path.join(os.environ.get('GITHUB_WORKSPACE', os.getcwd()), 'ingestion', 'pyproject.toml')" >> $SITE_CUSTOMIZE_PATH
|
||||
echo " coverage.process_startup()" >> $SITE_CUSTOMIZE_PATH
|
||||
echo "except ImportError:" >> $SITE_CUSTOMIZE_PATH
|
||||
echo " pass" >> $SITE_CUSTOMIZE_PATH
|
||||
coverage run --rcfile ingestion/pyproject.toml --branch -m pytest -c ingestion/pyproject.toml --junitxml=ingestion/junit/test-results-$E2E_TEST.xml --ignore=ingestion/tests/unit/source ingestion/tests/cli_e2e/test_cli_$E2E_TEST.py
|
||||
coverage combine --data-file=.coverage.$E2E_TEST --rcfile=ingestion/pyproject.toml --keep .coverage*
|
||||
coverage report --rcfile ingestion/pyproject.toml --data-file .coverage.$E2E_TEST || true
|
||||
|
||||
- name: Upload coverage artifact for Python unit tests
|
||||
if: matrix.e2e-test == 'python-unittests' && steps.python-unittest.outcome == 'success' && env.DEBUG == 'false'
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: coverage-${{ matrix.e2e-test }}
|
||||
path: .coverage.python-unittests
|
||||
include-hidden-files: true
|
||||
|
||||
- name: Rename coverage file for Python integration tests
|
||||
if: matrix.e2e-test == 'python-integration' && steps.python-integration-test.outcome == 'success' && env.DEBUG == 'false'
|
||||
run: mv ingestion/.coverage .coverage.python-integration
|
||||
|
||||
- name: Upload coverage artifact for Python integration tests
|
||||
if: matrix.e2e-test == 'python-integration' && steps.python-integration-test.outcome == 'success' && env.DEBUG == 'false'
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: coverage-${{ matrix.e2e-test }}
|
||||
path: .coverage.python-integration
|
||||
include-hidden-files: true
|
||||
|
||||
- name: Upload coverage artifact for CLI E2E tests
|
||||
if: matrix.e2e-test != 'python-unittests' && matrix.e2e-test != 'python-integration' && steps.e2e-test.outcome == 'success' && env.DEBUG == 'false'
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: coverage-${{ matrix.e2e-test }}
|
||||
path: .coverage.${{ matrix.e2e-test }}
|
||||
include-hidden-files: true
|
||||
|
||||
- name: Upload tests artifact
|
||||
if: (steps.e2e-test.outcome == 'success' || steps.python-unittest.outcome == 'success' || steps.python-integration-test.outcome == 'success') && env.DEBUG == 'false'
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: tests-${{ matrix.e2e-test }}
|
||||
path: ingestion/junit/test-results-*.xml
|
||||
|
||||
- name: Clean Up
|
||||
if: matrix.e2e-test != 'python-unittests'
|
||||
run: |
|
||||
cd ./docker/development
|
||||
docker compose down --remove-orphans
|
||||
sudo rm -rf ${PWD}/docker-volume
|
||||
|
||||
- name: Slack on Failure
|
||||
if: (steps.e2e-test.outcome == 'failure' || steps.python-unittest.outcome == 'failure' || steps.python-integration-test.outcome == 'failure') && env.DEBUG == 'false'
|
||||
uses: slackapi/slack-github-action@v1.23.0
|
||||
with:
|
||||
payload: |
|
||||
{
|
||||
"text": "🔥 Failed E2E Test for: ${{ matrix.e2e-test }} 🔥\nLogs: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"
|
||||
}
|
||||
env:
|
||||
SLACK_WEBHOOK_URL: ${{ secrets.E2E_SLACK_WEBHOOK }}
|
||||
SLACK_WEBHOOK_TYPE: INCOMING_WEBHOOK
|
||||
|
||||
- name: Force failure
|
||||
if: steps.e2e-test.outcome == 'failure' || steps.python-unittest.outcome == 'failure' || steps.python-integration-test.outcome == 'failure'
|
||||
run: |
|
||||
exit 1
|
||||
|
||||
sonar-cloud-coverage-upload:
|
||||
runs-on: ubuntu-latest
|
||||
needs:
|
||||
- py-cli-e2e-tests
|
||||
- check-debug
|
||||
if: needs.check-debug.outputs.DEBUG == 'false'
|
||||
steps:
|
||||
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Set up Python 3.10
|
||||
uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: '3.10'
|
||||
|
||||
- name: Install Ubuntu dependencies
|
||||
run: |
|
||||
sudo apt-get update && sudo apt-get install -y unixodbc-dev python3-venv librdkafka-dev gcc libsasl2-dev build-essential libssl-dev libffi-dev \
|
||||
unixodbc-dev libevent-dev python3-dev libkrb5-dev
|
||||
|
||||
- name: Install coverage dependencies
|
||||
run: |
|
||||
python3 -m venv env
|
||||
source env/bin/activate
|
||||
make install_all install_test
|
||||
|
||||
- name: Download all artifacts
|
||||
uses: actions/download-artifact@v4
|
||||
with:
|
||||
path: artifacts
|
||||
|
||||
- name: Generate report
|
||||
run: |
|
||||
# Copy coverage artifacts into ingestion/ so paths resolve relative to it
|
||||
for folder in artifacts/coverage-*; do
|
||||
cp -rT $folder/ ingestion/ ;
|
||||
done
|
||||
mkdir -p ingestion/junit
|
||||
for folder in artifacts/tests-*; do
|
||||
cp -rT $folder/ ingestion/junit ;
|
||||
done
|
||||
source env/bin/activate
|
||||
cd ingestion
|
||||
coverage combine --rcfile=pyproject.toml --keep .coverage*
|
||||
coverage xml --rcfile=pyproject.toml --data-file=.coverage
|
||||
shell: bash
|
||||
|
||||
- name: Push Results to Sonar
|
||||
uses: sonarsource/sonarcloud-github-action@master
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
SONAR_TOKEN: ${{ secrets.INGESTION_SONAR_SECRET }}
|
||||
with:
|
||||
projectBaseDir: ingestion/
|
||||
@@ -0,0 +1,91 @@
|
||||
# Copyright 2021 Collate
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
name: py-operator-build-test
|
||||
on:
|
||||
workflow_dispatch:
|
||||
pull_request_target:
|
||||
types: [labeled, opened, synchronize, reopened, ready_for_review]
|
||||
paths:
|
||||
- "ingestion/**"
|
||||
- "openmetadata-service/**"
|
||||
- "openmetadata-spec/src/main/resources/json/schema/**"
|
||||
- "pom.xml"
|
||||
- "Makefile"
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
|
||||
cancel-in-progress: ${{ github.event_name != 'pull_request_target' || github.event.action != 'labeled' || github.event.label.name == 'safe to test' }}
|
||||
|
||||
jobs:
|
||||
py-run-build-tests:
|
||||
runs-on: ubuntu-latest
|
||||
if: ${{ !github.event.pull_request.draft && (github.event_name != 'pull_request_target' || github.event.action != 'labeled' || github.event.label.name == 'safe to test') }}
|
||||
steps:
|
||||
- name: Free Disk Space (Ubuntu)
|
||||
uses: jlumbroso/free-disk-space@main
|
||||
with:
|
||||
tool-cache: false
|
||||
android: true
|
||||
dotnet: true
|
||||
haskell: true
|
||||
large-packages: false
|
||||
swap-storage: true
|
||||
docker-images: false
|
||||
|
||||
- name: Wait for the labeler
|
||||
uses: lewagon/wait-on-check-action@v1.7.0
|
||||
if: ${{ github.event_name == 'pull_request_target' }}
|
||||
with:
|
||||
ref: ${{ github.event.pull_request.head.sha }}
|
||||
check-name: Team Label
|
||||
repo-token: ${{ secrets.GITHUB_TOKEN }}
|
||||
wait-interval: 90
|
||||
|
||||
- name: Verify PR labels
|
||||
uses: jesusvasquez333/verify-pr-label-action@v1.4.0
|
||||
if: ${{ github.event_name == 'pull_request_target' }}
|
||||
with:
|
||||
github-token: "${{ secrets.GITHUB_TOKEN }}"
|
||||
valid-labels: "safe to test"
|
||||
pull-request-number: "${{ github.event.pull_request.number }}"
|
||||
disable-reviews: true # To not auto approve changes
|
||||
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
ref: ${{ github.event.pull_request.head.sha }}
|
||||
|
||||
- name: Setup Openmetadata Test Environment
|
||||
uses: ./.github/actions/setup-openmetadata-test-environment
|
||||
with:
|
||||
python-version: "3.10"
|
||||
args: "-m no-ui"
|
||||
ingestion_dependency: "mysql,elasticsearch,sample-data"
|
||||
|
||||
- name: Build Ingestion Operator & Ingest Sample Data
|
||||
run: |
|
||||
docker build -f ingestion/operators/docker/Dockerfile.ci . -t openmetadata/ingestion-base:test
|
||||
docker run --net=host --rm openmetadata/ingestion-base:test metadata ingest -c ./pipelines/sample_data.yaml
|
||||
|
||||
- name: Validate ingestion
|
||||
run: |
|
||||
source env/bin/activate
|
||||
python scripts/validate_sample_data.py
|
||||
|
||||
- name: Clean Up
|
||||
run: |
|
||||
cd ./docker/development
|
||||
docker compose down --remove-orphans
|
||||
sudo rm -rf ${PWD}/docker-volume
|
||||
@@ -0,0 +1,227 @@
|
||||
# Copyright 2021 Collate
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
name: SonarCloud Ingestion Nightly
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
schedule:
|
||||
- cron: "0 2 * * *"
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
concurrency:
|
||||
group: sonarcloud-ingestion-nightly
|
||||
cancel-in-progress: true
|
||||
|
||||
env:
|
||||
PYTHON_VERSION: "3.10"
|
||||
BRANCH_NAME: "main"
|
||||
PROJECT_BASE_DIR: "ingestion"
|
||||
|
||||
jobs:
|
||||
py-unit-tests:
|
||||
name: Unit Tests
|
||||
timeout-minutes: 60
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Free Disk Space (Ubuntu)
|
||||
uses: jlumbroso/free-disk-space@main
|
||||
with:
|
||||
tool-cache: false
|
||||
android: true
|
||||
dotnet: true
|
||||
haskell: true
|
||||
large-packages: false
|
||||
swap-storage: true
|
||||
docker-images: false
|
||||
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
ref: ${{ env.BRANCH_NAME }}
|
||||
|
||||
- name: Setup Openmetadata Test Environment
|
||||
uses: ./.github/actions/setup-openmetadata-test-environment
|
||||
with:
|
||||
python-version: ${{ env.PYTHON_VERSION }}
|
||||
install-server: 'false'
|
||||
|
||||
- name: Run Unit Tests
|
||||
run: |
|
||||
source env/bin/activate
|
||||
cd ingestion
|
||||
nox --no-venv -s unit-tests
|
||||
shell: bash
|
||||
|
||||
- name: Upload coverage artifact
|
||||
if: ${{ !cancelled() }}
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: coverage-unit
|
||||
path: ingestion/.coverage
|
||||
include-hidden-files: true
|
||||
|
||||
py-integration-tests:
|
||||
name: "Integration Tests (${{ matrix.shard.name }})"
|
||||
timeout-minutes: 180
|
||||
runs-on: ubuntu-latest
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
shard:
|
||||
- name: "shard-1"
|
||||
nox-args: >-
|
||||
tests/integration/ometa
|
||||
tests/integration/postgres
|
||||
tests/integration/mysql
|
||||
tests/integration/profiler
|
||||
tests/integration/data_quality
|
||||
- name: "shard-2"
|
||||
nox-args: >-
|
||||
--ignore=tests/integration/ometa
|
||||
--ignore=tests/integration/postgres
|
||||
--ignore=tests/integration/mysql
|
||||
--ignore=tests/integration/profiler
|
||||
--ignore=tests/integration/data_quality
|
||||
steps:
|
||||
- name: Free Disk Space (Ubuntu)
|
||||
uses: jlumbroso/free-disk-space@main
|
||||
with:
|
||||
tool-cache: false
|
||||
android: true
|
||||
dotnet: true
|
||||
haskell: true
|
||||
large-packages: false
|
||||
swap-storage: true
|
||||
docker-images: false
|
||||
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
ref: ${{ env.BRANCH_NAME }}
|
||||
|
||||
- name: Setup Openmetadata Test Environment
|
||||
uses: ./.github/actions/setup-openmetadata-test-environment
|
||||
with:
|
||||
python-version: ${{ env.PYTHON_VERSION }}
|
||||
args: "-m no-ui"
|
||||
ingestion_dependency: "mysql,elasticsearch,sample-data"
|
||||
|
||||
- name: Run Integration Tests
|
||||
run: |
|
||||
source env/bin/activate
|
||||
cd ingestion
|
||||
nox --no-venv -s integration-tests -- --standalone --durations=5 ${{ matrix.shard.nox-args }}
|
||||
env:
|
||||
TESTCONTAINERS_RYUK_DISABLED: true
|
||||
shell: bash
|
||||
|
||||
- name: Upload coverage artifact
|
||||
if: ${{ !cancelled() }}
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: coverage-integration-${{ matrix.shard.name }}
|
||||
path: ingestion/.coverage
|
||||
include-hidden-files: true
|
||||
|
||||
- name: Clean Up
|
||||
if: ${{ !cancelled() }}
|
||||
run: |
|
||||
cd ./docker/development
|
||||
docker compose down --remove-orphans
|
||||
sudo rm -rf ${PWD}/docker-volume
|
||||
|
||||
py-combine-coverage:
|
||||
if: ${{ !cancelled() }}
|
||||
needs: [py-unit-tests, py-integration-tests]
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 10
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
ref: ${{ env.BRANCH_NAME }}
|
||||
fetch-depth: 0
|
||||
filter: blob:none
|
||||
|
||||
- name: Setup Python
|
||||
uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: ${{ env.PYTHON_VERSION }}
|
||||
|
||||
- name: Install uv
|
||||
run: pip install uv
|
||||
shell: bash
|
||||
|
||||
- name: Install coverage
|
||||
run: |
|
||||
python3 -m venv env
|
||||
source env/bin/activate
|
||||
uv pip install "coverage[toml]" nox
|
||||
shell: bash
|
||||
|
||||
- name: Download coverage artifacts
|
||||
uses: actions/download-artifact@v4
|
||||
with:
|
||||
pattern: coverage-*
|
||||
path: ingestion/coverage-data/
|
||||
|
||||
- name: Prepare coverage files
|
||||
run: |
|
||||
cd ingestion
|
||||
[ -f coverage-data/coverage-unit/.coverage ] && mv coverage-data/coverage-unit/.coverage .coverage.unit
|
||||
for dir in coverage-data/coverage-integration-*/; do
|
||||
shard=$(basename "$dir" | sed 's/coverage-integration-//')
|
||||
[ -f "$dir/.coverage" ] && mv "$dir/.coverage" ".coverage.integration-$shard"
|
||||
done
|
||||
shell: bash
|
||||
|
||||
- name: Combine coverage
|
||||
run: |
|
||||
source env/bin/activate
|
||||
cd ingestion
|
||||
nox --no-venv -s combine-coverage
|
||||
shell: bash
|
||||
|
||||
- name: Remove pom.xml
|
||||
run: rm pom.xml
|
||||
shell: bash
|
||||
|
||||
- name: Push Results To Sonar
|
||||
if: ${{ !cancelled() }}
|
||||
id: push-to-sonar
|
||||
continue-on-error: true
|
||||
uses: SonarSource/sonarqube-scan-action@v7
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
SONAR_TOKEN: ${{ secrets.INGESTION_SONAR_SECRET }}
|
||||
with:
|
||||
projectBaseDir: ${{ env.PROJECT_BASE_DIR }}/
|
||||
args: >
|
||||
-Dsonar.branch.name=${{ env.BRANCH_NAME }}
|
||||
|
||||
- name: Wait to retry 'Push Results to Sonar'
|
||||
if: ${{ !cancelled() && steps.push-to-sonar.outcome != 'success' }}
|
||||
run: sleep 20s
|
||||
shell: bash
|
||||
|
||||
- name: Retry 'Push Results to Sonar'
|
||||
if: ${{ !cancelled() && steps.push-to-sonar.outcome != 'success' }}
|
||||
uses: SonarSource/sonarqube-scan-action@v7
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
SONAR_TOKEN: ${{ secrets.INGESTION_SONAR_SECRET }}
|
||||
with:
|
||||
projectBaseDir: ${{ env.PROJECT_BASE_DIR }}/
|
||||
args: >
|
||||
-Dsonar.branch.name=${{ env.BRANCH_NAME }}
|
||||
@@ -0,0 +1,141 @@
|
||||
# Copyright 2021 Collate
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
name: py-tests-postgres
|
||||
on:
|
||||
merge_group:
|
||||
workflow_dispatch:
|
||||
pull_request_target:
|
||||
types: [labeled, opened, synchronize, reopened, ready_for_review]
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
|
||||
cancel-in-progress: ${{ github.event_name != 'pull_request_target' || github.event.action != 'labeled' || github.event.label.name == 'safe to test' }}
|
||||
|
||||
jobs:
|
||||
# Detect whether relevant paths changed. When no Python/service/schema files
|
||||
# are modified the downstream jobs are skipped via their `if` condition.
|
||||
# A job skipped by `if` reports as "Success", so required checks still pass.
|
||||
changes:
|
||||
name: Detect Changes
|
||||
runs-on: ubuntu-latest
|
||||
if: ${{ !github.event.pull_request.draft && (github.event_name != 'pull_request_target' || github.event.action != 'labeled' || github.event.label.name == 'safe to test') }}
|
||||
outputs:
|
||||
python: ${{ github.event_name == 'workflow_dispatch' && 'true' || steps.filter.outputs.python }}
|
||||
steps:
|
||||
- uses: dorny/paths-filter@v3
|
||||
id: filter
|
||||
if: ${{ github.event_name != 'workflow_dispatch' }}
|
||||
with:
|
||||
filters: |
|
||||
python:
|
||||
- 'ingestion/**'
|
||||
- 'openmetadata-service/**'
|
||||
- 'openmetadata-spec/src/main/resources/json/schema/**'
|
||||
- 'pom.xml'
|
||||
- 'Makefile'
|
||||
|
||||
py-integration-tests:
|
||||
name: "Integration Tests (${{ matrix.shard.name }}, ${{ matrix.py-version }})"
|
||||
needs: changes
|
||||
if: ${{ needs.changes.outputs.python == 'true' }}
|
||||
timeout-minutes: 180
|
||||
runs-on: ubuntu-latest
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
py-version: ["3.10", "3.11", "3.12"]
|
||||
shard:
|
||||
- name: "shard-1"
|
||||
nox-args: >-
|
||||
tests/integration/ometa
|
||||
tests/integration/postgres
|
||||
tests/integration/mysql
|
||||
tests/integration/profiler
|
||||
tests/integration/data_quality
|
||||
- name: "shard-2"
|
||||
nox-args: >-
|
||||
--ignore=tests/integration/ometa
|
||||
--ignore=tests/integration/postgres
|
||||
--ignore=tests/integration/mysql
|
||||
--ignore=tests/integration/profiler
|
||||
--ignore=tests/integration/data_quality
|
||||
steps:
|
||||
- name: Free Disk Space (Ubuntu)
|
||||
uses: jlumbroso/free-disk-space@main
|
||||
with:
|
||||
tool-cache: false
|
||||
android: true
|
||||
dotnet: true
|
||||
haskell: true
|
||||
large-packages: false
|
||||
swap-storage: true
|
||||
docker-images: false
|
||||
|
||||
- name: Wait for the labeler
|
||||
uses: lewagon/wait-on-check-action@v1.7.0
|
||||
if: ${{ github.event_name == 'pull_request_target' }}
|
||||
with:
|
||||
ref: ${{ github.event.pull_request.head.sha }}
|
||||
check-name: Team Label
|
||||
repo-token: ${{ secrets.GITHUB_TOKEN }}
|
||||
wait-interval: 90
|
||||
|
||||
- name: Verify PR labels
|
||||
uses: jesusvasquez333/verify-pr-label-action@v1.4.0
|
||||
if: ${{ github.event_name == 'pull_request_target' }}
|
||||
with:
|
||||
github-token: "${{ secrets.GITHUB_TOKEN }}"
|
||||
valid-labels: "safe to test"
|
||||
pull-request-number: "${{ github.event.pull_request.number }}"
|
||||
disable-reviews: true # To not auto approve changes
|
||||
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
ref: ${{ github.event_name == 'merge_group' && github.sha || github.event.pull_request.head.sha }}
|
||||
|
||||
- name: Setup Openmetadata Test Environment
|
||||
uses: ./.github/actions/setup-openmetadata-test-environment
|
||||
with:
|
||||
python-version: ${{ matrix.py-version}}
|
||||
args: "-m no-ui -d postgresql"
|
||||
ingestion_dependency: "mysql,elasticsearch,sample-data"
|
||||
|
||||
- name: Run Integration Tests
|
||||
run: |
|
||||
source env/bin/activate
|
||||
cd ingestion
|
||||
nox --no-venv -s integration-tests -- --standalone --durations=5 ${{ matrix.shard.nox-args }}
|
||||
env:
|
||||
TESTCONTAINERS_RYUK_DISABLED: true
|
||||
shell: bash
|
||||
|
||||
- name: Clean Up
|
||||
run: |
|
||||
cd ./docker/development
|
||||
docker compose down --remove-orphans
|
||||
sudo rm -rf ${PWD}/docker-volume
|
||||
|
||||
# Single required-check gate for branch protection.
|
||||
# Skipped (= "Success") when all test jobs pass or are legitimately skipped.
|
||||
# Runs and exits 1 only when a test job fails or is cancelled.
|
||||
# Set "py-tests-postgres / py-tests-status" as the sole required check for this workflow.
|
||||
py-tests-status:
|
||||
name: py-tests-status
|
||||
needs: [changes, py-integration-tests]
|
||||
if: ${{ failure() || cancelled() }}
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- run: exit 1
|
||||
@@ -0,0 +1,324 @@
|
||||
# Copyright 2021 Collate
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
name: py-tests
|
||||
on:
|
||||
merge_group:
|
||||
workflow_dispatch:
|
||||
pull_request_target:
|
||||
types: [labeled, opened, synchronize, reopened, ready_for_review]
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
|
||||
cancel-in-progress: ${{ github.event_name != 'pull_request_target' || github.event.action != 'labeled' || github.event.label.name == 'safe to test' }}
|
||||
|
||||
env:
|
||||
# matrix can't use 'env'. When updating it, update it for both jobs.
|
||||
MAIN_PYTHON_VERSION: "3.10"
|
||||
SONAR_OPTS: >-
|
||||
-Dsonar.pullrequest.key=${{ github.event.pull_request.number }}
|
||||
-Dsonar.pullrequest.branch=${{ github.event.pull_request.head.ref }}
|
||||
-Dsonar.pullrequest.github.repository=OpenMetadata
|
||||
-Dsonar.scm.revision=${{ github.event.pull_request.head.sha }}
|
||||
-Dsonar.pullrequest.provider=github
|
||||
|
||||
jobs:
|
||||
# Detect whether relevant paths changed. When no Python/service/schema files
|
||||
# are modified the downstream jobs are skipped via their `if` condition.
|
||||
# A job skipped by `if` reports as "Success", so required checks still pass.
|
||||
# This replaces the old py-tests-skip.yml companion workflow.
|
||||
changes:
|
||||
name: Detect Changes
|
||||
runs-on: ubuntu-latest
|
||||
if: ${{ !github.event.pull_request.draft && (github.event_name != 'pull_request_target' || github.event.action != 'labeled' || github.event.label.name == 'safe to test') }}
|
||||
outputs:
|
||||
python: ${{ github.event_name == 'workflow_dispatch' && 'true' || steps.filter.outputs.python }}
|
||||
steps:
|
||||
- uses: dorny/paths-filter@v3
|
||||
id: filter
|
||||
if: ${{ github.event_name != 'workflow_dispatch' }}
|
||||
with:
|
||||
filters: |
|
||||
python:
|
||||
- 'ingestion/**'
|
||||
- 'openmetadata-service/**'
|
||||
- 'openmetadata-spec/src/main/resources/json/schema/**'
|
||||
- 'pom.xml'
|
||||
- 'Makefile'
|
||||
|
||||
py-unit-tests:
|
||||
name: Unit Tests & Static Checks
|
||||
needs: changes
|
||||
if: ${{ needs.changes.outputs.python == 'true' }}
|
||||
timeout-minutes: 60
|
||||
runs-on: ubuntu-latest
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
py-version: ["3.10", "3.11", "3.12"]
|
||||
steps:
|
||||
|
||||
- name: Wait for the labeler
|
||||
uses: lewagon/wait-on-check-action@v1.7.0
|
||||
if: ${{ github.event_name == 'pull_request_target' }}
|
||||
with:
|
||||
ref: ${{ github.event.pull_request.head.sha }}
|
||||
check-name: Team Label
|
||||
repo-token: ${{ secrets.GITHUB_TOKEN }}
|
||||
wait-interval: 30
|
||||
|
||||
- name: Verify PR labels
|
||||
uses: jesusvasquez333/verify-pr-label-action@v1.4.0
|
||||
if: ${{ github.event_name == 'pull_request_target' }}
|
||||
with:
|
||||
github-token: "${{ secrets.GITHUB_TOKEN }}"
|
||||
valid-labels: "safe to test"
|
||||
pull-request-number: "${{ github.event.pull_request.number }}"
|
||||
disable-reviews: true
|
||||
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
ref: ${{ github.event_name == 'merge_group' && github.sha || github.event.pull_request.head.sha }}
|
||||
|
||||
- name: Setup Openmetadata Test Environment
|
||||
uses: ./.github/actions/setup-openmetadata-test-environment
|
||||
with:
|
||||
python-version: ${{ matrix.py-version }}
|
||||
install-server: 'false'
|
||||
|
||||
- name: Run Static Checks
|
||||
# basedpyright is configured with `pythonVersion = "3.10"` (the lowest
|
||||
# supported version) so type-checking results are identical across the
|
||||
# 3.10/3.11/3.12 matrix. Run on the lowest version only to avoid
|
||||
# redundant work and keep the baseline file deterministic.
|
||||
if: matrix.py-version == '3.10'
|
||||
run: |
|
||||
source env/bin/activate
|
||||
cd ingestion
|
||||
nox --no-venv -s static-checks
|
||||
shell: bash
|
||||
|
||||
- name: Run Unit Tests
|
||||
run: |
|
||||
source env/bin/activate
|
||||
cd ingestion
|
||||
nox --no-venv -s unit-tests
|
||||
shell: bash
|
||||
|
||||
- name: Upload coverage artifact
|
||||
if: ${{ matrix.py-version == env.MAIN_PYTHON_VERSION && !cancelled() }}
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: coverage-unit
|
||||
path: ingestion/.coverage
|
||||
include-hidden-files: true
|
||||
|
||||
py-integration-tests:
|
||||
name: "Integration Tests (${{ matrix.shard.name }}, ${{ matrix.py-version }})"
|
||||
needs: changes
|
||||
if: ${{ needs.changes.outputs.python == 'true' }}
|
||||
timeout-minutes: 180
|
||||
runs-on: ubuntu-latest
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
py-version: ["3.10", "3.11", "3.12"]
|
||||
shard:
|
||||
- name: "shard-1"
|
||||
nox-args: >-
|
||||
tests/integration/ometa
|
||||
tests/integration/postgres
|
||||
tests/integration/mysql
|
||||
tests/integration/profiler
|
||||
tests/integration/data_quality
|
||||
- name: "shard-2"
|
||||
nox-args: >-
|
||||
--ignore=tests/integration/ometa
|
||||
--ignore=tests/integration/postgres
|
||||
--ignore=tests/integration/mysql
|
||||
--ignore=tests/integration/profiler
|
||||
--ignore=tests/integration/data_quality
|
||||
steps:
|
||||
- name: Free Disk Space (Ubuntu)
|
||||
uses: jlumbroso/free-disk-space@main
|
||||
with:
|
||||
tool-cache: false
|
||||
android: true
|
||||
dotnet: true
|
||||
haskell: true
|
||||
large-packages: false
|
||||
swap-storage: true
|
||||
docker-images: false
|
||||
|
||||
- name: Wait for the labeler
|
||||
uses: lewagon/wait-on-check-action@v1.7.0
|
||||
if: ${{ github.event_name == 'pull_request_target' }}
|
||||
with:
|
||||
ref: ${{ github.event.pull_request.head.sha }}
|
||||
check-name: Team Label
|
||||
repo-token: ${{ secrets.GITHUB_TOKEN }}
|
||||
wait-interval: 90
|
||||
|
||||
- name: Verify PR labels
|
||||
uses: jesusvasquez333/verify-pr-label-action@v1.4.0
|
||||
if: ${{ github.event_name == 'pull_request_target' }}
|
||||
with:
|
||||
github-token: "${{ secrets.GITHUB_TOKEN }}"
|
||||
valid-labels: "safe to test"
|
||||
pull-request-number: "${{ github.event.pull_request.number }}"
|
||||
disable-reviews: true # To not auto approve changes
|
||||
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
ref: ${{ github.event_name == 'merge_group' && github.sha || github.event.pull_request.head.sha }}
|
||||
|
||||
- name: Setup Openmetadata Test Environment
|
||||
uses: ./.github/actions/setup-openmetadata-test-environment
|
||||
with:
|
||||
python-version: ${{ matrix.py-version}}
|
||||
args: "-m no-ui"
|
||||
ingestion_dependency: "mysql,elasticsearch,sample-data"
|
||||
|
||||
- name: Run Integration Tests
|
||||
run: |
|
||||
source env/bin/activate
|
||||
cd ingestion
|
||||
read -r -a shard_args <<< "$NOX_ARGS"
|
||||
set +e
|
||||
nox --no-venv -s integration-tests -- --standalone --durations=5 "${shard_args[@]}" 2>&1 | tee integration-test-run.log
|
||||
status=${PIPESTATUS[0]}
|
||||
set -e
|
||||
if [ "$status" -ne 0 ] && { [ "$status" -eq 139 ] || [ "$status" -eq 245 ] || grep -Eq "failed with exit code -11|Segmentation fault|exit code 139" integration-test-run.log; }; then
|
||||
nox --no-venv -s integration-tests -- --standalone --durations=5 "${shard_args[@]}"
|
||||
else
|
||||
exit "$status"
|
||||
fi
|
||||
env:
|
||||
NOX_ARGS: ${{ matrix.shard.nox-args }}
|
||||
TESTCONTAINERS_RYUK_DISABLED: true
|
||||
shell: bash
|
||||
|
||||
- name: Upload coverage artifact
|
||||
if: ${{ matrix.py-version == env.MAIN_PYTHON_VERSION && !cancelled() }}
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: coverage-integration-${{ matrix.shard.name }}
|
||||
path: ingestion/.coverage
|
||||
include-hidden-files: true
|
||||
|
||||
- name: Clean Up
|
||||
run: |
|
||||
cd ./docker/development
|
||||
docker compose down --remove-orphans
|
||||
sudo rm -rf "${PWD}/docker-volume"
|
||||
|
||||
# Single required-check gate for branch protection.
|
||||
# Skipped (= "Success") when all test jobs pass or are legitimately skipped.
|
||||
# Runs and exits 1 only when a test job fails or is cancelled.
|
||||
# Set "py-tests / py-tests-status" as the sole required check for this workflow.
|
||||
py-tests-status:
|
||||
name: py-tests-status
|
||||
needs: [changes, py-unit-tests, py-integration-tests]
|
||||
if: ${{ failure() || cancelled() }}
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- run: exit 1
|
||||
|
||||
py-combine-coverage:
|
||||
needs: [changes, py-unit-tests, py-integration-tests]
|
||||
if: ${{ needs.changes.outputs.python == 'true' && !cancelled() }}
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 10
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
ref: ${{ github.event_name == 'merge_group' && github.sha || github.event.pull_request.head.sha }}
|
||||
fetch-depth: 0
|
||||
filter: blob:none
|
||||
|
||||
- name: Setup Python
|
||||
uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: ${{ env.MAIN_PYTHON_VERSION }}
|
||||
|
||||
- name: Install uv
|
||||
run: pip install uv
|
||||
shell: bash
|
||||
|
||||
- name: Install coverage
|
||||
run: |
|
||||
python3 -m venv env
|
||||
source env/bin/activate
|
||||
uv pip install "coverage[toml]" nox
|
||||
shell: bash
|
||||
|
||||
- name: Download coverage artifacts
|
||||
uses: actions/download-artifact@v4
|
||||
with:
|
||||
pattern: coverage-*
|
||||
path: ingestion/coverage-data/
|
||||
|
||||
- name: Prepare coverage files
|
||||
run: |
|
||||
cd ingestion
|
||||
[ -f coverage-data/coverage-unit/.coverage ] && mv coverage-data/coverage-unit/.coverage .coverage.unit
|
||||
for dir in coverage-data/coverage-integration-*/; do
|
||||
shard=$(basename "$dir" | sed 's/coverage-integration-//')
|
||||
[ -f "$dir/.coverage" ] && mv "$dir/.coverage" ".coverage.integration-$shard"
|
||||
done
|
||||
shell: bash
|
||||
|
||||
- name: Combine coverage
|
||||
run: |
|
||||
source env/bin/activate
|
||||
cd ingestion
|
||||
nox --no-venv -s combine-coverage
|
||||
shell: bash
|
||||
|
||||
- name: Remove pom.xml
|
||||
run: rm pom.xml
|
||||
shell: bash
|
||||
|
||||
# we have to pass these args values since we are working with the 'pull_request_target' trigger
|
||||
- name: Push Results in PR to Sonar
|
||||
id: push-to-sonar
|
||||
if: ${{ github.event_name == 'pull_request_target'}}
|
||||
continue-on-error: true
|
||||
uses: SonarSource/sonarqube-scan-action@v7
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
SONAR_TOKEN: ${{ secrets.INGESTION_SONAR_SECRET }}
|
||||
with:
|
||||
projectBaseDir: ingestion/
|
||||
args: ${{ env.SONAR_OPTS }}
|
||||
|
||||
# next two steps are for retrying "Push Results in PR to Sonar" step in case it fails
|
||||
- name: Wait to retry 'Push Results in PR to Sonar'
|
||||
if: ${{ github.event_name == 'pull_request_target' && steps.push-to-sonar.outcome != 'success' }}
|
||||
run: sleep 20s
|
||||
shell: bash
|
||||
|
||||
- name: Retry 'Push Results in PR to Sonar'
|
||||
uses: SonarSource/sonarqube-scan-action@v7
|
||||
if: ${{ github.event_name == 'pull_request_target' && steps.push-to-sonar.outcome != 'success' }}
|
||||
continue-on-error: true
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
SONAR_TOKEN: ${{ secrets.INGESTION_SONAR_SECRET }}
|
||||
with:
|
||||
projectBaseDir: ingestion/
|
||||
args: ${{ env.SONAR_OPTS }}
|
||||
@@ -0,0 +1,54 @@
|
||||
# Copyright 2021 Collate
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
name: Publish Python Packages
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
|
||||
jobs:
|
||||
publish:
|
||||
runs-on: ubuntu-latest
|
||||
environment: release
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- name: Set up Python 3.10
|
||||
uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: '3.10'
|
||||
- name: Install Ubuntu related dependencies
|
||||
run: |
|
||||
sudo apt-get update && sudo apt-get install -y libsasl2-dev unixodbc-dev python3-venv
|
||||
- name: Install and Publish PyPi packages for OpenMetadata Ingestion
|
||||
env:
|
||||
TWINE_USERNAME: '${{ secrets.TWINE_OPENMETADATA_INGESTION_USERNAME }}'
|
||||
TWINE_PASSWORD: '${{ secrets.TWINE_OPENMETADATA_INGESTION_PASSWORD }}'
|
||||
run: |
|
||||
python3 -m venv env
|
||||
source env/bin/activate
|
||||
sudo make install_antlr_cli
|
||||
make install_dev generate
|
||||
cd ingestion; \
|
||||
python -m build; \
|
||||
twine check dist/*; \
|
||||
twine upload dist/* --verbose
|
||||
- name: Install and Publish PyPi packages for OpenMetadata Managed (Airflow) APIs
|
||||
env:
|
||||
TWINE_USERNAME: '${{ secrets.TWINE_OPENMETADATA_AIRFLOW_MANAGED_APIS_USERNAME }}'
|
||||
TWINE_PASSWORD: '${{ secrets.TWINE_OPENMETADATA_AIRFLOW_MANAGED_APIS_PASSWORD }}'
|
||||
run: |
|
||||
python3 -m venv env
|
||||
source env/bin/activate
|
||||
make install_dev install_apis
|
||||
cd openmetadata-airflow-apis; \
|
||||
python -m build; \
|
||||
twine check dist/*; \
|
||||
twine upload dist/* --verbose
|
||||
@@ -0,0 +1,427 @@
|
||||
# Copyright 2021 Collate
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
name: security-scan
|
||||
on:
|
||||
schedule:
|
||||
- cron: "0 0 */2 * *"
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
SEND_SLACK_NOTIFICATION:
|
||||
description: "Post results to Slack (uncheck for test runs)"
|
||||
type: boolean
|
||||
required: false
|
||||
default: true
|
||||
|
||||
env:
|
||||
# Manual runs may opt out of Slack; schedule (where inputs are unset) always sends.
|
||||
# Explicit empty-check, since `||` would treat the string 'false' inconsistently when the
|
||||
# input is typed `boolean` — only an empty string should fall back to 'true'.
|
||||
SEND_SLACK_NOTIFICATION: ${{ github.event.inputs.SEND_SLACK_NOTIFICATION == '' && 'true' || github.event.inputs.SEND_SLACK_NOTIFICATION }}
|
||||
|
||||
jobs:
|
||||
vulnerability-scan:
|
||||
runs-on: ubuntu-latest
|
||||
environment: security-scan
|
||||
permissions:
|
||||
contents: read
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- name: Setup Node.js
|
||||
uses: actions/setup-node@v4
|
||||
with:
|
||||
node-version-file: "openmetadata-ui/src/main/resources/ui/.nvmrc"
|
||||
|
||||
- name: Enable yarn
|
||||
run: corepack enable
|
||||
|
||||
- name: Install UI dependencies
|
||||
working-directory: openmetadata-ui/src/main/resources/ui
|
||||
run: yarn install --frozen-lockfile --ignore-scripts
|
||||
|
||||
- name: Run Retire.js scan
|
||||
id: retire-scan
|
||||
continue-on-error: true
|
||||
working-directory: openmetadata-ui/src/main/resources/ui
|
||||
run: |
|
||||
npx retire@5 \
|
||||
--path node_modules/ \
|
||||
--severity high \
|
||||
--outputformat json \
|
||||
--outputpath retire-report.json
|
||||
|
||||
- name: Verify report was generated
|
||||
working-directory: openmetadata-ui/src/main/resources/ui
|
||||
run: |
|
||||
if [ ! -f retire-report.json ]; then
|
||||
echo '::error::retire-report.json was not generated — retire scan may have crashed'
|
||||
exit 1
|
||||
fi
|
||||
|
||||
- name: Upload Retire.js Report
|
||||
if: success()
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: retire-js-report
|
||||
path: openmetadata-ui/src/main/resources/ui/retire-report.json
|
||||
retention-days: 30
|
||||
|
||||
- name: Generate Retire.js Slack summary
|
||||
if: always()
|
||||
run: |
|
||||
mkdir -p retire-report
|
||||
python3 scripts/retire_slack_summary.py \
|
||||
openmetadata-ui/src/main/resources/ui/retire-report.json \
|
||||
--counts-file retire-report/_retire_counts.json \
|
||||
--slack-file retire-report/_retire_slack.txt \
|
||||
> /dev/null
|
||||
|
||||
- name: Upload Retire.js Slack artifact
|
||||
if: always()
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: retire-slack
|
||||
path: |
|
||||
retire-report/_retire_slack.txt
|
||||
retire-report/_retire_counts.json
|
||||
retention-days: 30
|
||||
|
||||
- name: Publish Retire.js Summary
|
||||
if: success()
|
||||
working-directory: openmetadata-ui/src/main/resources/ui
|
||||
run: |
|
||||
python3 - << 'EOF' >> $GITHUB_STEP_SUMMARY
|
||||
import json
|
||||
|
||||
SEVERITY_ICON = {"critical": "🚨", "high": "🔴", "medium": "🟠", "low": "🟡"}
|
||||
SEVERITY_ORDER = {"critical": 0, "high": 1, "medium": 2, "low": 3}
|
||||
NM = "node_modules/"
|
||||
|
||||
def escape(text):
|
||||
return str(text).replace('|', '\\|').replace('`', "'")
|
||||
|
||||
try:
|
||||
with open("retire-report.json") as f:
|
||||
data = json.load(f)
|
||||
except FileNotFoundError:
|
||||
print("## Retire.js Scan Results\n\n> Report file not found — scan may not have run.")
|
||||
raise SystemExit(0)
|
||||
|
||||
findings = data.get("data", [])
|
||||
libs = {}
|
||||
for item in findings:
|
||||
filepath = item.get("file", "")
|
||||
short = filepath[filepath.find(NM) + len(NM):] if NM in filepath else filepath
|
||||
for result in item.get("results", []):
|
||||
key = (result.get("component", ""), result.get("version", ""))
|
||||
if key not in libs:
|
||||
libs[key] = {"files": [], "vulns": result.get("vulnerabilities", [])}
|
||||
if short not in libs[key]["files"]:
|
||||
libs[key]["files"].append(short)
|
||||
|
||||
print("## Retire.js Scan Results\n")
|
||||
|
||||
if not libs:
|
||||
print("✅ No vulnerable libraries found.")
|
||||
else:
|
||||
total_vulns = sum(len(v["vulns"]) for v in libs.values())
|
||||
print(f"> **{len(libs)} vulnerable librar{'y' if len(libs) == 1 else 'ies'} · {total_vulns} CVE{'s' if total_vulns != 1 else ''} found**\n")
|
||||
|
||||
for (component, version), info in sorted(libs.items(), key=lambda x: min(
|
||||
(SEVERITY_ORDER.get(v.get("severity", "low"), 3) for v in x[1]["vulns"]), default=3)):
|
||||
top_sev = min(info["vulns"], key=lambda v: SEVERITY_ORDER.get(v.get("severity", "low"), 3))
|
||||
icon = SEVERITY_ICON.get(top_sev.get("severity", "low"), "⚪")
|
||||
print(f"### {icon} {component} {version}\n")
|
||||
print("| Severity | CVE | Summary |")
|
||||
print("|---|---|---|")
|
||||
for vuln in sorted(info["vulns"], key=lambda v: SEVERITY_ORDER.get(v.get("severity", "low"), 3)):
|
||||
sev = vuln.get("severity", "")
|
||||
ids = vuln.get("identifiers", {})
|
||||
cves = ids.get("CVE", [])
|
||||
summary = ids.get("summary", "").split("\n")[0][:120]
|
||||
cve_str = ", ".join(f"[{c}](https://nvd.nist.gov/vuln/detail/{c})" for c in cves) if cves else ids.get("githubID", "—")
|
||||
print(f"| {SEVERITY_ICON.get(sev, '')} {sev} | {escape(cve_str)} | {escape(summary)} |")
|
||||
print("\n**Bundled in:**")
|
||||
for f in info["files"]:
|
||||
print(f"- `{f}`")
|
||||
print()
|
||||
EOF
|
||||
|
||||
- name: Force failure on vulnerabilities found
|
||||
if: steps.retire-scan.outcome == 'failure'
|
||||
run: exit 1
|
||||
|
||||
security-scan:
|
||||
runs-on: ubuntu-latest
|
||||
environment: security-scan
|
||||
env:
|
||||
SNYK_TOKEN: ${{ secrets.SNYK_TOKEN }}
|
||||
SNYK_ORGANIZATION: ${{ secrets.SNYK_ORGANIZATION_ID }}
|
||||
|
||||
steps:
|
||||
- name: Free Disk Space (Ubuntu)
|
||||
uses: jlumbroso/free-disk-space@main
|
||||
with:
|
||||
tool-cache: false
|
||||
android: true
|
||||
dotnet: true
|
||||
haskell: true
|
||||
large-packages: false
|
||||
docker-images: true
|
||||
swap-storage: true
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- name: Set up Python 3.10
|
||||
uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: "3.10"
|
||||
|
||||
- name: Set up JDK 21
|
||||
uses: actions/setup-java@v4
|
||||
with:
|
||||
java-version: "21"
|
||||
distribution: "temurin"
|
||||
|
||||
- name: Install Ubuntu dependencies
|
||||
run: |
|
||||
# stop relying on apt cache of GitHub runners
|
||||
sudo apt-get update
|
||||
sudo apt-get install -y unixodbc-dev python3-venv librdkafka-dev gcc libsasl2-dev build-essential libssl-dev libffi-dev \
|
||||
librdkafka-dev unixodbc-dev libevent-dev wkhtmltopdf libkrb5-dev
|
||||
|
||||
# Install and Authenticate to Snyk
|
||||
- name: Install Snyk & Authenticate
|
||||
run: |
|
||||
sudo make install_antlr_cli
|
||||
sudo npm install -g snyk
|
||||
snyk auth ${SNYK_TOKEN}
|
||||
snyk config set org=${SNYK_ORGANIZATION}
|
||||
|
||||
- name: Install Python dependencies
|
||||
run: |
|
||||
python3 -m venv env
|
||||
source env/bin/activate
|
||||
make install_all install_apis
|
||||
|
||||
- name: Maven build
|
||||
id: maven-build
|
||||
continue-on-error: true
|
||||
run: mvn -DskipTests clean install
|
||||
|
||||
- name: Run Scan
|
||||
id: security-report
|
||||
if: steps.maven-build.outcome == 'success'
|
||||
continue-on-error: true
|
||||
run: |
|
||||
source env/bin/activate
|
||||
rm -rf security-report
|
||||
mkdir -p security-report
|
||||
# Run snyk subtargets directly; skip `export-snyk-pdf-report` which deletes JSONs after PDF conversion.
|
||||
make snyk-ingestion-report || true
|
||||
make snyk-ingestion-base-slim-report || true
|
||||
make snyk-airflow-apis-report || true
|
||||
make snyk-server-report || true
|
||||
make snyk-ui-report || true
|
||||
|
||||
- name: Publish Snyk Summary
|
||||
id: snyk-summary
|
||||
if: always() && steps.maven-build.outcome == 'success'
|
||||
run: |
|
||||
python3 scripts/snyk_summary.py security-report \
|
||||
--counts-file security-report/_counts.json \
|
||||
--slack-file security-report/_slack.txt \
|
||||
>> $GITHUB_STEP_SUMMARY
|
||||
# Expose counts as step output for downstream gating.
|
||||
counts=$(cat security-report/_counts.json)
|
||||
echo "counts=$counts" >> $GITHUB_OUTPUT
|
||||
high=$(jq '.high + .critical' security-report/_counts.json)
|
||||
echo "high_critical=$high" >> $GITHUB_OUTPUT
|
||||
|
||||
- name: Fail on high/critical Snyk findings
|
||||
if: always() && steps.snyk-summary.outputs.high_critical != '' && steps.snyk-summary.outputs.high_critical != '0'
|
||||
run: |
|
||||
echo "::error::Snyk found ${{ steps.snyk-summary.outputs.high_critical }} high/critical vulnerabilities (see Job Summary)"
|
||||
exit 1
|
||||
|
||||
- name: Generate Snyk HTML/PDF
|
||||
if: always() && steps.maven-build.outcome == 'success'
|
||||
run: |
|
||||
# Back up JSONs because html_to_pdf.py deletes them after PDF conversion.
|
||||
mkdir -p /tmp/snyk-json-backup
|
||||
cp security-report/*.json /tmp/snyk-json-backup/ 2>/dev/null || true
|
||||
make export-snyk-pdf-report || true
|
||||
# Restore JSONs alongside generated PDFs/HTMLs.
|
||||
cp /tmp/snyk-json-backup/*.json security-report/ 2>/dev/null || true
|
||||
|
||||
- name: Upload Snyk Reports
|
||||
if: always() && steps.maven-build.outcome == 'success'
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: security-report
|
||||
path: security-report
|
||||
retention-days: 30
|
||||
|
||||
- name: Force failure
|
||||
if: steps.maven-build.outcome != 'success' || steps.security-report.outcome != 'success'
|
||||
run: |
|
||||
exit 1
|
||||
|
||||
notify:
|
||||
runs-on: ubuntu-latest
|
||||
environment: security-scan
|
||||
needs: [vulnerability-scan, security-scan]
|
||||
if: always()
|
||||
steps:
|
||||
- name: Download Snyk artifact
|
||||
if: needs.security-scan.result != 'skipped'
|
||||
uses: actions/download-artifact@v4
|
||||
with:
|
||||
name: security-report
|
||||
path: security-report
|
||||
continue-on-error: true
|
||||
|
||||
- name: Download Retire.js Slack artifact
|
||||
if: needs.vulnerability-scan.result != 'skipped'
|
||||
uses: actions/download-artifact@v4
|
||||
with:
|
||||
name: retire-slack
|
||||
path: retire-report
|
||||
continue-on-error: true
|
||||
|
||||
- name: Build Slack header payload
|
||||
id: build-header
|
||||
env:
|
||||
RETIRE_RESULT: ${{ needs.vulnerability-scan.result }}
|
||||
SNYK_RESULT: ${{ needs.security-scan.result }}
|
||||
REF_NAME: ${{ github.ref_name }}
|
||||
RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
|
||||
run: |
|
||||
python3 - <<'PY' > header.json
|
||||
import json, os, pathlib
|
||||
|
||||
retire = os.environ["RETIRE_RESULT"]
|
||||
snyk = os.environ["SNYK_RESULT"]
|
||||
ref_name = os.environ["REF_NAME"]
|
||||
run_url = os.environ["RUN_URL"]
|
||||
|
||||
def status_icon(s):
|
||||
return {"success": "✅", "cancelled": "⚠️ (cancelled)", "skipped": "⚠️ (skipped)"}.get(s, "❌")
|
||||
|
||||
def load_counts(path):
|
||||
p = pathlib.Path(path)
|
||||
if not p.exists():
|
||||
return None
|
||||
try:
|
||||
return json.loads(p.read_text())
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
def totals_line(counts):
|
||||
if not counts:
|
||||
return ""
|
||||
return (
|
||||
f"\n🚨 {counts.get('critical', 0)} critical · "
|
||||
f"🔴 {counts.get('high', 0)} high · "
|
||||
f"🟠 {counts.get('medium', 0)} medium · "
|
||||
f"🟡 {counts.get('low', 0)} low"
|
||||
)
|
||||
|
||||
retire_counts = load_counts("retire-report/_retire_counts.json")
|
||||
snyk_counts = load_counts("security-report/_counts.json")
|
||||
|
||||
if retire == "success" and snyk == "success":
|
||||
overall = "🟢"
|
||||
elif retire == "failure" or snyk == "failure":
|
||||
overall = "🚨"
|
||||
else:
|
||||
overall = "⚠️"
|
||||
|
||||
header = (
|
||||
f"{overall} *Security scan* — *OpenMetadata Repo*\n"
|
||||
f"_branch_ `{ref_name}` · <{run_url}|Open run details>\n"
|
||||
f"• Vulnerability scan (Retire.js): {status_icon(retire)}"
|
||||
f"{totals_line(retire_counts)}\n"
|
||||
f"• Security scan (Snyk): {status_icon(snyk)}"
|
||||
f"{totals_line(snyk_counts)}"
|
||||
)
|
||||
fallback = f"{overall} Security scan — OpenMetadata Repo on {ref_name}. {run_url}"
|
||||
payload = {
|
||||
"text": fallback,
|
||||
"blocks": [{"type": "section", "text": {"type": "mrkdwn", "text": header}}],
|
||||
}
|
||||
print(json.dumps(payload))
|
||||
PY
|
||||
|
||||
- name: Send Slack — header
|
||||
id: send-header
|
||||
if: env.SEND_SLACK_NOTIFICATION == 'true'
|
||||
uses: slackapi/slack-github-action@v1.27.1
|
||||
with:
|
||||
channel-id: ${{ secrets.SLACK_CHANNEL_IDS }}
|
||||
payload-file-path: header.json
|
||||
env:
|
||||
SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }}
|
||||
|
||||
- name: Build Slack thread payload
|
||||
id: build-thread
|
||||
if: always() && steps.send-header.outputs.ts != ''
|
||||
env:
|
||||
THREAD_TS: ${{ steps.send-header.outputs.ts }}
|
||||
run: |
|
||||
python3 - <<'PY' > thread.json
|
||||
import json, os, pathlib
|
||||
|
||||
thread_ts = os.environ["THREAD_TS"]
|
||||
|
||||
def read_sections(path):
|
||||
p = pathlib.Path(path)
|
||||
if not p.exists():
|
||||
return []
|
||||
text = p.read_text().strip()
|
||||
return [s.strip() for s in text.split("\n\n") if s.strip()]
|
||||
|
||||
retire_sections = read_sections("retire-report/_retire_slack.txt")
|
||||
snyk_sections = read_sections("security-report/_slack.txt")
|
||||
|
||||
blocks = []
|
||||
MAX_TEXT = 2850
|
||||
|
||||
def push(section):
|
||||
text = section if len(section) <= MAX_TEXT else section[:MAX_TEXT].rstrip() + "\n_…truncated, see Job Summary_"
|
||||
blocks.append({"type": "section", "text": {"type": "mrkdwn", "text": text}})
|
||||
|
||||
for section in retire_sections[:24]:
|
||||
push(section)
|
||||
if retire_sections and snyk_sections:
|
||||
blocks.append({"type": "divider"})
|
||||
for section in snyk_sections[:23]:
|
||||
push(section)
|
||||
|
||||
if not blocks:
|
||||
push("_No detailed scan output was produced._")
|
||||
|
||||
payload = {
|
||||
"thread_ts": thread_ts,
|
||||
"text": "Security scan details (thread reply)",
|
||||
"blocks": blocks,
|
||||
}
|
||||
print(json.dumps(payload))
|
||||
PY
|
||||
|
||||
- name: Send Slack — thread reply
|
||||
if: always() && env.SEND_SLACK_NOTIFICATION == 'true' && steps.send-header.outputs.ts != '' && steps.build-thread.outcome == 'success'
|
||||
uses: slackapi/slack-github-action@v1.27.1
|
||||
with:
|
||||
channel-id: ${{ secrets.SLACK_CHANNEL_IDS }}
|
||||
payload-file-path: thread.json
|
||||
env:
|
||||
SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }}
|
||||
@@ -0,0 +1,36 @@
|
||||
name: Close Stale PRs
|
||||
|
||||
on:
|
||||
schedule:
|
||||
- cron: '0 9 * * *' # Runs daily at 9AM UTC
|
||||
workflow_dispatch:
|
||||
|
||||
permissions:
|
||||
pull-requests: write
|
||||
issues: write
|
||||
|
||||
jobs:
|
||||
stale:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/stale@v9
|
||||
with:
|
||||
repo-token: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
# PRs only (disable for issues)
|
||||
stale-issue-message: ''
|
||||
close-issue-message: ''
|
||||
days-before-issue-stale: -1
|
||||
days-before-issue-close: -1
|
||||
|
||||
# PR settings
|
||||
days-before-pr-stale: 30 # Mark stale after 30 days
|
||||
days-before-pr-close: 7 # Close 7 days after marking
|
||||
stale-pr-message: |
|
||||
This PR has had no activity for 30 days and will be closed in 7 days if no further activity occurs.
|
||||
Feel free to reopen it if you'd like to continue working on it.
|
||||
close-pr-message: |
|
||||
Closing due to inactivity. Reopen anytime to continue.
|
||||
exempt-pr-labels: 'wip'
|
||||
exempt-draft-pr: true
|
||||
operations-per-run: 50
|
||||
@@ -0,0 +1,94 @@
|
||||
# Copyright 2024 Collate
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
# Builds openmetadata-ui-core-components Storybook nightly and pushes the image
|
||||
# to Docker Hub so the design team can review components without a local dev
|
||||
# environment.
|
||||
#
|
||||
# Required repository secrets (same ones used by other docker-* workflows):
|
||||
# DOCKERHUB_OPENMETADATA_USERNAME
|
||||
# DOCKERHUB_OPENMETADATA_TOKEN
|
||||
#
|
||||
# Image: openmetadata/storybook-ui:latest
|
||||
# openmetadata/storybook-ui:<short-sha>
|
||||
#
|
||||
# To pull and run:
|
||||
# docker run -p 6006:6006 openmetadata/storybook-ui:latest
|
||||
|
||||
name: Storybook nightly build
|
||||
|
||||
on:
|
||||
schedule:
|
||||
# 02:00 UTC every day.
|
||||
- cron: '0 2 * * *'
|
||||
# Allow a one-click manual run from the Actions tab.
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
push_image:
|
||||
description: "Push the built image to Docker Hub?"
|
||||
type: boolean
|
||||
default: true
|
||||
|
||||
concurrency:
|
||||
group: storybook-nightly
|
||||
cancel-in-progress: true
|
||||
|
||||
jobs:
|
||||
build-and-push:
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 30
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Get short commit SHA
|
||||
id: sha
|
||||
run: echo "short=$(git rev-parse --short HEAD)" >> "$GITHUB_OUTPUT"
|
||||
|
||||
- name: Prepare Docker build
|
||||
id: prepare
|
||||
uses: ./.github/actions/prepare-for-docker-build-and-push
|
||||
with:
|
||||
image: openmetadata/storybook-ui
|
||||
tag: value=${{ steps.sha.outputs.short }}
|
||||
push_latest: "true"
|
||||
is_ingestion: "false"
|
||||
dockerhub_username: ${{ secrets.DOCKERHUB_OPENMETADATA_USERNAME }}
|
||||
dockerhub_token: ${{ secrets.DOCKERHUB_OPENMETADATA_TOKEN }}
|
||||
|
||||
- name: Build and push
|
||||
uses: docker/build-push-action@v6
|
||||
env:
|
||||
DOCKER_BUILD_NO_SUMMARY: true
|
||||
with:
|
||||
context: .
|
||||
file: docker/storybook/Dockerfile
|
||||
platforms: linux/amd64,linux/arm64
|
||||
# On scheduled runs always push; on workflow_dispatch respect the input.
|
||||
push: ${{ github.event_name == 'schedule' || inputs.push_image }}
|
||||
tags: ${{ steps.prepare.outputs.tags }}
|
||||
|
||||
- name: Print image reference
|
||||
run: |
|
||||
echo "### Storybook image published" >> "$GITHUB_STEP_SUMMARY"
|
||||
echo "" >> "$GITHUB_STEP_SUMMARY"
|
||||
echo '```' >> "$GITHUB_STEP_SUMMARY"
|
||||
echo "${{ steps.prepare.outputs.tags }}" >> "$GITHUB_STEP_SUMMARY"
|
||||
echo '```' >> "$GITHUB_STEP_SUMMARY"
|
||||
echo "" >> "$GITHUB_STEP_SUMMARY"
|
||||
echo "Run locally:" >> "$GITHUB_STEP_SUMMARY"
|
||||
echo '```bash' >> "$GITHUB_STEP_SUMMARY"
|
||||
echo "docker run -p 6006:6006 openmetadata/storybook-ui:latest" >> "$GITHUB_STEP_SUMMARY"
|
||||
echo '```' >> "$GITHUB_STEP_SUMMARY"
|
||||
@@ -0,0 +1,40 @@
|
||||
on:
|
||||
pull_request_target:
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
pull-requests: write
|
||||
issues: write
|
||||
|
||||
name: Team Label
|
||||
jobs:
|
||||
labeler:
|
||||
runs-on: ubuntu-latest
|
||||
name: Team Label
|
||||
steps:
|
||||
- uses: JulienKode/team-labeler-action@v0.1.1
|
||||
with:
|
||||
repo-token: "${{ secrets.GITHUB_TOKEN }}"
|
||||
|
||||
- name: Verify PR labels
|
||||
id: verify
|
||||
continue-on-error: true
|
||||
uses: jesusvasquez333/verify-pr-label-action@v1.4.0
|
||||
with:
|
||||
github-token: '${{ secrets.GITHUB_TOKEN }}'
|
||||
valid-labels: 'safe to test'
|
||||
pull-request-number: '${{ github.event.pull_request.number }}'
|
||||
disable-reviews: true # To not auto approve changes
|
||||
|
||||
- name: Add verification comment
|
||||
if: steps.verify.outcome != 'success'
|
||||
uses: peter-evans/create-or-update-comment@v1
|
||||
with:
|
||||
issue-number: ${{ github.event.pull_request.number }}
|
||||
body: |
|
||||
**Hi there 👋 Thanks for your contribution!**
|
||||
|
||||
The OpenMetadata team will review the PR shortly! Once it has been labeled as `safe to test`, the CI workflows
|
||||
will start executing and we'll be able to make sure everything is working as expected.
|
||||
|
||||
Let us know if you need any help!
|
||||
@@ -0,0 +1,57 @@
|
||||
name: Trivy Scan For OpenMetadata Ingestion Base Slim Docker Image
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
|
||||
concurrency:
|
||||
group: trivy-ingestion-base-slim-scan-${{ github.run_id }}
|
||||
cancel-in-progress: true
|
||||
|
||||
jobs:
|
||||
build-and-scan:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Free Disk Space (Ubuntu)
|
||||
uses: jlumbroso/free-disk-space@main
|
||||
with:
|
||||
tool-cache: false
|
||||
android: true
|
||||
dotnet: true
|
||||
haskell: true
|
||||
large-packages: false
|
||||
swap-storage: true
|
||||
docker-images: false
|
||||
|
||||
- name: Checkout Repository
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Prepare for Docker Build
|
||||
id: prepare
|
||||
uses: ./.github/actions/prepare-for-docker-build
|
||||
with:
|
||||
image: openmetadata-ingestion-base-slim
|
||||
tag: trivy
|
||||
is_ingestion: true
|
||||
|
||||
|
||||
- name: Build Docker Image
|
||||
run: |
|
||||
docker build -t openmetadata-ingestion-base-slim:trivy -f ingestion/operators/docker/Dockerfile.ci .
|
||||
|
||||
|
||||
- name: Run Trivy Image Scan
|
||||
id: trivy_scan
|
||||
uses: aquasecurity/trivy-action@0.35.0
|
||||
with:
|
||||
scan-type: "image"
|
||||
image-ref: openmetadata-ingestion-base-slim:trivy
|
||||
hide-progress: false
|
||||
ignore-unfixed: true
|
||||
severity: "HIGH,CRITICAL"
|
||||
skip-dirs: "/opt/airflow/dags,/home/airflow/ingestion/pipelines"
|
||||
scan-ref: .
|
||||
format: 'template'
|
||||
template: "@.github/trivy/templates/github.tpl"
|
||||
output: "trivy-result-ingestion-base-slim.md"
|
||||
env:
|
||||
TRIVY_DISABLE_VEX_NOTICE: "true"
|
||||
@@ -0,0 +1,57 @@
|
||||
name: Trivy Scan For OpenMetadata Ingestion Docker Image
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
|
||||
concurrency:
|
||||
group: trivy-ingestion-scan-${{ github.run_id }}
|
||||
cancel-in-progress: true
|
||||
|
||||
jobs:
|
||||
build-and-scan:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Free Disk Space (Ubuntu)
|
||||
uses: jlumbroso/free-disk-space@main
|
||||
with:
|
||||
tool-cache: false
|
||||
android: true
|
||||
dotnet: true
|
||||
haskell: true
|
||||
large-packages: false
|
||||
swap-storage: true
|
||||
docker-images: false
|
||||
|
||||
- name: Checkout Repository
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Prepare for Docker Build
|
||||
id: prepare
|
||||
uses: ./.github/actions/prepare-for-docker-build
|
||||
with:
|
||||
image: openmetadata-ingestion
|
||||
tag: trivy
|
||||
is_ingestion: true
|
||||
|
||||
|
||||
- name: Build Docker Image
|
||||
run: |
|
||||
docker build -t openmetadata-ingestion:trivy -f ingestion/Dockerfile.ci .
|
||||
|
||||
|
||||
- name: Run Trivy Image Scan
|
||||
id: trivy_scan
|
||||
uses: aquasecurity/trivy-action@0.35.0
|
||||
with:
|
||||
scan-type: "image"
|
||||
image-ref: openmetadata-ingestion:trivy
|
||||
hide-progress: false
|
||||
ignore-unfixed: true
|
||||
severity: "HIGH,CRITICAL"
|
||||
skip-dirs: "/opt/airflow/dags,/home/airflow/ingestion/pipelines"
|
||||
scan-ref: .
|
||||
format: 'template'
|
||||
template: "@.github/trivy/templates/github.tpl"
|
||||
output: "trivy-results-ingestion.md"
|
||||
env:
|
||||
TRIVY_DISABLE_VEX_NOTICE: "true"
|
||||
@@ -0,0 +1,56 @@
|
||||
name: Trivy Scan For OpenMetadata Server Docker Image
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
|
||||
concurrency:
|
||||
group: trivy-server-scan-${{ github.run_id }}
|
||||
cancel-in-progress: true
|
||||
|
||||
jobs:
|
||||
build-and-scan:
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
steps:
|
||||
- name: Checkout Repository
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Prepare for Docker Build
|
||||
id: prepare
|
||||
uses: ./.github/actions/prepare-for-docker-build
|
||||
with:
|
||||
image: openmetadata-server
|
||||
tag: trivy
|
||||
is_ingestion: false
|
||||
|
||||
- name: Build Docker Image
|
||||
run: |
|
||||
docker build -t openmetadata-server:trivy -f docker/development/Dockerfile .
|
||||
- name: Run Trivy Image Scan
|
||||
uses: aquasecurity/trivy-action@0.35.0
|
||||
with:
|
||||
scan-type: "image"
|
||||
image-ref: openmetadata-server:trivy
|
||||
hide-progress: true
|
||||
ignore-unfixed: true
|
||||
severity: "HIGH,CRITICAL,MEDIUM"
|
||||
format: "table"
|
||||
output: trivy.txt
|
||||
env:
|
||||
TRIVY_DISABLE_VEX_NOTICE: "true"
|
||||
|
||||
- name: Publish Trivy Output to Summary
|
||||
run: |
|
||||
if [[ -s trivy.txt ]]; then
|
||||
{
|
||||
echo "### Trivy Security Scan Results"
|
||||
echo "<details><summary>Click to expand</summary>"
|
||||
echo ""
|
||||
echo '```text'
|
||||
cat trivy.txt
|
||||
echo '```'
|
||||
echo "</details>"
|
||||
} >> $GITHUB_STEP_SUMMARY
|
||||
else
|
||||
echo "No vulnerabilities found." >> $GITHUB_STEP_SUMMARY
|
||||
fi
|
||||
@@ -0,0 +1,200 @@
|
||||
name: TypeScript Type Generation
|
||||
|
||||
on:
|
||||
merge_group:
|
||||
pull_request_target:
|
||||
types: [opened, synchronize, reopened, labeled, ready_for_review]
|
||||
paths:
|
||||
- 'openmetadata-spec/src/main/resources/json/schema/**'
|
||||
- 'openmetadata-ui/src/main/resources/ui/src/generated/**'
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
branch:
|
||||
description: 'Branch to run the workflow on (ignored if pr_number is provided)'
|
||||
required: false
|
||||
default: 'main'
|
||||
type: string
|
||||
pr_number:
|
||||
description: 'PR number to run the workflow for (works for both forked and non-forked PRs)'
|
||||
required: false
|
||||
type: string
|
||||
|
||||
concurrency:
|
||||
group: typescript-generation-${{ github.event.pull_request.number || github.run_id }}
|
||||
cancel-in-progress: ${{ github.event_name != 'pull_request_target' || github.event.action != 'labeled' || github.event.label.name == 'safe to test' }}
|
||||
|
||||
jobs:
|
||||
generate-types:
|
||||
runs-on: ubuntu-latest
|
||||
if: ${{ !github.event.pull_request.draft && (github.event_name != 'pull_request_target' || github.event.action != 'labeled' || github.event.label.name == 'safe to test') }}
|
||||
permissions:
|
||||
contents: write
|
||||
pull-requests: write
|
||||
steps:
|
||||
- name: Get PR details for workflow_dispatch
|
||||
if: github.event_name == 'workflow_dispatch' && github.event.inputs.pr_number != ''
|
||||
id: pr-details
|
||||
env:
|
||||
PR_NUMBER: ${{ github.event.inputs.pr_number }}
|
||||
BASE_REPO: ${{ github.repository }}
|
||||
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
run: |
|
||||
PR_DATA=$(gh pr view "$PR_NUMBER" --json headRepository,headRefName,headRepositoryOwner,headRefOid --repo "$BASE_REPO")
|
||||
echo "head_repo=$(echo "$PR_DATA" | jq -r '.headRepository.name')" >> $GITHUB_OUTPUT
|
||||
echo "head_owner=$(echo "$PR_DATA" | jq -r '.headRepositoryOwner.login')" >> $GITHUB_OUTPUT
|
||||
echo "head_ref=$(echo "$PR_DATA" | jq -r '.headRefName')" >> $GITHUB_OUTPUT
|
||||
echo "head_sha=$(echo "$PR_DATA" | jq -r '.headRefOid')" >> $GITHUB_OUTPUT
|
||||
echo "full_repo=$(echo "$PR_DATA" | jq -r '.headRepositoryOwner.login + "/" + .headRepository.name')" >> $GITHUB_OUTPUT
|
||||
|
||||
- name: Wait for the labeler
|
||||
uses: lewagon/wait-on-check-action@v1.7.0
|
||||
if: ${{ github.event_name == 'pull_request_target' && github.actor != 'github-actions[bot]' }}
|
||||
with:
|
||||
ref: ${{ github.event.pull_request.head.sha }}
|
||||
check-name: Team Label
|
||||
repo-token: ${{ secrets.GITHUB_TOKEN }}
|
||||
wait-interval: 90
|
||||
|
||||
- name: Verify PR labels
|
||||
uses: jesusvasquez333/verify-pr-label-action@v1.4.0
|
||||
if: ${{ github.event_name == 'pull_request_target' && github.actor != 'github-actions[bot]' }}
|
||||
with:
|
||||
github-token: '${{ secrets.GITHUB_TOKEN }}'
|
||||
valid-labels: 'safe to test'
|
||||
pull-request-number: '${{ github.event.pull_request.number }}'
|
||||
disable-reviews: true
|
||||
|
||||
- name: Check if repository is a fork
|
||||
id: check-fork
|
||||
env:
|
||||
EVENT_NAME: ${{ github.event_name }}
|
||||
FULL_REPO: ${{ steps.pr-details.outputs.full_repo }}
|
||||
HEAD_REPO_FULL_NAME: ${{ github.event.pull_request.head.repo.full_name }}
|
||||
BASE_REPO: ${{ github.repository }}
|
||||
run: |
|
||||
if [ "$EVENT_NAME" == "workflow_dispatch" ] && [ -n "$FULL_REPO" ]; then
|
||||
# For workflow_dispatch with PR number
|
||||
if [ "$FULL_REPO" != "$BASE_REPO" ]; then
|
||||
echo "is_fork=true" >> $GITHUB_OUTPUT
|
||||
else
|
||||
echo "is_fork=false" >> $GITHUB_OUTPUT
|
||||
fi
|
||||
elif [ "$EVENT_NAME" == "pull_request_target" ]; then
|
||||
# For pull_request_target events
|
||||
if [ "$HEAD_REPO_FULL_NAME" != "$BASE_REPO" ]; then
|
||||
echo "is_fork=true" >> $GITHUB_OUTPUT
|
||||
else
|
||||
echo "is_fork=false" >> $GITHUB_OUTPUT
|
||||
fi
|
||||
else
|
||||
# Default to non-fork for direct branch runs
|
||||
echo "is_fork=false" >> $GITHUB_OUTPUT
|
||||
fi
|
||||
|
||||
- name: Determine checkout ref
|
||||
id: checkout-ref
|
||||
env:
|
||||
IS_FORK: ${{ steps.check-fork.outputs.is_fork }}
|
||||
EVENT_NAME: ${{ github.event_name }}
|
||||
PR_HEAD_SHA: ${{ github.event.pull_request.head.sha }}
|
||||
PR_HEAD_REF: ${{ github.event.pull_request.head.ref }}
|
||||
PR_DETAILS_HEAD_SHA: ${{ steps.pr-details.outputs.head_sha }}
|
||||
PR_DETAILS_HEAD_REF: ${{ steps.pr-details.outputs.head_ref }}
|
||||
INPUT_BRANCH: ${{ github.event.inputs.branch }}
|
||||
GITHUB_REF: ${{ github.ref }}
|
||||
GITHUB_SHA_VAL: ${{ github.sha }}
|
||||
run: |
|
||||
if [ "$EVENT_NAME" == "merge_group" ]; then
|
||||
echo "ref=$GITHUB_SHA_VAL" >> $GITHUB_OUTPUT
|
||||
elif [ "$IS_FORK" == "true" ] && [ "$EVENT_NAME" == "pull_request_target" ]; then
|
||||
echo "ref=$PR_HEAD_SHA" >> $GITHUB_OUTPUT
|
||||
elif [ "$EVENT_NAME" == "pull_request_target" ]; then
|
||||
echo "ref=$PR_HEAD_REF" >> $GITHUB_OUTPUT
|
||||
elif [ "$IS_FORK" == "true" ] && [ -n "$PR_DETAILS_HEAD_SHA" ]; then
|
||||
echo "ref=$PR_DETAILS_HEAD_SHA" >> $GITHUB_OUTPUT
|
||||
elif [ -n "$PR_DETAILS_HEAD_REF" ]; then
|
||||
echo "ref=$PR_DETAILS_HEAD_REF" >> $GITHUB_OUTPUT
|
||||
elif [ -n "$INPUT_BRANCH" ]; then
|
||||
echo "ref=$INPUT_BRANCH" >> $GITHUB_OUTPUT
|
||||
else
|
||||
echo "ref=$GITHUB_REF" >> $GITHUB_OUTPUT
|
||||
fi
|
||||
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
repository: ${{ steps.pr-details.outputs.full_repo || github.repository }}
|
||||
ref: ${{ steps.checkout-ref.outputs.ref }}
|
||||
token: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
- name: Set up Node.js
|
||||
uses: actions/setup-node@v4
|
||||
with:
|
||||
node-version-file: 'openmetadata-ui/src/main/resources/ui/.nvmrc'
|
||||
|
||||
- name: Install dependencies
|
||||
run: |
|
||||
yarn install --frozen-lockfile
|
||||
|
||||
- name: Generate TypeScript types
|
||||
run: |
|
||||
cd openmetadata-ui/src/main/resources/ui
|
||||
# Create a symlink to the root node_modules
|
||||
ln -sf ../../../../../node_modules .
|
||||
./json2ts-generate-all.sh -l true
|
||||
|
||||
- name: Check for changes
|
||||
id: git-check
|
||||
continue-on-error: true
|
||||
run: |
|
||||
git add openmetadata-ui/src/main/resources/ui/src/generated/
|
||||
git diff --cached --quiet
|
||||
|
||||
- name: Configure Git
|
||||
if: steps.git-check.outcome != 'success'
|
||||
run: |
|
||||
git config --global user.name 'github-actions[bot]'
|
||||
git config --global user.email 'github-actions[bot]@users.noreply.github.com'
|
||||
|
||||
- name: Commit and push changes
|
||||
if: steps.git-check.outcome != 'success' && steps.check-fork.outputs.is_fork == 'false'
|
||||
run: |
|
||||
git commit -m "Update generated TypeScript types"
|
||||
git push --force-with-lease
|
||||
|
||||
- name: Create PR comment about auto-update
|
||||
if: steps.git-check.outcome != 'success' && steps.check-fork.outputs.is_fork == 'false' && (github.event_name == 'pull_request_target' || github.event.inputs.pr_number != '')
|
||||
uses: peter-evans/create-or-update-comment@v4
|
||||
with:
|
||||
issue-number: ${{ github.event.pull_request.number || github.event.inputs.pr_number }}
|
||||
body: |
|
||||
## ✅ TypeScript Types Auto-Updated
|
||||
|
||||
The generated TypeScript types have been automatically updated based on JSON schema changes in this PR.
|
||||
|
||||
- name: Create PR comment for fork repository
|
||||
if: steps.git-check.outcome != 'success' && steps.check-fork.outputs.is_fork == 'true' && (github.event_name == 'pull_request_target' || github.event.inputs.pr_number != '')
|
||||
uses: peter-evans/create-or-update-comment@v4
|
||||
with:
|
||||
issue-number: ${{ github.event.pull_request.number || github.event.inputs.pr_number }}
|
||||
body: |
|
||||
## ⚠️ TypeScript Types Need Update
|
||||
|
||||
**The generated TypeScript types are out of sync with the JSON schema changes.**
|
||||
|
||||
Since this is a pull request from a forked repository, the types cannot be automatically committed.
|
||||
Please generate and commit the types manually:
|
||||
|
||||
```bash
|
||||
cd openmetadata-ui/src/main/resources/ui
|
||||
./json2ts-generate-all.sh -l true
|
||||
git add src/generated/
|
||||
git commit -m "Update generated TypeScript types"
|
||||
git push
|
||||
```
|
||||
|
||||
After pushing the changes, this check will pass automatically.
|
||||
|
||||
- name: Fail workflow for fork PRs with type changes
|
||||
if: steps.git-check.outcome != 'success' && steps.check-fork.outputs.is_fork == 'true'
|
||||
run: exit 1
|
||||
@@ -0,0 +1,418 @@
|
||||
# Copyright 2021 Collate
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
name: UI Checkstyle
|
||||
|
||||
on:
|
||||
merge_group:
|
||||
# Note: paths filter removed — the workflow always triggers so that the
|
||||
# required status check (report) always completes. Path filtering is handled
|
||||
# inside the workflow via dorny/paths-filter so PRs without UI changes skip
|
||||
# the expensive jobs while still reporting a passing status.
|
||||
pull_request_target:
|
||||
types:
|
||||
- opened
|
||||
- synchronize
|
||||
- reopened
|
||||
- ready_for_review
|
||||
- labeled
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
concurrency:
|
||||
group: ui-checkstyle-${{ github.head_ref || github.run_id }}
|
||||
cancel-in-progress: ${{ github.event_name != 'pull_request_target' || github.event.action != 'labeled' || github.event.label.name == 'safe to test' }}
|
||||
|
||||
env:
|
||||
UI_WORKING_DIRECTORY: openmetadata-ui/src/main/resources/ui
|
||||
CORE_COMPONENTS_WORKING_DIRECTORY: openmetadata-ui-core-components/src/main/resources/ui
|
||||
|
||||
jobs:
|
||||
check-changes:
|
||||
runs-on: ubuntu-latest
|
||||
outputs:
|
||||
ui-changed: ${{ steps.filter.outputs.ui }}
|
||||
steps:
|
||||
- uses: dorny/paths-filter@v4
|
||||
id: filter
|
||||
with:
|
||||
filters: |
|
||||
ui:
|
||||
- 'openmetadata-ui/src/main/resources/ui/**'
|
||||
- 'openmetadata-spec/src/main/resources/json/schema/**'
|
||||
- '.github/workflows/ui-checkstyle.yml'
|
||||
- 'openmetadata-ui-core-components/src/main/resources/ui/**'
|
||||
|
||||
authorize:
|
||||
needs: check-changes
|
||||
if: |
|
||||
github.event_name == 'merge_group' ||
|
||||
(github.event_name == 'pull_request_target' && needs.check-changes.outputs.ui-changed == 'true' &&
|
||||
(github.event.action != 'labeled' || github.event.label.name == 'safe to test'))
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Wait for the labeler
|
||||
uses: lewagon/wait-on-check-action@v1.7.0
|
||||
if: ${{ github.event_name == 'pull_request_target' }}
|
||||
with:
|
||||
ref: ${{ github.event_name == 'merge_group' && github.sha || github.event.pull_request.head.sha }}
|
||||
check-name: Team Label
|
||||
repo-token: ${{ secrets.GITHUB_TOKEN }}
|
||||
wait-interval: 90
|
||||
|
||||
- name: Verify PR labels
|
||||
uses: jesusvasquez333/verify-pr-label-action@v1.4.0
|
||||
if: ${{ github.event_name == 'pull_request_target' }}
|
||||
with:
|
||||
github-token: "${{ secrets.GITHUB_TOKEN }}"
|
||||
valid-labels: "safe to test"
|
||||
pull-request-number: "${{ github.event.pull_request.number }}"
|
||||
disable-reviews: true
|
||||
|
||||
checkstyle:
|
||||
needs: authorize
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
contents: read
|
||||
outputs:
|
||||
lint_src_result: ${{ steps.lint_src.outcome }}
|
||||
lint_src_changed_files: ${{ steps.lint_src.outputs.changed_files }}
|
||||
license_result: ${{ steps.license.outcome }}
|
||||
license_changed_files: ${{ steps.license.outputs.changed_files }}
|
||||
i18n_result: ${{ steps.i18n.outcome }}
|
||||
i18n_changed_files: ${{ steps.i18n.outputs.changed_files }}
|
||||
app_docs_result: ${{ steps.app_docs.outcome }}
|
||||
app_docs_changed_files: ${{ steps.app_docs.outputs.changed_files }}
|
||||
lint_playwright_result: ${{ steps.lint_playwright.outcome }}
|
||||
lint_playwright_changed_files: ${{ steps.lint_playwright.outputs.changed_files }}
|
||||
lint_core_components_result: ${{ steps.lint_core_components.outcome }}
|
||||
lint_core_components_changed_files: ${{ steps.lint_core_components.outputs.changed_files }}
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
ref: ${{ github.event_name == 'merge_group' && github.sha || github.event.pull_request.head.sha }}
|
||||
fetch-depth: 0
|
||||
filter: blob:none
|
||||
|
||||
- uses: actions/setup-node@v4
|
||||
with:
|
||||
node-version-file: "${{ env.UI_WORKING_DIRECTORY }}/.nvmrc"
|
||||
cache: yarn
|
||||
cache-dependency-path: |
|
||||
${{ env.UI_WORKING_DIRECTORY }}/yarn.lock
|
||||
${{ env.CORE_COMPONENTS_WORKING_DIRECTORY }}/yarn.lock
|
||||
|
||||
- name: Install Antlr4 CLI
|
||||
run: sudo make install_antlr_cli
|
||||
|
||||
- name: Install UI Yarn Packages
|
||||
working-directory: ${{ env.UI_WORKING_DIRECTORY }}
|
||||
run: yarn install --frozen-lockfile
|
||||
|
||||
- name: Get changed src files
|
||||
id: changed-src-files
|
||||
uses: tj-actions/changed-files@22103cc46bda19c2b464ffe86db46df6922fd323
|
||||
with:
|
||||
path: ${{ env.UI_WORKING_DIRECTORY }}
|
||||
files_ignore: |
|
||||
src/generated/**
|
||||
files: |
|
||||
src/**/*.{ts,tsx,js,jsx,json}
|
||||
|
||||
- name: ESLint + Prettier + Organise Imports (src)
|
||||
id: lint_src
|
||||
if: steps.changed-src-files.outputs.any_changed == 'true'
|
||||
continue-on-error: true
|
||||
working-directory: ${{ env.UI_WORKING_DIRECTORY }}
|
||||
env:
|
||||
CHANGED_FILES: ${{ steps.changed-src-files.outputs.all_changed_files }}
|
||||
run: |
|
||||
if [ -z "$CHANGED_FILES" ]; then
|
||||
echo "No added or modified files to process."
|
||||
exit 0
|
||||
fi
|
||||
TS_FILES=$(echo "$CHANGED_FILES" | tr ' ' '\n' | awk '/\.(ts|tsx|js|jsx)$/ { printf "%s ", $0 }')
|
||||
if [ -n "$TS_FILES" ]; then
|
||||
yarn organize-imports:cli $TS_FILES
|
||||
fi
|
||||
yarn lint:base --fix $CHANGED_FILES
|
||||
yarn pretty:base --write $CHANGED_FILES
|
||||
if [ -n "$(git status --porcelain)" ]; then
|
||||
FILES=$(git status --porcelain | awk '{print " - `" $2 "`"}' | head -30)
|
||||
echo "changed_files<<EOF" >> "$GITHUB_OUTPUT"
|
||||
echo "$FILES" >> "$GITHUB_OUTPUT"
|
||||
echo "EOF" >> "$GITHUB_OUTPUT"
|
||||
git checkout -- .
|
||||
git clean -fd
|
||||
exit 1
|
||||
fi
|
||||
|
||||
- name: Get all changed UI files
|
||||
id: changed-ui-files
|
||||
uses: tj-actions/changed-files@22103cc46bda19c2b464ffe86db46df6922fd323
|
||||
with:
|
||||
path: ${{ env.UI_WORKING_DIRECTORY }}
|
||||
|
||||
- name: Licence Header Check
|
||||
id: license
|
||||
if: steps.changed-ui-files.outputs.any_changed == 'true'
|
||||
continue-on-error: true
|
||||
working-directory: ${{ env.UI_WORKING_DIRECTORY }}
|
||||
env:
|
||||
CHANGED_FILES_ALL: ${{ steps.changed-ui-files.outputs.all_changed_files }}
|
||||
run: |
|
||||
if [ -z "$CHANGED_FILES_ALL" ]; then
|
||||
echo "No added or modified files to process."
|
||||
exit 0
|
||||
fi
|
||||
yarn license-header-fix $CHANGED_FILES_ALL
|
||||
if [ -n "$(git status --porcelain)" ]; then
|
||||
FILES=$(git status --porcelain | awk '{print " - `" $2 "`"}' | head -30)
|
||||
echo "changed_files<<EOF" >> "$GITHUB_OUTPUT"
|
||||
echo "$FILES" >> "$GITHUB_OUTPUT"
|
||||
echo "EOF" >> "$GITHUB_OUTPUT"
|
||||
git checkout -- .
|
||||
git clean -fd
|
||||
exit 1
|
||||
fi
|
||||
|
||||
- name: I18n Sync
|
||||
id: i18n
|
||||
continue-on-error: true
|
||||
working-directory: ${{ env.UI_WORKING_DIRECTORY }}
|
||||
run: |
|
||||
yarn i18n
|
||||
if [ -n "$(git status --porcelain)" ]; then
|
||||
FILES=$(git status --porcelain | awk '{print " - `" $2 "`"}' | head -20)
|
||||
echo "changed_files<<EOF" >> "$GITHUB_OUTPUT"
|
||||
echo "$FILES" >> "$GITHUB_OUTPUT"
|
||||
echo "EOF" >> "$GITHUB_OUTPUT"
|
||||
git checkout -- .
|
||||
git clean -fd
|
||||
exit 1
|
||||
fi
|
||||
|
||||
- name: generate:app-docs
|
||||
id: app_docs
|
||||
continue-on-error: true
|
||||
working-directory: ${{ env.UI_WORKING_DIRECTORY }}
|
||||
run: |
|
||||
yarn generate:app-docs
|
||||
if [ -n "$(git status --porcelain)" ]; then
|
||||
FILES=$(git status --porcelain | awk '{print " - `" $2 "`"}' | head -20)
|
||||
echo "changed_files<<EOF" >> "$GITHUB_OUTPUT"
|
||||
echo "$FILES" >> "$GITHUB_OUTPUT"
|
||||
echo "EOF" >> "$GITHUB_OUTPUT"
|
||||
git checkout -- .
|
||||
git clean -fd
|
||||
exit 1
|
||||
fi
|
||||
|
||||
- name: Get changed Playwright files
|
||||
id: changed-playwright-files
|
||||
uses: tj-actions/changed-files@22103cc46bda19c2b464ffe86db46df6922fd323
|
||||
with:
|
||||
path: ${{ env.UI_WORKING_DIRECTORY }}
|
||||
files_ignore: |
|
||||
playwright/test-data/**
|
||||
files: |
|
||||
playwright/**/*.{ts,tsx,js,jsx}
|
||||
|
||||
- name: ESLint + Prettier + Organise Imports (playwright)
|
||||
id: lint_playwright
|
||||
if: steps.changed-playwright-files.outputs.any_changed == 'true'
|
||||
continue-on-error: true
|
||||
working-directory: ${{ env.UI_WORKING_DIRECTORY }}
|
||||
env:
|
||||
CHANGED_FILES: ${{ steps.changed-playwright-files.outputs.all_changed_files }}
|
||||
run: |
|
||||
if [ -z "$CHANGED_FILES" ]; then
|
||||
echo "No added or modified files to process."
|
||||
exit 0
|
||||
fi
|
||||
yarn organize-imports:cli $CHANGED_FILES
|
||||
yarn lint:base --fix $CHANGED_FILES
|
||||
yarn pretty:base --write $CHANGED_FILES
|
||||
if [ -n "$(git status --porcelain)" ]; then
|
||||
FILES=$(git status --porcelain | awk '{print " - `" $2 "`"}' | head -30)
|
||||
echo "changed_files<<EOF" >> "$GITHUB_OUTPUT"
|
||||
echo "$FILES" >> "$GITHUB_OUTPUT"
|
||||
echo "EOF" >> "$GITHUB_OUTPUT"
|
||||
git checkout -- .
|
||||
git clean -fd
|
||||
exit 1
|
||||
fi
|
||||
|
||||
- name: Install Core Components Yarn Packages
|
||||
working-directory: ${{ env.CORE_COMPONENTS_WORKING_DIRECTORY }}
|
||||
run: yarn install --frozen-lockfile
|
||||
|
||||
- name: Get changed core-components files
|
||||
id: changed-core-components-files
|
||||
uses: tj-actions/changed-files@22103cc46bda19c2b464ffe86db46df6922fd323
|
||||
with:
|
||||
path: ${{ env.CORE_COMPONENTS_WORKING_DIRECTORY }}
|
||||
files: |
|
||||
src/**/*.{ts,tsx,js,jsx,json}
|
||||
|
||||
- name: ESLint + Prettier (core-components)
|
||||
id: lint_core_components
|
||||
if: steps.changed-core-components-files.outputs.any_changed == 'true'
|
||||
continue-on-error: true
|
||||
working-directory: ${{ env.CORE_COMPONENTS_WORKING_DIRECTORY }}
|
||||
env:
|
||||
CHANGED_FILES: ${{ steps.changed-core-components-files.outputs.all_changed_files }}
|
||||
run: |
|
||||
if [ -z "${CHANGED_FILES// }" ]; then
|
||||
echo "No added or modified files to process."
|
||||
exit 0
|
||||
fi
|
||||
yarn lint:base --fix $CHANGED_FILES
|
||||
yarn pretty:base --write $CHANGED_FILES
|
||||
if [ -n "$(git status --porcelain)" ]; then
|
||||
FILES=$(git status --porcelain | awk '{print " - `" $2 "`"}' | head -30)
|
||||
echo "changed_files<<EOF" >> "$GITHUB_OUTPUT"
|
||||
echo "$FILES" >> "$GITHUB_OUTPUT"
|
||||
echo "EOF" >> "$GITHUB_OUTPUT"
|
||||
git checkout -- .
|
||||
git clean -fd
|
||||
exit 1
|
||||
fi
|
||||
|
||||
ui-checkstyle:
|
||||
needs: [authorize, checkstyle]
|
||||
if: always()
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
pull-requests: write
|
||||
steps:
|
||||
- name: Find existing summary comment
|
||||
uses: peter-evans/find-comment@v3
|
||||
id: fc
|
||||
if: ${{ github.event_name == 'pull_request_target' }}
|
||||
with:
|
||||
issue-number: ${{ github.event.pull_request.number }}
|
||||
comment-author: github-actions[bot]
|
||||
body-includes: '<!-- check:ui-checkstyle-summary -->'
|
||||
|
||||
- name: Post or update summary comment
|
||||
uses: actions/github-script@v7
|
||||
if: ${{ github.event_name == 'pull_request_target' }}
|
||||
with:
|
||||
script: |
|
||||
const checks = [
|
||||
{
|
||||
name: 'ESLint + Prettier + Organise Imports (src)',
|
||||
reason: 'One or more source files have linting or formatting issues.',
|
||||
result: '${{ needs.checkstyle.outputs.lint_src_result }}',
|
||||
files: ${{ toJSON(needs.checkstyle.outputs.lint_src_changed_files) }} || '',
|
||||
},
|
||||
{
|
||||
name: 'Licence Header',
|
||||
reason: 'One or more files are missing or have an outdated Apache 2.0 licence header.',
|
||||
result: '${{ needs.checkstyle.outputs.license_result }}',
|
||||
files: ${{ toJSON(needs.checkstyle.outputs.license_changed_files) }} || '',
|
||||
},
|
||||
{
|
||||
name: 'I18n Sync',
|
||||
reason: 'Translation locale files are out of sync with `en-us.json`.',
|
||||
result: '${{ needs.checkstyle.outputs.i18n_result }}',
|
||||
files: ${{ toJSON(needs.checkstyle.outputs.i18n_changed_files) }} || '',
|
||||
},
|
||||
{
|
||||
name: 'App Docs',
|
||||
reason: 'Generated application docs are stale and need to be regenerated.',
|
||||
result: '${{ needs.checkstyle.outputs.app_docs_result }}',
|
||||
files: ${{ toJSON(needs.checkstyle.outputs.app_docs_changed_files) }} || '',
|
||||
},
|
||||
{
|
||||
name: 'Playwright - ESLint + Prettier + Organise Imports',
|
||||
reason: 'One or more Playwright test files have linting or formatting issues.',
|
||||
result: '${{ needs.checkstyle.outputs.lint_playwright_result }}',
|
||||
files: ${{ toJSON(needs.checkstyle.outputs.lint_playwright_changed_files) }} || '',
|
||||
},
|
||||
{
|
||||
name: 'Core Components - ESLint + Prettier',
|
||||
reason: 'One or more core-component files have linting or formatting issues.',
|
||||
result: '${{ needs.checkstyle.outputs.lint_core_components_result }}',
|
||||
files: ${{ toJSON(needs.checkstyle.outputs.lint_core_components_changed_files) }} || '',
|
||||
},
|
||||
];
|
||||
|
||||
const failures = checks.filter(c => c.result === 'failure');
|
||||
const commentId = '${{ steps.fc.outputs.comment-id }}';
|
||||
|
||||
if (failures.length === 0) {
|
||||
if (commentId) {
|
||||
await github.rest.issues.deleteComment({
|
||||
owner: context.repo.owner,
|
||||
repo: context.repo.repo,
|
||||
comment_id: parseInt(commentId, 10),
|
||||
});
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
const sections = failures.map(c => {
|
||||
const lines = [`#### ❌ ${c.name}`, c.reason];
|
||||
if (c.files && c.files.trim()) {
|
||||
lines.push('', '<details><summary>Affected files</summary>', '', c.files.trim(), '', '</details>');
|
||||
}
|
||||
return lines.join('\n');
|
||||
});
|
||||
|
||||
const body = [
|
||||
'<!-- check:ui-checkstyle-summary -->',
|
||||
'### ❌ UI Checkstyle Failed',
|
||||
'',
|
||||
...sections.flatMap(s => [s, '']),
|
||||
'---',
|
||||
'**Fix locally (fast - only checks files changed in this branch):**',
|
||||
'```bash',
|
||||
'make ui-checkstyle-changed',
|
||||
'```',
|
||||
].join('\n');
|
||||
|
||||
if (commentId) {
|
||||
await github.rest.issues.updateComment({
|
||||
owner: context.repo.owner,
|
||||
repo: context.repo.repo,
|
||||
comment_id: parseInt(commentId, 10),
|
||||
body,
|
||||
});
|
||||
} else {
|
||||
await github.rest.issues.createComment({
|
||||
owner: context.repo.owner,
|
||||
repo: context.repo.repo,
|
||||
issue_number: context.issue.number,
|
||||
body,
|
||||
});
|
||||
}
|
||||
|
||||
- name: Check final results
|
||||
if: always()
|
||||
run: |
|
||||
checkstyle_result="${{ needs.checkstyle.result }}"
|
||||
if [[ "$checkstyle_result" != "success" && "$checkstyle_result" != "skipped" ]]; then
|
||||
echo "Checkstyle job ended with status: $checkstyle_result"
|
||||
exit 1
|
||||
fi
|
||||
if [ "${{ needs.checkstyle.outputs.lint_src_result }}" = 'failure' ] || \
|
||||
[ "${{ needs.checkstyle.outputs.license_result }}" = 'failure' ] || \
|
||||
[ "${{ needs.checkstyle.outputs.i18n_result }}" = 'failure' ] || \
|
||||
[ "${{ needs.checkstyle.outputs.app_docs_result }}" = 'failure' ] || \
|
||||
[ "${{ needs.checkstyle.outputs.lint_playwright_result }}" = 'failure' ] || \
|
||||
[ "${{ needs.checkstyle.outputs.lint_core_components_result }}" = 'failure' ]; then
|
||||
echo "One or more checks failed."
|
||||
exit 1
|
||||
fi
|
||||
echo "All checks passed or were not required for this PR."
|
||||
@@ -0,0 +1,93 @@
|
||||
# Copyright 2021 Collate
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
name: Update Playwright E2E Documentation
|
||||
on:
|
||||
workflow_dispatch:
|
||||
|
||||
jobs:
|
||||
update-docs:
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
contents: write
|
||||
pull-requests: write
|
||||
steps:
|
||||
- name: Validate Branch
|
||||
run: |
|
||||
if [ "${{ github.ref }}" != "refs/heads/main" ]; then
|
||||
echo "This workflow can only be run on the main branch."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
- name: Checkout Repository
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
fetch-depth: 0
|
||||
|
||||
- name: Setup Node.js
|
||||
uses: actions/setup-node@v4
|
||||
with:
|
||||
node-version-file: "openmetadata-ui/src/main/resources/ui/.nvmrc"
|
||||
cache: "yarn"
|
||||
cache-dependency-path: openmetadata-ui/src/main/resources/ui/yarn.lock
|
||||
|
||||
- name: Install Yarn
|
||||
run: corepack enable
|
||||
|
||||
- name: Install Dependencies
|
||||
working-directory: openmetadata-ui/src/main/resources/ui
|
||||
run: yarn install --frozen-lockfile --ignore-scripts
|
||||
|
||||
- name: Install Playwright Browsers
|
||||
working-directory: openmetadata-ui/src/main/resources/ui
|
||||
run: npx playwright install chromium --with-deps
|
||||
|
||||
- name: Generate E2E Docs
|
||||
working-directory: openmetadata-ui/src/main/resources/ui
|
||||
run: yarn generate:e2e-docs
|
||||
|
||||
- name: Detect Changes
|
||||
id: git-check
|
||||
run: |
|
||||
if [ -z "$(git status --porcelain openmetadata-ui/src/main/resources/ui/playwright/docs)" ]; then
|
||||
echo "No changes detected."
|
||||
echo "changed=false" >> $GITHUB_OUTPUT
|
||||
else
|
||||
echo "Changes detected."
|
||||
echo "changed=true" >> $GITHUB_OUTPUT
|
||||
fi
|
||||
|
||||
- name: Create Pull Request
|
||||
if: steps.git-check.outputs.changed == 'true'
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
run: |
|
||||
BRANCH_NAME="chore/update-playwright-docs"
|
||||
git config user.name "github-actions[bot]"
|
||||
git config user.email "github-actions[bot]@users.noreply.github.com"
|
||||
|
||||
git checkout -B $BRANCH_NAME
|
||||
git add openmetadata-ui/src/main/resources/ui/playwright/docs
|
||||
git commit -m "chore: update Playwright E2E documentation"
|
||||
git push origin $BRANCH_NAME --force
|
||||
|
||||
gh pr create \
|
||||
--title "chore: update Playwright E2E documentation" \
|
||||
--body "This is an automated PR to update the Playwright E2E documentation. Generated by the \`Update Playwright E2E Documentation\` workflow." \
|
||||
--base main \
|
||||
--head $BRANCH_NAME || {
|
||||
if gh pr list --head $BRANCH_NAME --state open --json number -jq '.[0].number' | grep -q .; then
|
||||
echo "PR already exists"
|
||||
else
|
||||
echo "Failed to create PR"
|
||||
exit 1
|
||||
fi
|
||||
}
|
||||
@@ -0,0 +1,42 @@
|
||||
# Copyright 2025 Collate
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
name: Validate Docker Compose Quickstart
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
pull_request:
|
||||
paths:
|
||||
- 'docker/docker-compose-quickstart/**'
|
||||
- '.github/workflows/validate-docker-compose-quickstart.yml'
|
||||
- '.github/actions/validate-omd-docker-compose/**'
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
validate-docker-compose:
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 30
|
||||
env:
|
||||
OPENMETADATA_DB_IMAGE: docker.getcollate.io/openmetadata/db:latest
|
||||
OPENMETADATA_SERVER_IMAGE: docker.getcollate.io/openmetadata/server:latest
|
||||
OPENMETADATA_INGESTION_IMAGE: docker.getcollate.io/openmetadata/ingestion:latest
|
||||
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Validate OpenMetadata Docker Compose
|
||||
uses: ./.github/actions/validate-omd-docker-compose
|
||||
with:
|
||||
compose_file: ./docker/docker-compose-quickstart/docker-compose.yml
|
||||
health_check_timeout: 300
|
||||
@@ -0,0 +1,85 @@
|
||||
# Copyright 2024 Collate
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
name: Validate JSON/YAML
|
||||
|
||||
# read-write repo token
|
||||
# access to secrets
|
||||
on:
|
||||
merge_group:
|
||||
pull_request_target:
|
||||
types: [labeled, opened, synchronize, reopened]
|
||||
paths:
|
||||
- "**.json"
|
||||
- "**.yaml"
|
||||
- "**.yml"
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
validate-json-yaml:
|
||||
runs-on: ubuntu-latest
|
||||
if: ${{ github.event_name != 'pull_request_target' || github.event.action != 'labeled' || github.event.label.name == 'safe to test' }}
|
||||
permissions:
|
||||
pull-requests: write
|
||||
steps:
|
||||
- name: Wait for the labeler
|
||||
uses: lewagon/wait-on-check-action@v1.7.0
|
||||
if: ${{ github.event_name == 'pull_request_target' }}
|
||||
with:
|
||||
ref: ${{ github.event.pull_request.head.sha }}
|
||||
check-name: Team Label
|
||||
repo-token: ${{ secrets.GITHUB_TOKEN }}
|
||||
wait-interval: 90
|
||||
|
||||
- name: Verify PR labels
|
||||
uses: jesusvasquez333/verify-pr-label-action@v1.4.0
|
||||
if: ${{ github.event_name == 'pull_request_target' }}
|
||||
with:
|
||||
github-token: '${{ secrets.GITHUB_TOKEN }}'
|
||||
valid-labels: 'safe to test'
|
||||
pull-request-number: '${{ github.event.pull_request.number }}'
|
||||
disable-reviews: true # To not auto approve changes
|
||||
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
ref: ${{ github.event_name == 'merge_group' && github.sha || github.event.pull_request.head.sha }}
|
||||
|
||||
- name: Set up Python 3.10
|
||||
uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: '3.10'
|
||||
|
||||
- name: Install Ubuntu related dependencies
|
||||
run: |
|
||||
sudo apt-get update && sudo apt-get install -y libsasl2-dev unixodbc-dev python3-venv
|
||||
|
||||
- name: Install Python & Openmetadata related dependencies
|
||||
run: |
|
||||
python3 -m venv env
|
||||
source env/bin/activate
|
||||
pip install pyyaml
|
||||
|
||||
# Add back linting once we have 10/10 on main
|
||||
- name: Code style check
|
||||
id: style
|
||||
continue-on-error: true
|
||||
run: |
|
||||
source env/bin/activate
|
||||
pip install pyyaml
|
||||
./scripts/validate_json_yaml.sh
|
||||
|
||||
- name: JSON/Yaml validation failed, check the comment in the PR
|
||||
if: steps.style.outcome != 'success'
|
||||
run: |
|
||||
exit 1
|
||||
@@ -0,0 +1,179 @@
|
||||
name: SonarCloud + Jest Coverage
|
||||
|
||||
on:
|
||||
merge_group:
|
||||
workflow_dispatch:
|
||||
# Trigger analysis when pushing in master or pull requests, and when creating
|
||||
# a pull request.
|
||||
# Note: paths filter removed — the workflow always triggers so that the
|
||||
# required status check (coverage-result) always completes. Path filtering
|
||||
# is handled inside the workflow via dorny/paths-filter, which lets PRs
|
||||
# without UI changes skip the expensive jobs while still reporting a
|
||||
# passing status.
|
||||
pull_request_target:
|
||||
types: [opened, synchronize, reopened, labeled, ready_for_review]
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
pull-requests: write # Required for Providing Jest Coverage Comment
|
||||
|
||||
env:
|
||||
UI_WORKING_DIRECTORY: openmetadata-ui/src/main/resources/ui
|
||||
UI_COVERAGE_DIRECTORY: openmetadata-ui/src/main/resources/ui/src/test/unit/coverage
|
||||
concurrency:
|
||||
group: yarn-coverage-${{ github.event.pull_request.number || github.run_id }}
|
||||
cancel-in-progress: ${{ github.event_name != 'pull_request_target' || github.event.action != 'labeled' || github.event.label.name == 'safe to test' }}
|
||||
jobs:
|
||||
check-changes:
|
||||
runs-on: ubuntu-latest
|
||||
outputs:
|
||||
ui-changed: ${{ steps.filter.outputs.ui }}
|
||||
steps:
|
||||
- uses: dorny/paths-filter@v4
|
||||
id: filter
|
||||
with:
|
||||
filters: |
|
||||
ui:
|
||||
- 'openmetadata-ui/src/main/resources/ui/src/**'
|
||||
- 'openmetadata-ui/src/main/resources/ui/jest.config.js'
|
||||
- 'openmetadata-ui/src/main/resources/ui/package.json'
|
||||
- 'openmetadata-ui/src/main/resources/ui/yarn.lock'
|
||||
- 'openmetadata-ui/src/main/resources/ui/tsconfig.json'
|
||||
- 'openmetadata-ui/src/main/resources/ui/babel.config.json'
|
||||
- 'openmetadata-ui/src/main/resources/ui/.nvmrc'
|
||||
- 'openmetadata-ui/src/main/resources/ui/sonar-project.properties'
|
||||
|
||||
ui-coverage-tests:
|
||||
needs: check-changes
|
||||
runs-on: ubuntu-latest
|
||||
if: |
|
||||
!github.event.pull_request.draft &&
|
||||
(github.event_name == 'merge_group' || github.event_name == 'workflow_dispatch' || needs.check-changes.outputs.ui-changed == 'true') &&
|
||||
(github.event_name != 'pull_request_target' || github.event.action != 'labeled' || github.event.label.name == 'safe to test')
|
||||
steps:
|
||||
- name: Wait for the labeler
|
||||
uses: lewagon/wait-on-check-action@v1.7.0
|
||||
if: ${{ github.event_name == 'pull_request_target' }}
|
||||
with:
|
||||
ref: ${{ github.event.pull_request.head.sha }}
|
||||
check-name: Team Label
|
||||
repo-token: ${{ secrets.GITHUB_TOKEN }}
|
||||
wait-interval: 90
|
||||
|
||||
- name: Verify PR labels
|
||||
uses: jesusvasquez333/verify-pr-label-action@v1.4.0
|
||||
if: ${{ github.event_name == 'pull_request_target' }}
|
||||
with:
|
||||
github-token: "${{ secrets.GITHUB_TOKEN }}"
|
||||
valid-labels: "safe to test"
|
||||
pull-request-number: "${{ github.event.pull_request.number }}"
|
||||
disable-reviews: true # To not auto approve changes
|
||||
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
ref: ${{ github.event_name == 'merge_group' && github.sha || github.event.pull_request.head.sha }}
|
||||
# Disabling shallow clone is recommended for improving relevancy of reporting.
|
||||
# Use partial clone (blob:none) to avoid downloading every blob in history —
|
||||
# commits/trees still available for Sonar blame; blobs fetched lazily as needed.
|
||||
fetch-depth: 0
|
||||
filter: blob:none
|
||||
|
||||
- name: Setup Node.js
|
||||
uses: actions/setup-node@v4
|
||||
with:
|
||||
node-version-file: "openmetadata-ui/src/main/resources/ui/.nvmrc"
|
||||
|
||||
- name: Get yarn cache directory path
|
||||
id: yarn-cache-dir-path
|
||||
run: echo "dir=$(yarn cache dir)" >> $GITHUB_OUTPUT
|
||||
|
||||
- name: Caching NPM dependencies
|
||||
uses: actions/cache@v4
|
||||
id: yarn-cache
|
||||
with:
|
||||
path: ${{ steps.yarn-cache-dir-path.outputs.dir }}
|
||||
key: ${{ runner.os }}-node-yarn-${{ hashFiles('**/yarn.lock') }}
|
||||
restore-keys: |
|
||||
${{ runner.os }}-node-yarn-
|
||||
|
||||
- name: Get npm cache directory
|
||||
id: npm-cache-dir
|
||||
shell: bash
|
||||
run: echo "dir=$(npm config get cache)" >> ${GITHUB_OUTPUT}
|
||||
|
||||
- name: Caching NPM dependencies
|
||||
uses: actions/cache@v4
|
||||
id: npm-cache
|
||||
with:
|
||||
path: ${{ steps.npm-cache-dir.outputs.dir }}
|
||||
key: ${{ runner.os }}-node-${{ hashFiles('**/package-lock.json') }}
|
||||
restore-keys: |
|
||||
${{ runner.os }}-node-
|
||||
|
||||
- name: Install Antlr4 CLI
|
||||
run: |
|
||||
sudo make install_antlr_cli
|
||||
- name: Install Yarn Packages
|
||||
working-directory: ${{ env.UI_WORKING_DIRECTORY }}
|
||||
run: yarn install
|
||||
- name: Run Coverage
|
||||
working-directory: ${{ env.UI_WORKING_DIRECTORY }}
|
||||
run: yarn test:cov-summary
|
||||
id: yarn_coverage
|
||||
- name: Jest coverage comment
|
||||
uses: MishaKav/jest-coverage-comment@v1.0.22
|
||||
with:
|
||||
coverage-summary-path: ${{env.UI_COVERAGE_DIRECTORY}}/coverage-summary.json
|
||||
title: Jest test Coverage
|
||||
summary-title: UI tests summary
|
||||
badge-title: Coverage
|
||||
- name: yarn add sonarqube-scanner
|
||||
working-directory: ${{ env.UI_WORKING_DIRECTORY }}
|
||||
run: npm install -g sonarqube-scanner
|
||||
id: npm_install_sonar_scanner
|
||||
- name: SonarCloud Scan On PR
|
||||
if: github.event_name == 'pull_request_target' && steps.npm_install_sonar_scanner.outcome == 'success'
|
||||
working-directory: ${{ env.UI_WORKING_DIRECTORY }}
|
||||
run: |
|
||||
sonar-scanner -Dsonar.host.url=${SONARCLOUD_URL} \
|
||||
-Dproject.settings=sonar-project.properties \
|
||||
-Dsonar.pullrequest.key=${{ github.event.pull_request.number }} \
|
||||
-Dsonar.pullrequest.branch=$PR_HEAD_REF \
|
||||
-Dsonar.pullrequest.base=main \
|
||||
-Dsonar.pullrequest.github.repository=OpenMetadata \
|
||||
-Dsonar.scm.revision=${{ github.event.pull_request.head.sha }} \
|
||||
-Dsonar.pullrequest.provider=github \
|
||||
-Dsonar.scm.disabled=true
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
SONAR_TOKEN: ${{ secrets.UI_SONAR_TOKEN }}
|
||||
SONARCLOUD_URL: https://sonarcloud.io
|
||||
PR_HEAD_REF: ${{ github.event.pull_request.head.ref }}
|
||||
- name: SonarCloud Scan
|
||||
if: github.event_name == 'push' && steps.npm_install_sonar_scanner.outcome == 'success'
|
||||
working-directory: ${{ env.UI_WORKING_DIRECTORY }}
|
||||
run: |
|
||||
sonar-scanner -Dsonar.host.url=${SONARCLOUD_URL} \
|
||||
-Dproject.settings=sonar-project.properties
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
SONAR_TOKEN: ${{ secrets.UI_SONAR_TOKEN }}
|
||||
SONARCLOUD_URL: https://sonarcloud.io
|
||||
|
||||
# This job is the one to set as the required status check in branch protection.
|
||||
# It always runs and passes when ui-coverage-tests succeeded OR was skipped
|
||||
# (i.e. no relevant UI paths changed), so PRs without UI changes are never blocked.
|
||||
ui-coverage:
|
||||
if: always()
|
||||
needs: [ui-coverage-tests]
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Check coverage result
|
||||
run: |
|
||||
result="${{ needs.ui-coverage-tests.result }}"
|
||||
if [[ "$result" == "success" || "$result" == "skipped" ]]; then
|
||||
echo "Coverage check passed or was not required for this PR"
|
||||
else
|
||||
echo "Coverage check failed with status: $result"
|
||||
exit 1
|
||||
fi
|
||||
Reference in New Issue
Block a user