chore: import upstream snapshot with attribution

This commit is contained in:
wehub-resource-sync
2026-07-13 12:32:25 +08:00
commit e014feafe1
2285 changed files with 1131979 additions and 0 deletions
+210
View File
@@ -0,0 +1,210 @@
name: Bump Deps
on:
repository_dispatch:
types: [ bump-dependency ]
jobs:
get-label:
name: Get Label
outputs:
label: ${{ steps.get-label.outputs.label }}
runs-on: ubuntu-22.04
steps:
- name: Get Label
id: get-label
env:
REPO: ${{ github.event.client_payload.dependency }}
run: |
if [ "$REPO" == "dolt" ]
then
echo "label=dolt-bump" >> $GITHUB_OUTPUT
else
echo "$REPO is unsupported"
exit 1
fi
stale-bump-prs:
name: Retrieving Stale Bump PRs
needs: get-label
outputs:
stale-pulls: ${{ steps.get-stale-prs.outputs.open-pulls }}
runs-on: ubuntu-22.04
steps:
- name: Get Open Bump PRs
id: get-stale-prs
uses: actions/github-script@v7
env:
LABEL: ${{ needs.get-label.outputs.label }}
with:
debug: true
github-token: ${{ secrets.GITHUB_TOKEN }}
script: |
try {
const { LABEL } = process.env;
const { owner, repo } = context.repo;
const res = await github.rest.pulls.list({
owner,
repo,
state: 'open',
sort: 'created',
direction: 'desc',
});
const { data } = res;
const reduced = data.reduce((acc, p) => {
if (p.labels.length < 1) return acc;
let keepAlive = false;
let shouldPush = false;
for (const label of p.labels) {
if (label.name === LABEL) {
shouldPush = true;
}
if (label.name === "keep-alive") {
keepAlive = true;
}
}
if (shouldPush) {
acc.push({
number: p.number,
keepAlive,
headRef: p.head.ref,
});
}
return acc;
}, []);
console.log(reduced);
if (reduced.length > 0) core.setOutput("open-pulls", JSON.stringify(reduced));
process.exit(0);
} catch(err) {
console.log("Error:", err);
process.exit(1);
}
open-bump-pr:
needs: [get-label, stale-bump-prs]
name: Open Bump PR
runs-on: ubuntu-22.04
outputs:
latest-pr: ${{ steps.latest-pr.outputs.pr_url }}
steps:
- uses: actions/checkout@v6
with:
token: ${{ secrets.GITHUB_TOKEN }}
- name: Set up Go 1.x
uses: actions/setup-go@v5
with:
go-version-file: go.mod
- name: Bump dependency
run: |
go get github.com/dolthub/${{ github.event.client_payload.dependency }}/go@${{ github.event.client_payload.head_commit_sha }}
go mod tidy
- name: Get short hash
id: short-sha
run: |
commit=${{ github.event.client_payload.head_commit_sha }}
short=${commit:0:8}
echo "short=$short" >> $GITHUB_OUTPUT
- name: Get Assignee and Reviewer
id: get_reviewer
run: |
if [ "${{ github.event.client_payload.assignee }}" == "github-actions[bot]" ]
then
echo "assignee=zachmu" >> $GITHUB_OUTPUT
else
assignee="${{ github.event.client_payload.assignee }}"
echo "assignee=$assignee" >> $GITHUB_OUTPUT
fi
if [ "${{ github.event.client_payload.assignee }}" == "zachmu" ]
then
echo "reviewer=Hydrocharged" >> $GITHUB_OUTPUT
else
echo "reviewer=zachmu" >> $GITHUB_OUTPUT
fi
- name: Create and Push new branch
run: |
git config --global --add user.name "${{ steps.get_reviewer.outputs.assignee }}"
git config --global --add user.email "${{ github.event.client_payload.assignee_email }}"
branchname=${{ format('{0}-{1}', steps.get_reviewer.outputs.assignee, steps.short-sha.outputs.short) }}
git checkout -b "$branchname"
git add .
git commit -m "${{ format('[ga-bump-dep] Bump dependency in Doltgres by {0}', steps.get_reviewer.outputs.assignee) }}"
git push origin "$branchname"
- name: pull-request
uses: repo-sync/pull-request@v2
id: latest-pr
with:
source_branch: ${{ format('{0}-{1}', steps.get_reviewer.outputs.assignee, steps.short-sha.outputs.short ) }}
destination_branch: "main"
github_token: ${{ secrets.PUBLIC_REPO_ACCESS_TOKEN || secrets.REPO_ACCESS_TOKEN || secrets.GITHUB_TOKEN }}
pr_title: "[auto-bump] [no-release-notes] dependency by ${{ steps.get_reviewer.outputs.assignee }}"
pr_template: ".github/markdown-templates/dep-bump.md"
pr_reviewer: ${{ steps.get_reviewer.outputs.reviewer }}
pr_assignee: ${{ github.event.client_payload.assignee }}
pr_label: ${{ needs.get-label.outputs.label }}
comment-on-stale-prs:
needs: [open-bump-pr, stale-bump-prs]
if: ${{ needs.stale-bump-prs.outputs.stale-pulls != '' }}
runs-on: ubuntu-22.04
strategy:
matrix:
pull: ${{ fromJson(needs.stale-bump-prs.outputs.stale-pulls) }}
steps:
- name: Comment/Close Stale PRs
id: get-stale-prs
uses: actions/github-script@v7
env:
PULL: ${{ toJson(matrix.pull) }}
SUPERSEDED_BY: ${{ needs.open-bump-pr.outputs.latest-pr }}
with:
debug: true
github-token: ${{ secrets.GITHUB_TOKEN }}
script: |
try {
const { owner, repo } = context.repo;
const { PULL, SUPERSEDED_BY } = process.env;
const pull = JSON.parse(PULL);
if (pull.keepAlive) process.exit(0);
const checkSuiteRes = await github.rest.checks.listSuitesForRef({
owner,
repo,
ref: pull.headRef,
});
if (checkSuiteRes.data) {
for (const suite of checkSuiteRes.data.check_suites) {
console.log("suite id:", suite.id);
console.log("suite app slug:", suite.app.slug);
console.log("suite status:", suite.status);
console.log("suite conclusion:", suite.conclusion);
}
}
console.log(`Closing open pr ${pull.number}`);
await github.rest.issues.createComment({
issue_number: pull.number,
owner,
repo,
body: `This PR has been superseded by ${SUPERSEDED_BY}`
});
await github.rest.pulls.update({
owner,
repo,
pull_number: pull.number,
state: 'closed',
});
process.exit(0);
} catch(err) {
console.log("Error:", err);
process.exit(1);
}
@@ -0,0 +1,69 @@
name: Create Release Notes
on:
workflow_dispatch:
inputs:
version:
description: 'SemVer format release tag, i.e. 0.2.4'
required: true
repository_dispatch:
types: [ release-notes ]
jobs:
create-release-notes:
runs-on: ubuntu-22.04
steps:
- name: Get Vars
id: get_vars
run: |
if [ "$EVENT_NAME" == "workflow_dispatch" ]
then
release_id=$(curl -H "Accept: application/vnd.github.v3+json" https://api.github.com/repos/dolthub/doltgresql/releases/tags/v${{ github.event.inputs.version }} | jq '.id')
echo "version=${{ github.event.inputs.version }}" >> $GITHUB_OUTPUT
echo "release_id=$release_id" >> $GITHUB_OUTPUT
else
echo "version=${{ github.event.client_payload.version }}" >> $GITHUB_OUTPUT
echo "release_id=${{ github.event.client_payload.release_id }}" >> $GITHUB_OUTPUT
fi
env:
EVENT_NAME: ${{ github.event_name }}
- name: Checkout Release Notes Generator
uses: actions/checkout@v6
with:
repository: dolthub/release-notes-generator
token: ${{ secrets.GITHUB_TOKEN }}
- name: Install Dependencies
run: sudo ./install-deps.sh
env:
PERL_MM_USE_DEFAULT: 1
- name: Create Notes
run: |
git clone https://github.com/dolthub/doltgresql.git
./gen_release_notes.pl \
--token "$TOKEN" dolthub/doltgresql v${{ steps.get_vars.outputs.version }} > changelog.txt
env:
TOKEN: ${{ secrets.GITHUB_TOKEN }}
- name: Post Changelog to Release
uses: actions/github-script@v7
with:
debug: true
github-token: ${{ secrets.GITHUB_TOKEN }}
script: |
const fs = require('fs');
const path = require('path')
try {
const body = fs.readFileSync(path.join(process.env.WORKSPACE, "changelog.txt"), { encoding: "utf8" })
const res = await github.rest.repos.updateRelease({
owner: "dolthub",
repo: "doltgresql",
release_id: parseInt(process.env.RELEASE_ID, 10),
body,
});
console.log("Successfully updated release notes", res)
} catch (err) {
console.log("Error", err);
process.exit(1);
}
env:
WORKSPACE: ${{ github.workspace }}
RELEASE_ID: ${{ steps.get_vars.outputs.release_id }}
@@ -0,0 +1,77 @@
name: Push Docker Image to DockerHub
on:
workflow_dispatch:
inputs:
version:
description: 'SemVer format release tag, i.e. 0.24.5'
required: true
repository_dispatch:
types: [ push-docker-image ]
jobs:
get-release-id:
name: Get Doltgres Release Id
runs-on: ubuntu-22.04
outputs:
release_id: ${{ steps.get_release.outputs.release_id }}
steps:
- name: Get Release
id: get_release
run: |
release_id="$RELEASE_ID"
if [ "$EVENT_TYPE" == "workflow_dispatch" ]; then
release_id=$(curl -H "Accept: application/vnd.github.v3+json" https://api.github.com/repos/dolthub/doltgresql/releases/tags/v${{ github.event.inputs.version }} | jq '.id')
fi
echo "release_id=$release_id" >> $GITHUB_OUTPUT
env:
EVENT_TYPE: ${{ github.event_name }}
RELEASE_ID: ${{ github.event.client_payload.release_id }}
docker-image-push:
name: Push Docker Image
needs: get-release-id
runs-on: ubuntu-22.04
steps:
- name: Checkout
uses: actions/checkout@v6
- name: Login to Docker Hub
uses: docker/login-action@v3
with:
username: ${{ secrets.DOCKER_HUB_USERNAME }}
password: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }}
- name: Set up QEMU
uses: docker/setup-qemu-action@v3
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v3
with:
platforms: linux/amd64,linux/arm64
- name: Build and push doltgres image
uses: docker/build-push-action@v5
with:
platforms: linux/amd64,linux/arm64
context: .
file: ./Dockerfile
push: true
tags: dolthub/doltgresql:${{ github.event.inputs.version || github.event.client_payload.version }} , dolthub/doltgresql:latest
build-args: |
DOLTGRES_VERSION=${{ github.event.inputs.version || github.event.client_payload.version }}
- name: Update Docker Hub Readme for doltgres image
uses: peter-evans/dockerhub-description@v4
with:
username: ${{ secrets.DOCKER_HUB_USERNAME }}
password: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }}
repository: dolthub/doltgresql
readme-filepath: ./dockerREADME.md
- run: |
gh api \
--method PATCH \
-H "Accept: application/vnd.github+json" \
-H "X-GitHub-Api-Version: 2022-11-28" \
/repos/$REPO_OWNER/$REPO_NAME/releases/$RELEASE_ID \
-F "draft=false" -F "prerelease=false" -F "make_latest=true"
env:
GH_TOKEN: ${{ secrets.REPO_ACCESS_TOKEN || secrets.GITHUB_TOKEN }}
REPO_OWNER: dolthub
REPO_NAME: doltgresql
RELEASE_ID: ${{ needs.get-release-id.outputs.release_id }}
+238
View File
@@ -0,0 +1,238 @@
name: Release DoltgreSQL
on:
workflow_dispatch:
inputs:
version:
description: 'SemVer format release tag, i.e. 0.2.4'
required: true
jobs:
format-version:
runs-on: ubuntu-22.04
outputs:
version: ${{ steps.format_version.outputs.version }}
steps:
- name: Format Input
id: format_version
run: |
version="${{ github.event.inputs.version }}"
if [[ $version == v* ]];
then
version="${version:1}"
fi
echo "version=$version" >> $GITHUB_OUTPUT
windows-extension-support:
runs-on: windows-latest
steps:
- name: Checkout code
uses: actions/checkout@v6
with:
token: ${{ secrets.PUBLIC_REPO_ACCESS_TOKEN || secrets.REPO_ACCESS_TOKEN || secrets.GITHUB_TOKEN }}
- name: Build SQL Syntax
run: ./build.sh
working-directory: ./postgres/parser
shell: bash
- name: Upload Extension Artifacts
uses: actions/upload-artifact@v4
with:
name: windows-extension-artifacts
path: |
./core/extensions/pg_extension/output/pg_extension.dll
./core/extensions/pg_extension/output/postgres.exe
if-no-files-found: error
retention-days: 1
create-release:
needs: [format-version, windows-extension-support]
name: Create release
runs-on: ubuntu-22.04
outputs:
release_id: ${{ steps.create_release.outputs.id }}
steps:
- name: Checkout code
uses: actions/checkout@v6
with:
token: ${{ secrets.PUBLIC_REPO_ACCESS_TOKEN || secrets.REPO_ACCESS_TOKEN || secrets.GITHUB_TOKEN }}
- name: Build SQL Syntax
run: ./build.sh
working-directory: ./postgres/parser
shell: bash
- name: Update Doltgres version command
run: sed -i -e 's/Version\s*=\s*".*"/Version = "'"$NEW_VERSION"'"/' "$FILE"
env:
FILE: ${{ format('{0}/server/server.go', github.workspace) }}
NEW_VERSION: ${{ needs.format-version.outputs.version }}
- name: Set minver TBD to version
run: sed -i -e 's/minver:"TBD"/minver:"'"$NEW_VERSION"'"/' "$FILE"
env:
FILE: ${{ format('{0}/servercfg/config.go', github.workspace) }}
NEW_VERSION: ${{ needs.format-version.outputs.version }}
- name: update minver_validation.txt
working-directory: ./
run: go run -mod=readonly ./utils/genminver_validation/ $FILE
env:
FILE: ${{ format('{0}/servercfg/cfgdetails/testdata/minver_validation.txt', github.workspace) }}
- uses: EndBug/add-and-commit@v9.1.1
with:
message: ${{ format('Update DoltgreSQL version to {0}', needs.format-version.outputs.version) }}
add: ${{ format('["{0}/server/server.go", "{0}/servercfg/config.go", "{0}/servercfg/cfgdetails/testdata/minver_validation.txt"]', github.workspace) }}
cwd: "."
new_branch: cd-release
push: 'origin cd-release --set-upstream --force'
- name: Create Pull Request
run: gh pr create --base main --head "cd-release" --title "[no-release-notes] Release v${{ needs.format-version.outputs.version }}" --body "Created by the Release workflow to update DoltgreSQL's version"
env:
GH_TOKEN: ${{ secrets.PUBLIC_REPO_ACCESS_TOKEN || secrets.REPO_ACCESS_TOKEN || secrets.GITHUB_TOKEN }}
- name: Enable Pull Request Auto-Merge
run: gh pr merge --merge --auto "cd-release"
env:
GH_TOKEN: ${{ secrets.PUBLIC_REPO_ACCESS_TOKEN || secrets.REPO_ACCESS_TOKEN || secrets.GITHUB_TOKEN }}
- name: Download Extension Artifacts
uses: actions/download-artifact@v4
with:
name: windows-extension-artifacts
path: ./core/extensions/pg_extension/output
- name: Install Go
uses: actions/setup-go@v5
with:
go-version-file: go.mod
- name: Free up disk space
shell: bash
run: |
sudo find /opt -maxdepth 1 -mindepth 1 \
! -path /opt/containerd \
! -path /opt/actionarchivecache \
! -path /opt/runner \
! -path /opt/runner-cache \
-exec rm -rf {} \;
- name: Build Binaries
id: build_binaries
shell: bash -ex {0}
run: |
latest=$(git rev-parse HEAD)
echo "commitish=$latest" >> $GITHUB_OUTPUT
GO_BUILD_VERSION=1.26 scripts/build_all_binaries.sh
- name: Fix ownership of build artifacts
run: sudo chown -R "$USER:$USER" out
- name: Create Release
id: create_release
uses: dolthub/create-release@v2
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
with:
tag_name: v${{ needs.format-version.outputs.version }}
release_name: ${{ needs.format-version.outputs.version }}
draft: false
prerelease: true
commitish: ${{ steps.build_binaries.outputs.commitish }}
- name: Create install script
id: create-install-script
shell: bash
run: scripts/build_install_script.sh
env:
DOLTGRES_VERSION: ${{ needs.format-version.outputs.version }}
- name: Upload Linux AMD64 Distro
id: upload-linux-amd64-distro
uses: dolthub/upload-release-asset@v2
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
with:
upload_url: ${{ steps.create_release.outputs.upload_url }}
asset_path: out/doltgresql-linux-amd64.tar.gz
asset_name: doltgresql-linux-amd64.tar.gz
asset_content_type: application/zip
- name: Upload Linux ARM64 Distro
id: upload-linux-arm64-distro
uses: dolthub/upload-release-asset@v2
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
with:
upload_url: ${{ steps.create_release.outputs.upload_url }}
asset_path: out/doltgresql-linux-arm64.tar.gz
asset_name: doltgresql-linux-arm64.tar.gz
asset_content_type: application/zip
- name: Upload OSX AMD64 Distro
id: upload-osx-amd64-distro
uses: dolthub/upload-release-asset@v2
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
with:
upload_url: ${{ steps.create_release.outputs.upload_url }}
asset_path: out/doltgresql-darwin-amd64.tar.gz
asset_name: doltgresql-darwin-amd64.tar.gz
asset_content_type: application/zip
- name: Upload OSX ARM64 Distro
id: upload-osx-arm64-distro
uses: dolthub/upload-release-asset@v2
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
with:
upload_url: ${{ steps.create_release.outputs.upload_url }}
asset_path: out/doltgresql-darwin-arm64.tar.gz
asset_name: doltgresql-darwin-arm64.tar.gz
asset_content_type: application/zip
- name: Upload Windows Distro
id: upload-windows-distro
uses: dolthub/upload-release-asset@v2
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
with:
upload_url: ${{ steps.create_release.outputs.upload_url }}
asset_path: out/doltgresql-windows-amd64.zip
asset_name: doltgresql-windows-amd64.zip
asset_content_type: application/zip
- name: Upload Windows Distro 7z
id: upload-windows-distro-7z
uses: dolthub/upload-release-asset@v2
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
with:
upload_url: ${{ steps.create_release.outputs.upload_url }}
asset_path: out/doltgresql-windows-amd64.7z
asset_name: doltgresql-windows-amd64.7z
asset_content_type: application/x-7z-compressed
- name: Upload install script
id: upload-install-script
uses: dolthub/upload-release-asset@v2
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
with:
upload_url: ${{ steps.create_release.outputs.upload_url }}
asset_path: out/install.sh
asset_name: install.sh
asset_content_type: text/plain
create-release-notes:
needs: [format-version, create-release]
runs-on: ubuntu-22.04
steps:
- name: Trigger Release Notes
uses: peter-evans/repository-dispatch@v2.0.0
with:
token: ${{ secrets.GITHUB_TOKEN }}
event-type: release-notes
client-payload: '{"version": "${{ needs.format-version.outputs.version }}", "release_id": "${{ needs.create-release.outputs.release_id }}"}'
trigger-performance-benchmark-email:
needs: [format-version, create-release]
runs-on: ubuntu-22.04
steps:
- name: Trigger Performance Benchmarks
uses: peter-evans/repository-dispatch@v2.0.0
with:
token: ${{ secrets.REPO_ACCESS_TOKEN }}
event-type: release-doltgres
client-payload: '{"version": "${{ needs.format-version.outputs.version }}", "actor": "${{ github.actor }}"}'
docker-image-push:
needs: [format-version, create-release]
runs-on: ubuntu-22.04
steps:
- name: Trigger Push Docker Image
uses: peter-evans/repository-dispatch@v3
with:
token: ${{ secrets.REPO_ACCESS_TOKEN }}
event-type: push-docker-image
client-payload: '{"version": "${{ needs.format-version.outputs.version }}", "release_id": "${{ needs.create-release.outputs.release_id }}" }'
+76
View File
@@ -0,0 +1,76 @@
name: Test Bats Unix
on: [pull_request]
concurrency:
group: ci-bats-unix-${{ github.event.pull_request.number || github.ref }}
cancel-in-progress: true
jobs:
test:
name: Bats tests
defaults:
run:
shell: bash
runs-on: ${{ matrix.os }}
strategy:
fail-fast: false
matrix:
os: [ ubuntu-22.04, macos-latest ]
env:
use_credentials: ${{ secrets.AWS_SECRET_ACCESS_KEY != '' && secrets.AWS_ACCESS_KEY_ID != '' }}
steps:
- uses: actions/checkout@v6
- name: Set up Go
uses: actions/setup-go@v5
with:
go-version-file: go.mod
id: go
- uses: actions/setup-node@v4
with:
node-version: ^16
- name: Create CI Bin
run: |
mkdir -p ./.ci_bin
echo "$(pwd)/.ci_bin" >> $GITHUB_PATH
- name: Install ICU4C (MacOS)
if: ${{ matrix.os == 'macos-latest' }}
run: |
dir=$(brew --cellar icu4c)
dir="$dir"/$(ls "$dir")
echo CGO_CPPFLAGS=-I$dir/include >> $GITHUB_ENV
echo CGO_LDFLAGS=-L$dir/lib >> $GITHUB_ENV
- name: Verify Bats Naming Scheme
run: ./check_bats_fmt.sh
working-directory: ./testing/bats/setup
- name: Build SQL Syntax
run: ./build.sh
working-directory: ./postgres/parser
shell: bash
- name: Install Bats
run: |
npm i bats
echo "$(pwd)/node_modules/.bin" >> $GITHUB_PATH
working-directory: ./.ci_bin
- name: Install DoltgreSQL
working-directory: ./
run: |
go build -mod=readonly -o .ci_bin/doltgres ./cmd/doltgres
- name: Install PSQL Ubuntu
if: matrix.os == 'ubuntu-22.04'
run: |
sudo sh -c 'echo "deb https://apt.postgresql.org/pub/repos/apt $(lsb_release -cs)-pgdg main" > /etc/apt/sources.list.d/pgdg.list'
wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | sudo apt-key add -
sudo apt-get update
sudo apt-get install --yes --no-install-recommends postgresql-client-15
- name: Install PSQL MacOS
if: matrix.os == 'macos-latest'
run: |
brew install postgresql@15
brew link --overwrite postgresql@15
- name: Test all Unix
env:
SQL_ENGINE: "local-engine"
BATS_TEST_RETRIES: "3"
run: |
bats --tap .
working-directory: ./testing/bats
+84
View File
@@ -0,0 +1,84 @@
name: Check Formatting, Committers and Generated Code
on:
pull_request:
branches: [ main ]
concurrency:
group: ci-check-repo-${{ github.event.pull_request.number || github.ref }}
cancel-in-progress: true
jobs:
verify:
name: Verify format
runs-on: ubuntu-22.04
outputs:
format: ${{ steps.should_format.outputs.format }}
steps:
- uses: actions/checkout@v6
with:
submodules: true
- name: Set up Go
uses: actions/setup-go@v5
with:
go-version-file: go.mod
- name: Check all
id: should_format
run: |
./scripts/check_bats_fmt.sh
if ./scripts/check_fmt.sh ; then
echo "code is formatted"
else
echo "code is not formatted"
if [ "${{ github.repository }}" != "dolthub/doltgresql" ]; then
echo "Pull requests from forks must be manually formatted."
echo "Please run scripts/format_repo.sh to format this pull request."
exit 1;
fi
echo "format=true" >> $GITHUB_OUTPUT
fi
./postgres/parser/build.sh
GOFLAGS="-mod=readonly" go build ./...
go vet -unsafeptr=false -mod=readonly ./...
env:
BRANCH_NAME: ${{ github.head_ref }}
CHANGE_TARGET: ${{ github.base_ref }}
format:
needs: verify
if: ${{ needs.verify.outputs.format == 'true' }}
name: Format PR
runs-on: ubuntu-22.04
steps:
- uses: actions/checkout@v6
with:
ref: ${{ github.event.pull_request.head.ref || github.ref }}
repository: ${{ github.event.pull_request.head.repo.full_name || github.repository }}
submodules: true
token: ${{ secrets.REPO_ACCESS_TOKEN || secrets.GITHUB_TOKEN }}
- name: Set up Go
uses: actions/setup-go@v5
with:
go-version-file: go.mod
- name: Install goimports
run: go install golang.org/x/tools/cmd/goimports@latest
- name: Format repo
run: ./scripts/format_repo.sh
env:
BRANCH_NAME: ${{ github.head_ref }}
CHANGE_TARGET: ${{ github.base_ref }}
- name: Changes detected
id: detect-changes
run: |
changes=$(git status --porcelain)
if [ ! -z "$changes" ]; then
echo "has-changes=true" >> $GITHUB_OUTPUT
fi
- uses: EndBug/add-and-commit@v9.1.4
if: ${{ steps.detect-changes.outputs.has-changes == 'true' }}
with:
message: "[ga-format-pr] Run scripts/format_repo.sh"
add: "."
cwd: "."
pull: "--ff"
@@ -0,0 +1,66 @@
name: Check Compatibility
on:
pull_request:
branches: [ main ]
paths:
- 'go/**'
- 'core/**'
- 'server/**'
- 'integration-tests/**'
concurrency:
group: ci-compatibility-tests-${{ github.event.pull_request.number || github.ref }}
cancel-in-progress: true
jobs:
test:
name: Compatibility Test
runs-on: ubuntu-22.04
steps:
- uses: actions/checkout@v4
- name: Set up Go
uses: actions/setup-go@v5
with:
go-version-file: go.mod
id: go
- uses: actions/setup-node@v4
with:
node-version: ^16
- name: Build SQL Syntax
run: ./build.sh
working-directory: ./postgres/parser
shell: bash
- name: Create CI Bin
run: |
mkdir -p ./.ci_bin
echo "$(pwd)/.ci_bin" >> $GITHUB_PATH
- name: Install Bats
run: |
npm i bats
echo "$(pwd)/node_modules/.bin" >> $GITHUB_PATH
working-directory: ./.ci_bin
- name: Install DoltgreSQL
run: |
go build -mod=readonly -o .ci_bin/doltgres ./cmd/doltgres
- name: Install PSQL
run: |
sudo sh -c 'echo "deb https://apt.postgresql.org/pub/repos/apt $(lsb_release -cs)-pgdg main" > /etc/apt/sources.list.d/pgdg.list'
wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | sudo apt-key add -
sudo apt-get update
sudo apt-get install --yes --no-install-recommends postgresql-client-15
- name: Make scripts executable
run: chmod +x runner.sh test_files/setup_repo.sh
working-directory: ./integration-tests/compatibility
- name: Run compatibility tests
run: ./runner.sh
working-directory: ./integration-tests/compatibility
@@ -0,0 +1,50 @@
name: Test Go SQL Server Driver
on: [pull_request]
concurrency:
group: ci-go-sql-server-driver-${{ github.event.pull_request.number || github.ref }}
cancel-in-progress: true
jobs:
test:
name: Go SQL Server Driver tests
defaults:
run:
shell: bash
runs-on: ${{ matrix.os }}
strategy:
fail-fast: false
matrix:
os: [ ubuntu-22.04, macos-latest ]
steps:
- uses: actions/checkout@v6
- name: Set up Go
uses: actions/setup-go@v5
with:
go-version-file: go.mod
id: go
- name: Create CI Bin
run: |
mkdir -p ./.ci_bin
echo "$(pwd)/.ci_bin" >> $GITHUB_PATH
- name: Install ICU4C (MacOS)
if: ${{ matrix.os == 'macos-latest' }}
run: |
dir=$(brew --cellar icu4c)
dir="$dir"/$(ls "$dir")
echo CGO_CPPFLAGS=-I$dir/include >> $GITHUB_ENV
echo CGO_LDFLAGS=-L$dir/lib >> $GITHUB_ENV
- name: Build SQL Syntax
run: ./build.sh
working-directory: ./postgres/parser
shell: bash
- name: Install DoltgreSQL
working-directory: ./
run: |
go build -mod=readonly -o .ci_bin/doltgres ./cmd/doltgres
echo "DOLTGRES_BIN_PATH=$(pwd)/.ci_bin/doltgres" >> $GITHUB_ENV
- name: Test Go SQL Server Driver
working-directory: ./integration-tests/go-sql-server-driver
run: |
go test --timeout=20m ./...
@@ -0,0 +1,30 @@
name: Test PostgreSQL Client integrations
on: [pull_request]
concurrency:
group: ci-postgres-client-tests-${{ github.event.pull_request.number || github.ref }}
cancel-in-progress: true
jobs:
postgres_client_integrations_job:
runs-on: ubuntu-22.04
timeout-minutes: 45
name: Run tests
steps:
- name: Free disk space
run: |
NAME="DISK-CLEANUP"
echo "[${NAME}] Starting background cleanup..."
[ -d "$AGENT_TOOLSDIRECTORY" ] && sudo rm -rf "$AGENT_TOOLSDIRECTORY" &
[ -d /usr/share/dotnet ] && sudo rm -rf /usr/share/dotnet &
[ -d /usr/local/lib/android ] && sudo rm -rf /usr/local/lib/android &
[ -d /opt/ghc ] && sudo rm -rf /opt/ghc &
[ -d /usr/local/share/boost ] && sudo rm -rf /usr/local/share/boost &
- name: Checkout
uses: actions/checkout@v6
- name: Set up Docker
uses: docker/setup-docker-action@v4
- name: Build Docker image
run: docker build -t postgres-client-tests --file testing/postgres-client-tests/Dockerfile .
- name: Run tests
run: docker run --detach=false postgres-client-tests
+26
View File
@@ -0,0 +1,26 @@
name: Static Analysis & Linter
on: [pull_request]
concurrency:
group: ci-staticcheck-${{ github.event.pull_request.number || github.ref }}
cancel-in-progress: true
jobs:
ci:
name: Run Staticcheck
runs-on: ubuntu-22.04
steps:
- uses: actions/checkout@v6
with:
submodules: true
- name: Set up Go
uses: actions/setup-go@v5
with:
go-version-file: go.mod
- name: Build SQL Syntax
run: ./build.sh
working-directory: ./postgres/parser
shell: bash
- name: Run check
run: ./run_staticcheck.sh
working-directory: ./scripts
+48
View File
@@ -0,0 +1,48 @@
name: Email Team Members
on:
repository_dispatch:
types: [ email-report ]
permissions:
id-token: write # This is required for requesting the JWT
contents: read # This is required for actions/checkout
jobs:
email-team:
runs-on: ubuntu-22.04
name: Email Team Members
steps:
- uses: actions/checkout@v6
- name: Configure AWS Credentials
uses: aws-actions/configure-aws-credentials@v4
with:
role-session-name: GitHub_to_AWS_via_FederatedOIDC
role-to-assume: ${{ secrets.AWS_ROLE_TO_ASSUME_CORP }}
aws-region: us-west-2
- name: Get Results
id: get-results
run: aws s3api get-object --bucket="$BUCKET" --key="$KEY" results.log
env:
KEY: ${{ github.event.client_payload.key }}
BUCKET: ${{ github.event.client_payload.bucket }}
- name: Get Addresses
id: get-addresses
run: |
addresses="$TEAM"
if [ ! -z "$RECIPIENT" ]; then
addresses="[\"$RECIPIENT\"]"
fi
echo "addresses=$addresses" >> $GITHUB_OUTPUT
env:
RECIPIENT: ${{ github.event.client_payload.email_recipient }}
TEAM: '["${{ secrets.PERF_REPORTS_EMAIL_ADDRESS }}"]'
- name: Send Email
uses: ./.github/actions/ses-email-action
with:
template: ${{ github.event.client_payload.template }}
region: us-west-2
version: ${{ github.event.client_payload.version }}
format: ${{ github.event.client_payload.noms_bin_format }}
toAddresses: ${{ steps.get-addresses.outputs.addresses }}
dataFile: ${{ format('{0}/results.log', github.workspace) }}
@@ -0,0 +1,75 @@
name: Benchmark Latency
on:
repository_dispatch:
types: [ benchmark-latency ]
permissions:
id-token: write # This is required for requesting the JWT
contents: read # This is required for actions/checkout
jobs:
performance:
runs-on: ubuntu-22.04
name: Benchmark Performance
steps:
- name: Checkout
uses: actions/checkout@v6
- uses: azure/setup-kubectl@v4
with:
version: 'v1.23.6'
- name: Install aws-iam-authenticator
run: |
curl -o aws-iam-authenticator https://amazon-eks.s3.us-west-2.amazonaws.com/1.18.8/2020-09-18/bin/linux/amd64/aws-iam-authenticator && \
chmod +x ./aws-iam-authenticator && \
sudo cp ./aws-iam-authenticator /usr/local/bin/aws-iam-authenticator
aws-iam-authenticator version
- name: Configure AWS Credentials
uses: aws-actions/configure-aws-credentials@v4
with:
role-session-name: GitHub_to_AWS_via_FederatedOIDC
role-to-assume: ${{ secrets.AWS_ROLE_TO_ASSUME_CORP }}
aws-region: us-west-2
- name: Create and Auth kubeconfig
run: |
echo "$CONFIG" > kubeconfig
KUBECONFIG=kubeconfig kubectl config set-credentials github-actions-doltgresql --exec-api-version=client.authentication.k8s.io/v1alpha1 --exec-command=aws-iam-authenticator --exec-arg=token --exec-arg=-i --exec-arg=eks-cluster-1
KUBECONFIG=kubeconfig kubectl config set-context github-actions-doltgresql-context --cluster=eks-cluster-1 --user=github-actions-doltgresql --namespace=performance-benchmarking
KUBECONFIG=kubeconfig kubectl config use-context github-actions-doltgresql-context
env:
CONFIG: ${{ secrets.CORP_KUBECONFIG }}
- name: Create Sysbench Performance Benchmarking K8s Job
run: ./.github/scripts/performance-benchmarking/run-benchmarks.sh
env:
FROM_SERVER: ${{ github.event.client_payload.from_server }}
FROM_VERSION: ${{ github.event.client_payload.from_version }}
TO_SERVER: ${{ github.event.client_payload.to_server }}
TO_VERSION: ${{ github.event.client_payload.to_version }}
MODE: ${{ github.event.client_payload.mode }}
ISSUE_NUMBER: ${{ github.event.client_payload.issue_number }}
ACTOR: ${{ github.event.client_payload.actor }}
ACTOR_EMAIL: ${{ github.event.client_payload.actor_email }}
REPO_ACCESS_TOKEN: ${{ secrets.REPO_ACCESS_TOKEN }}
KUBECONFIG: "./kubeconfig"
INIT_BIG_REPO: ${{ github.event.client_payload.init_big_repo }}
NOMS_BIN_FORMAT: "__DOLT__"
TEMPLATE_SCRIPT: ${{ github.event.client_payload.template_script }}
# TODO: skipping tpcc with Doltgres and Postgres, not currently working
# - name: Create TPCC Performance Benchmarking K8s Job
# run: ./.github/scripts/performance-benchmarking/run-benchmarks.sh
# env:
# FROM_SERVER: ${{ github.event.client_payload.from_server }}
# FROM_VERSION: ${{ github.event.client_payload.from_version }}
# TO_SERVER: ${{ github.event.client_payload.to_server }}
# TO_VERSION: ${{ github.event.client_payload.to_version }}
# MODE: ${{ github.event.client_payload.mode }}
# ISSUE_NUMBER: ${{ github.event.client_payload.issue_number }}
# ACTOR: ${{ github.event.client_payload.actor }}
# ACTOR_EMAIL: ${{ github.event.client_payload.actor_email }}
# REPO_ACCESS_TOKEN: ${{ secrets.REPO_ACCESS_TOKEN }}
# KUBECONFIG: "./kubeconfig"
# INIT_BIG_REPO: ${{ github.event.client_payload.init_big_repo }}
# NOMS_BIN_FORMAT: "__DOLT__"
# WITH_TPCC: "true"
# TEMPLATE_SCRIPT: ${{ github.event.client_payload.template_script }}
@@ -0,0 +1,53 @@
name: SQL Correctness
on:
repository_dispatch:
types: [ sql-correctness ]
permissions:
id-token: write # This is required for requesting the JWT
contents: read # This is required for actions/checkout
jobs:
correctness:
runs-on: ubuntu-22.04
name: Dolt SQL Correctness
steps:
- uses: actions/checkout@v6
- uses: azure/setup-kubectl@v4
with:
version: 'v1.23.6'
- name: Configure AWS Credentials
uses: aws-actions/configure-aws-credentials@v4
with:
role-session-name: GitHub_to_AWS_via_FederatedOIDC
role-to-assume: ${{ secrets.AWS_ROLE_TO_ASSUME_CORP }}
aws-region: us-west-2
- name: Install aws-iam-authenticator
run: |
curl -o aws-iam-authenticator https://amazon-eks.s3.us-west-2.amazonaws.com/1.18.8/2020-09-18/bin/linux/amd64/aws-iam-authenticator && \
chmod +x ./aws-iam-authenticator && \
sudo cp ./aws-iam-authenticator /usr/local/bin/aws-iam-authenticator
aws-iam-authenticator version
- name: Create and Auth kubeconfig
run: |
echo "$CONFIG" > kubeconfig
KUBECONFIG=kubeconfig kubectl config set-credentials github-actions-doltgresql --exec-api-version=client.authentication.k8s.io/v1alpha1 --exec-command=aws-iam-authenticator --exec-arg=token --exec-arg=-i --exec-arg=eks-cluster-1
KUBECONFIG=kubeconfig kubectl config set-context github-actions-doltgresql-context --cluster=eks-cluster-1 --user=github-actions-doltgresql --namespace=performance-benchmarking
KUBECONFIG=kubeconfig kubectl config use-context github-actions-doltgresql-context
env:
CONFIG: ${{ secrets.CORP_KUBECONFIG }}
- name: Create SQL Correctness K8s Job
run: ./.github/scripts/sql-correctness/run-correctness.sh
env:
PR_BRANCH_REF: ${{ github.event.client_payload.branch_ref }}
REGRESS_COMP: ${{ github.event.client_payload.regress_comp }}
PR_NUMBER: ${{ github.event.client_payload.issue_number }}
VERSION: ${{ github.event.client_payload.version }}
MODE: ${{ github.event.client_payload.mode }}
ACTOR: ${{ github.event.client_payload.actor }}
ACTOR_EMAIL: ${{ github.event.client_payload.actor_email }}
REPO_ACCESS_TOKEN: ${{ secrets.REPO_ACCESS_TOKEN }}
KUBECONFIG: "./kubeconfig"
NOMS_BIN_FORMAT: "__DOLT__"
TEMPLATE_SCRIPT: ${{ github.event.client_payload.template_script }}
@@ -0,0 +1,19 @@
name: Label Customer Issues
on:
issues:
types: [opened]
pull_request_target:
branches: [main]
types: [opened]
jobs:
label_customer_issues:
runs-on: ubuntu-latest
steps:
- uses: dolthub/label-customer-issues@main
with:
repo-token: ${{ secrets.REPO_ACCESS_TOKEN }}
issue-label: customer issue
pr-label: contribution
exclude: dependabot
+129
View File
@@ -0,0 +1,129 @@
name: Mini Sysbench
on:
pull_request:
types: [opened, synchronize, reopened]
permissions:
contents: read
pull-requests: write
jobs:
mini-sysbench:
runs-on: ubuntu-latest
steps:
- name: Checkout DoltgreSQL
uses: actions/checkout@v6
with:
ref: ${{ github.event.pull_request.head.sha }}
- name: Setup Git User
uses: fregante/setup-git-user@v2
- name: Install Go
uses: actions/setup-go@v5
with:
go-version-file: go.mod
- name: Install Sysbench
run: |
curl -s https://packagecloud.io/install/repositories/akopytov/sysbench/script.deb.sh | sudo bash
sudo apt -y install sysbench
- name: Test PR branch
id: test_doltgresql_pr
continue-on-error: true
run: |
./postgres/parser/build.sh
./scripts/quick_sysbench.sh
mv ./scripts/mini_sysbench/results.log ./scripts/mini_sysbench/results1.log
cat ./scripts/mini_sysbench/results1.log
- name: Test main branch
id: test_doltgresql_main
continue-on-error: true
run: |
git reset --hard
git fetch --all --unshallow
git checkout origin/main
./postgres/parser/build.sh
./scripts/quick_sysbench.sh
mv ./scripts/mini_sysbench/results.log ./scripts/mini_sysbench/results2.log
cat ./scripts/mini_sysbench/results2.log
- name: Check Sysbench Logs
id: check_logs
run: |
cd scripts/mini_sysbench
if [[ -f "results1.log" && -f "results2.log" ]]; then
echo "logs_exist=true" >> $GITHUB_OUTPUT
echo "logs exist"
else
echo "logs_exist=false" >> $GITHUB_OUTPUT
echo "One of the branches could not successfully run the benchmarks."
echo "Please review them for errors, which should be fixed."
exit 1
fi
- name: Build Sysbench Results Comment
id: build_results
if: steps.check_logs.outputs.logs_exist == 'true'
run: |
cd testing/go/benchmark
output=$(go run .)
echo "program_output<<EOF" >> $GITHUB_OUTPUT
echo "$output" >> $GITHUB_OUTPUT
echo "EOF" >> $GITHUB_OUTPUT
echo "$output"
- name: Is PR From Fork
id: from_fork
run: |
if [ "${{ github.event.pull_request.head.repo.full_name }}" != "${{ github.repository }}" ]; then
echo "This is running from a fork, skipping commenting"
echo "fork=true" >> $GITHUB_OUTPUT
else
echo "This is not running from a fork"
echo "fork=false" >> $GITHUB_OUTPUT
fi
- name: Post Comment
if: steps.from_fork.outputs.fork == 'false' && steps.build_results.outputs.program_output
uses: actions/github-script@v6
env:
PROGRAM_OUTPUT: ${{ steps.build_results.outputs.program_output }}
with:
github-token: ${{ secrets.GITHUB_TOKEN }}
script: |
const commentMarker = '<!-- go-run-output-sysbench -->'
const output = process.env.PROGRAM_OUTPUT
const body = `${commentMarker}\n${output}`
// List comments on the PR
const { data: comments } = await github.rest.issues.listComments({
issue_number: context.issue.number,
owner: context.repo.owner,
repo: context.repo.repo,
})
// Check if a comment already exists
const comment = comments.find(comment => comment.body.includes(commentMarker))
if (comment) {
// Update the existing comment
await github.rest.issues.updateComment({
comment_id: comment.id,
owner: context.repo.owner,
repo: context.repo.repo,
body: body
})
} else {
// Create a new comment
await github.rest.issues.createComment({
issue_number: context.issue.number,
owner: context.repo.owner,
repo: context.repo.repo,
body: body
})
}
@@ -0,0 +1,22 @@
name: Nightly Benchmarks
on:
workflow_dispatch:
schedule:
- cron: '0 2 * * *'
jobs:
perf:
runs-on: ubuntu-22.04
name: Trigger Benchmark Latency and SQL Correctness K8s Workflows
steps:
- uses: peter-evans/repository-dispatch@v2.0.0
with:
token: ${{ secrets.REPO_ACCESS_TOKEN }}
event-type: benchmark-latency
client-payload: '{"from_server": "postgres", "from_version": "15.5", "to_server": "doltgres", "to_version": "${{ github.sha }}", "mode": "nightly", "actor": "${{ github.actor }}", "template_script": "./.github/scripts/performance-benchmarking/get-postgres-doltgres-job-json.sh"}'
- uses: peter-evans/repository-dispatch@v2.0.0
with:
token: ${{ secrets.REPO_ACCESS_TOKEN }}
event-type: sql-correctness
client-payload: '{"version": "${{ github.sha }}", "mode": "nightly", "actor": "${{ github.actor }}", "template_script": "./.github/scripts/sql-correctness/get-doltgres-correctness-job-json.sh"}'
+22
View File
@@ -0,0 +1,22 @@
name: Open update README PR
on:
repository_dispatch:
types: [ open-update-pr ]
jobs:
open-pr:
runs-on: ubuntu-22.04
steps:
- name: Checkout code
uses: actions/checkout@v6
with:
ref: main
token: ${{ secrets.REPO_ACCESS_TOKEN || secrets.GITHUB_TOKEN }}
- name: pull-request
uses: repo-sync/pull-request@v2
with:
source_branch: ${{ github.event.client_payload.source_branch }}
destination_branch: ${{ github.event.client_payload.destination_branch }}
github_token: ${{ secrets.REPO_ACCESS_TOKEN || secrets.GITHUB_TOKEN }}
pr_title: ${{ github.event.client_payload.title }}
@@ -0,0 +1,46 @@
name: Benchmark DoltgreSQL vs PostgreSQL
on:
repository_dispatch:
types: [ release-doltgres ]
workflow_dispatch:
inputs:
version:
description: 'SemVer format release tag, i.e. 0.24.5'
required: true
default: ''
email:
description: 'Email address to receive results'
required: true
default: ''
jobs:
set-version-actor:
name: Set Version and Actor
runs-on: ubuntu-22.04
outputs:
version: ${{ steps.set-vars.outputs.version }}
actor: ${{ steps.set-vars.outputs.actor }}
actor_email: ${{ steps.set-vars.outputs.actor_email }}
steps:
- name: Set variables
id: set-vars
run: |
echo "version=$VERSION" >> $GITHUB_OUTPUT
echo "actor=$ACTOR" >> $GITHUB_OUTPUT
echo "actor_email=$ACTOR_EMAIL" >> $GITHUB_OUTPUT
env:
VERSION: ${{ github.event.inputs.version || github.event.client_payload.version }}
ACTOR: ${{ github.event.client_payload.actor || github.actor }}
ACTOR_EMAIL: ${{ github.event.inputs.email }}
benchmark-dolt-mysql:
runs-on: ubuntu-22.04
needs: set-version-actor
name: Trigger Benchmark Latency and Benchmark Import K8s Workflows
steps:
- uses: peter-evans/repository-dispatch@v2.0.0
with:
token: ${{ secrets.REPO_ACCESS_TOKEN }}
event-type: benchmark-latency
client-payload: '{"from_server": "postgres", "from_version": "15.5", "to_server": "doltgres", "to_version": "${{ needs.set-version-actor.outputs.version }}", "mode": "release", "actor": "${{ needs.set-version-actor.outputs.actor }}", "actor_email": "${{ needs.set-version-actor.outputs.actor_email }}", "template_script": "./.github/scripts/performance-benchmarking/get-postgres-doltgres-job-json.sh"}'
@@ -0,0 +1,64 @@
name: Benchmark Pull Requests
on:
pull_request:
types: [ opened ]
issue_comment:
types: [ created ]
jobs:
validate-commentor:
runs-on: ubuntu-22.04
outputs:
valid: ${{ steps.set_valid.outputs.valid }}
steps:
- uses: actions/checkout@v6
- name: Validate Commentor
id: set_valid
run: ./.github/scripts/performance-benchmarking/validate-commentor.sh "$ACTOR"
env:
ACTOR: ${{ github.actor }}
check-comments:
runs-on: ubuntu-22.04
needs: validate-commentor
if: ${{ needs.validate-commentor.outputs.valid == 'true' }}
outputs:
benchmark: ${{ steps.set_benchmark.outputs.benchmark }}
comment-body: ${{ steps.set_body.outputs.body }}
steps:
- name: Check for Deploy Trigger
uses: dolthub/pull-request-comment-trigger@master
id: check
with:
trigger: '#benchmark'
reaction: rocket
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
- name: Set Benchmark
if: ${{ steps.check.outputs.triggered == 'true' }}
id: set_benchmark
run: |
echo "benchmark=true" >> $GITHUB_OUTPUT
performance:
runs-on: ubuntu-22.04
needs: [validate-commentor, check-comments]
if: ${{ needs.check-comments.outputs.benchmark == 'true' }}
name: Trigger Benchmark Latency K8s Workflow
steps:
- uses: dolthub/pull-request-comment-branch@v4
id: comment-branch
with:
repo_token: ${{ secrets.GITHUB_TOKEN }}
- name: Get pull number
uses: actions/github-script@v7
id: get_pull_number
with:
github-token: ${{ secrets.GITHUB_TOKEN }}
script: core.setOutput("pull_number", JSON.stringify(context.issue.number));
- uses: peter-evans/repository-dispatch@v2.0.0
with:
token: ${{ secrets.REPO_ACCESS_TOKEN }}
event-type: benchmark-latency
client-payload: '{"from_server": "doltgres", "from_version": "${{ github.sha }}", "to_server": "doltgres", "to_version": "${{ steps.comment-branch.outputs.head_sha }}", "mode": "pullRequest", "issue_number": "${{ steps.get_pull_number.outputs.pull_number }}", "init_big_repo": "true", "actor": "${{ github.actor }}", "template_script": "./.github/scripts/performance-benchmarking/get-doltgres-doltgres-job-json.sh"}'
+54
View File
@@ -0,0 +1,54 @@
name: Post to Pull Request
on:
repository_dispatch:
types: [ pull-report ]
permissions:
id-token: write # This is required for requesting the JWT
contents: read # This is required for actions/checkout
jobs:
report-pull-request:
name: Report Performance Benchmarks on Pull Request
runs-on: ubuntu-22.04
if: ${{ github.event.client_payload.issue_number != -1 }}
steps:
- name: Checkout
uses: actions/checkout@v6
- name: Configure AWS Credentials
uses: aws-actions/configure-aws-credentials@v4
with:
role-session-name: GitHub_to_AWS_via_FederatedOIDC
role-to-assume: ${{ secrets.AWS_ROLE_TO_ASSUME_CORP }}
aws-region: us-west-2
- name: Get benchmark results
id: get-results
run: aws s3api get-object --bucket="$BUCKET" --key="$KEY" results.log
env:
KEY: ${{ github.event.client_payload.key }}
BUCKET: ${{ github.event.client_payload.bucket }}
- name: Post results to PR
uses: actions/github-script@v7
with:
github-token: ${{ secrets.REPO_ACCESS_TOKEN || secrets.GITHUB_TOKEN }}
script: |
const { ACTOR, FORMAT, ISSUE_NUMBER, GITHUB_WORKSPACE } = process.env;
const issue_number = parseInt(ISSUE_NUMBER, 10);
const { owner, repo } = context.repo;
fs = require('fs');
fs.readFile(`${GITHUB_WORKSPACE}/results.log`, 'utf8', function (err,data) {
if (err) {
return console.log(err);
}
return github.rest.issues.createComment({
issue_number,
owner,
repo,
body: `@${ACTOR} ${FORMAT}\n ${data}`
});
});
env:
ACTOR: ${{ github.event.client_payload.actor }}
ISSUE_NUMBER: ${{ github.event.client_payload.issue_number }}
FORMAT: ${{ github.event.client_payload.noms_bin_format }}
+283
View File
@@ -0,0 +1,283 @@
name: Regression Tests
on:
pull_request:
types: [opened, synchronize, reopened]
env:
REGRESSION_TESTING: true
permissions:
contents: read
pull-requests: write
concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number }}
cancel-in-progress: true
jobs:
resolve-main:
name: Resolve main
runs-on: ubuntu-latest
outputs:
main_sha: ${{ steps.resolve.outputs.main_sha }}
steps:
- name: Checkout DoltgreSQL
uses: actions/checkout@v6
with:
ref: main
- name: Resolve main SHA
id: resolve
run: echo "main_sha=$(git rev-parse HEAD)" >> "$GITHUB_OUTPUT"
pr-regression:
name: PR regression
runs-on: ubuntu-latest
needs: resolve-main
steps:
- name: Checkout DoltgreSQL
uses: actions/checkout@v6
with:
ref: ${{ github.event.pull_request.head.sha }}
fetch-depth: 0
- name: Setup Git User
uses: fregante/setup-git-user@v2
- name: Merge base into PR
run: |
git fetch origin ${{ needs.resolve-main.outputs.main_sha }}
git merge ${{ needs.resolve-main.outputs.main_sha }} --no-commit --no-ff
- name: Install Go
uses: actions/setup-go@v5
with:
go-version-file: go.mod
cache: true
- name: Test PR branch
continue-on-error: true
run: |
./postgres/parser/build.sh
cd testing/go/regression
mkdir -p out
{
echo "pr_head=${{ github.event.pull_request.head.sha }}"
echo "main_head=${{ needs.resolve-main.outputs.main_sha }}"
echo "merge_base=$(git merge-base HEAD ${{ needs.resolve-main.outputs.main_sha }})"
git status --short --branch
git diff --cached --stat
go version
go env
go list -m github.com/dolthub/dolt/go github.com/dolthub/go-mysql-server
} > out/pr-metadata.txt
cd tool
set +e
go test --timeout=90m ./... --count=1 2>&1 | tee ../out/pr-go-test.log
test_status=${PIPESTATUS[0]}
set -e
if [ -f ../out/results.trackers ]; then
cp ../out/results.trackers ../out/results2.trackers
fi
exit "$test_status"
- name: Upload PR regression artifacts
if: always()
uses: actions/upload-artifact@v4
with:
name: regression-pr-${{ github.run_id }}-${{ github.run_attempt }}
path: |
testing/go/regression/out/results2.trackers
testing/go/regression/out/pr-go-test.log
testing/go/regression/out/pr-metadata.txt
if-no-files-found: warn
retention-days: 14
compression-level: 0
main-regression:
name: Main regression
runs-on: ubuntu-latest
needs: resolve-main
steps:
- name: Checkout DoltgreSQL
uses: actions/checkout@v6
with:
ref: ${{ needs.resolve-main.outputs.main_sha }}
- name: Install Go
uses: actions/setup-go@v5
with:
go-version-file: go.mod
cache: true
- name: Test main branch
continue-on-error: true
run: |
./postgres/parser/build.sh
cd testing/go/regression
mkdir -p out
{
echo "main_head=${{ needs.resolve-main.outputs.main_sha }}"
git status --short --branch
go version
go env
go list -m github.com/dolthub/dolt/go github.com/dolthub/go-mysql-server
} > out/main-metadata.txt
cd tool
set +e
go test --timeout=90m ./... --count=1 2>&1 | tee ../out/main-go-test.log
test_status=${PIPESTATUS[0]}
set -e
if [ -f ../out/results.trackers ]; then
cp ../out/results.trackers ../out/results1.trackers
fi
exit "$test_status"
- name: Upload main regression artifacts
if: always()
uses: actions/upload-artifact@v4
with:
name: regression-main-${{ github.run_id }}-${{ github.run_attempt }}
path: |
testing/go/regression/out/results1.trackers
testing/go/regression/out/main-go-test.log
testing/go/regression/out/main-metadata.txt
if-no-files-found: warn
retention-days: 14
compression-level: 0
regression-tests:
runs-on: ubuntu-latest
needs:
- resolve-main
- pr-regression
- main-regression
if: always() && needs.pr-regression.result != 'cancelled' && needs.main-regression.result != 'cancelled'
steps:
- name: Checkout DoltgreSQL
uses: actions/checkout@v6
with:
ref: ${{ needs.resolve-main.outputs.main_sha }}
- name: Install Go
uses: actions/setup-go@v5
with:
go-version-file: go.mod
cache: true
- name: Download PR regression artifacts
uses: actions/download-artifact@v4
continue-on-error: true
with:
name: regression-pr-${{ github.run_id }}-${{ github.run_attempt }}
path: testing/go/regression/out
- name: Download main regression artifacts
uses: actions/download-artifact@v4
continue-on-error: true
with:
name: regression-main-${{ github.run_id }}-${{ github.run_attempt }}
path: testing/go/regression/out
- name: Check result trackers
id: check_trackers
run: |
cd testing/go/regression/out
if [[ -f "results1.trackers" && -f "results2.trackers" ]]; then
echo "trackers_exist=true" >> $GITHUB_OUTPUT
echo "trackers exist"
else
echo "trackers_exist=false" >> $GITHUB_OUTPUT
echo "One of the branches could not successfully complete their tests."
echo "Please review uploaded regression artifacts for errors, which must be fixed."
exit 1
fi
- name: Build Regression Test Results Comment
id: build_results
if: steps.check_trackers.outputs.trackers_exist == 'true'
run: |
./postgres/parser/build.sh
cd testing/go/regression/tool
# Pass via file, not $GITHUB_OUTPUT: large output through env trips ARG_MAX in github-script.
go run . results1.trackers results2.trackers > "$RUNNER_TEMP/regression-comment.md"
cp "$RUNNER_TEMP/regression-comment.md" ../out/regression-comment.md
cat "$RUNNER_TEMP/regression-comment.md"
if [ -s "$RUNNER_TEMP/regression-comment.md" ]; then
echo "has_output=true" >> "$GITHUB_OUTPUT"
else
echo "has_output=false" >> "$GITHUB_OUTPUT"
fi
- name: Upload comparison artifacts
if: always()
uses: actions/upload-artifact@v4
with:
name: regression-comparison-${{ github.run_id }}-${{ github.run_attempt }}
path: testing/go/regression/out/regression-comment.md
if-no-files-found: warn
retention-days: 14
compression-level: 0
- name: Is PR From Fork
id: from_fork
run: |
if [ "${{ github.event.pull_request.head.repo.full_name }}" != "${{ github.repository }}" ]; then
echo "This is running from a fork, skipping commenting"
echo "fork=true" >> $GITHUB_OUTPUT
else
echo "This is not running from a fork"
echo "fork=false" >> $GITHUB_OUTPUT
fi
- name: Post Comment
if: steps.from_fork.outputs.fork == 'false' && steps.build_results.outputs.has_output == 'true'
uses: actions/github-script@v7
env:
COMMENT_PATH: ${{ runner.temp }}/regression-comment.md
with:
github-token: ${{ secrets.GITHUB_TOKEN }}
script: |
const fs = require('fs')
const commentMarker = '<!-- go-run-output -->'
// GitHub caps comment bodies at 65536 chars.
const MAX_BODY = 65000
let output = fs.readFileSync(process.env.COMMENT_PATH, 'utf8')
let body = `${commentMarker}\n${output}`
if (body.length > MAX_BODY) {
const notice = '\n\n_Output truncated; see workflow logs for the full report._'
body = body.slice(0, MAX_BODY - notice.length) + notice
}
// List comments on the PR
const { data: comments } = await github.rest.issues.listComments({
issue_number: context.issue.number,
owner: context.repo.owner,
repo: context.repo.repo,
})
// Check if a comment already exists
const comment = comments.find(comment => comment.body.includes(commentMarker))
if (comment) {
// Update the existing comment
await github.rest.issues.updateComment({
comment_id: comment.id,
owner: context.repo.owner,
repo: context.repo.repo,
body: body
})
} else {
// Create a new comment
await github.rest.issues.createComment({
issue_number: context.issue.number,
owner: context.repo.owner,
repo: context.repo.repo,
body: body
})
}
+19
View File
@@ -0,0 +1,19 @@
name: Replication Tests
on: [pull_request]
concurrency:
group: ci-replication-tests-${{ github.event.pull_request.number || github.ref }}
cancel-in-progress: true
jobs:
testing-job:
runs-on: ubuntu-22.04
timeout-minutes: 10
name: Run tests
steps:
- name: Check out code
uses: actions/checkout@v6
- name: Build docker image
run: docker build -t local --file testing/ReplicationTestDockerfile .
- name: Run tests
run: docker run --detach=false local
+43
View File
@@ -0,0 +1,43 @@
name: Benchmark SQL Correctness
on:
repository_dispatch:
types: [ release-doltgres ]
workflow_dispatch:
inputs:
version:
description: 'SemVer format release tag, i.e. 0.24.5'
required: true
default: ''
email:
description: 'Email address to receive results'
required: true
default: ''
jobs:
set-version-actor:
name: Set Version and Actor
runs-on: ubuntu-22.04
outputs:
version: ${{ steps.set-vars.outputs.version }}
actor: ${{ steps.set-vars.outputs.actor }}
steps:
- name: Set variables
id: set-vars
run: |
echo "version=$VERSION" >> $GITHUB_OUTPUT
echo "actor=$ACTOR" >> $GITHUB_OUTPUT
env:
VERSION: ${{ github.event.inputs.version || github.event.client_payload.version }}
ACTOR: ${{ github.event.client_payload.actor || github.actor }}
correctness:
runs-on: ubuntu-22.04
needs: set-version-actor
name: Trigger SQL Correctness K8s Workflow
steps:
- uses: peter-evans/repository-dispatch@v2.0.0
with:
token: ${{ secrets.REPO_ACCESS_TOKEN }}
event-type: sql-correctness
client-payload: '{"version": "${{ needs.set-version-actor.outputs.version }}", "mode": "release", "actor": "${{ needs.set-version-actor.outputs.actor }}", "actor_email": "${{ needs.set-version-actor.outputs.actor_email }}", "template_script": "./.github/scripts/sql-correctness/get-doltgres-correctness-job-json.sh"}'
+47
View File
@@ -0,0 +1,47 @@
name: Enginetests
on: [pull_request]
concurrency:
group: test-enginetests-${{ github.event.pull_request.number || github.ref }}
cancel-in-progress: true
jobs:
test:
strategy:
fail-fast: false
matrix:
platform: [ubuntu-latest, macos-latest, windows-latest]
runs-on: ${{ matrix.platform }}
steps:
- name: Checkout code
uses: actions/checkout@v6
- name: Set up Go
uses: actions/setup-go@v5
with:
go-version-file: go.mod
id: go
- name: Install ICU4C (MacOS)
if: ${{ matrix.platform == 'macos-latest' }}
run: |
dir=$(brew --cellar icu4c)
dir="$dir"/$(ls "$dir")
echo CGO_CPPFLAGS=-I$dir/include >> $GITHUB_ENV
echo CGO_LDFLAGS=-L$dir/lib >> $GITHUB_ENV
- name: Install ICU4C (Windows)
if: ${{ matrix.platform == 'windows-latest' }}
uses: msys2/setup-msys2@v2
with:
path-type: inherit
msystem: UCRT64
pacboy: icu:p toolchain:p pkg-config:p
- name: Build SQL Syntax
run: ./build.sh
working-directory: ./postgres/parser
shell: bash
- name: Test (*nix)
if: ${{ matrix.platform != 'windows-latest' }}
run: go test --timeout=20m $(go list ./testing/go/enginetest)
- name: Test (Windows)
if: ${{ matrix.platform == 'windows-latest' }}
shell: msys2 {0}
run: go.exe test --timeout=20m $(go.exe list ./testing/go/enginetest)
+59
View File
@@ -0,0 +1,59 @@
name: Test Import Regressions
on: [pull_request]
concurrency:
group: test-import-dumps-${{ github.event.pull_request.number || github.ref }}
cancel-in-progress: true
jobs:
test-import-dumps:
strategy:
fail-fast: false
matrix:
platform: [ubuntu-latest, macos-latest, windows-latest]
runs-on: ${{ matrix.platform }}
env:
GITHUB_ACTION_IMPORT_DUMPS: true
steps:
- name: Checkout code
uses: actions/checkout@v6
- name: Set up Go
uses: actions/setup-go@v5
with:
go-version-file: go.mod
id: go
- name: Install ICU4C (MacOS)
if: ${{ matrix.platform == 'macos-latest' }}
run: |
dir=$(brew --cellar icu4c)
dir="$dir"/$(ls "$dir")
echo CGO_CPPFLAGS=-I$dir/include >> $GITHUB_ENV
echo CGO_LDFLAGS=-L$dir/lib >> $GITHUB_ENV
- name: Install ICU4C (Windows)
if: ${{ matrix.platform == 'windows-latest' }}
uses: msys2/setup-msys2@v2
with:
path-type: inherit
msystem: UCRT64
pacboy: icu:p toolchain:p pkg-config:p
- name: Build SQL Syntax
run: ./build.sh
working-directory: ./postgres/parser
shell: bash
- name: Setup PostgreSQL
uses: tj-actions/install-postgresql@v3
with:
postgresql-version: 15
- name: Test (MacOS)
if: ${{ matrix.platform == 'macos-latest' }}
working-directory: ./testing/go
run: go test --timeout=30m -run TestImportingDumps .
- name: Test (Windows)
if: ${{ matrix.platform == 'windows-latest' }}
working-directory: ./testing/go
shell: msys2 {0}
run: go.exe test --timeout=30m -run TestImportingDumps .
- name: Test (Linux)
if: ${{ matrix.platform == 'ubuntu-latest' }}
working-directory: ./testing/go
run: go test --timeout=30m -run TestImportingDumps .
+67
View File
@@ -0,0 +1,67 @@
name: Test
on: [pull_request]
concurrency:
group: test-${{ github.event.pull_request.number || github.ref }}
cancel-in-progress: true
jobs:
test:
strategy:
fail-fast: false
matrix:
platform: [ubuntu-latest, macos-latest, windows-latest]
runs-on: ${{ matrix.platform }}
steps:
- name: Set timezone for Unix
uses: szenius/set-timezone@v2.0
with:
timezoneLinux: "America/Los_Angeles"
timezoneMacos: "America/Los_Angeles"
- name: Set timezone for Windows
if: ${{ matrix.platform == 'windows-latest' }}
shell: pwsh
run: |
tzutil /s "Pacific Standard Time"
tzutil /g
Get-TimeZone
- name: Checkout code
uses: actions/checkout@v6
- name: Set up Go
uses: actions/setup-go@v5
with:
go-version-file: go.mod
id: go
- name: Install ICU4C (MacOS)
if: ${{ matrix.platform == 'macos-latest' }}
run: |
dir=$(brew --cellar icu4c)
dir="$dir"/$(ls "$dir")
echo CGO_CPPFLAGS=-I$dir/include >> $GITHUB_ENV
echo CGO_LDFLAGS=-L$dir/lib >> $GITHUB_ENV
- name: Install ICU4C (Windows)
if: ${{ matrix.platform == 'windows-latest' }}
uses: msys2/setup-msys2@v2
with:
path-type: inherit
msystem: UCRT64
pacboy: icu:p toolchain:p pkg-config:p
- name: Build SQL Syntax
run: ./build.sh
working-directory: ./postgres/parser
shell: bash
- name: Setup PostgreSQL
uses: tj-actions/install-postgresql@v3
with:
postgresql-version: 15
- name: Test (MacOS)
if: ${{ matrix.platform == 'macos-latest' }}
run: go test --timeout=20m -skip="TestReplication" $(go list ./... | grep -v enginetest)
- name: Test (Windows)
if: ${{ matrix.platform == 'windows-latest' }}
shell: msys2 {0}
run: go.exe test --timeout=20m -skip="TestReplication" $(go.exe list ./... | grep -v enginetest)
- name: Test (Linux)
if: ${{ matrix.platform == 'ubuntu-latest' }}
# Enginetest harness breaks with race testing, not sure why yet
run: go test --timeout=20m -race -skip="TestReplication" $(go list ./... | grep -v enginetest)