wehub/dolthub--doltgresql
1
0
Fork 0
Code Issues Pull Requests Actions 1 Deployments Models Agent Notes Packages Projects Releases Wiki Activity

chore: import upstream snapshot with attribution

Browse Source
This commit is contained in:
wehub-resource-sync
2026-07-13 12:32:25 +08:00
commit e014feafe1
2285 changed files with 1131979 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
integration-tests/compatibility/runner.sh
Executable
+306
View File
@@ -0,0 +1,306 @@
#!/bin/bash
# Doltgresql compatibility test runner.
#
# Downloads specified doltgresql release binaries, creates test repositories
# using them, and runs BATS test suites to verify backward, forward, and
# bidirectional compatibility.
#
# Usage: ./runner.sh
# Run from the integration-tests/compatibility directory.
#
# Environment variables (optional):
# DOLTGRES_SKIP_BACKWARD — skip backward-compatibility tests if set
# DOLTGRES_SKIP_FORWARD — skip forward-compatibility tests if set
# DOLTGRES_SKIP_BIDIR — skip bidirectional-compatibility tests if set
set -eo pipefail
PLATFORM_TUPLE=""
# ---------------------------------------------------------------------------
# Platform detection
# ---------------------------------------------------------------------------
get_platform_tuple() {
local OS ARCH
OS=$(uname)
ARCH=$(uname -m)
if [ "$OS" != Linux ] && [ "$OS" != Darwin ]; then
echo "tests only support linux or macOS." >&2
exit 1
fi
if [ "$OS" = Linux ]; then
PLATFORM_TUPLE=linux
else
PLATFORM_TUPLE=darwin
fi
if [ "$ARCH" = x86_64 ]; then
PLATFORM_TUPLE="${PLATFORM_TUPLE}-amd64"
elif [ "$ARCH" = arm64 ] || [ "$ARCH" = aarch64 ]; then
PLATFORM_TUPLE="${PLATFORM_TUPLE}-arm64"
else
echo "unsupported architecture: $ARCH" >&2
exit 1
fi
echo "$PLATFORM_TUPLE"
}
# ---------------------------------------------------------------------------
# Release download
# ---------------------------------------------------------------------------
# download_release <version>
# Downloads doltgresql-<platform>.tar.gz for the given version tag, extracts it
# into binaries/<version>/, and prints the path to the bin directory.
download_release() {
local ver="$1"
local dirname="binaries/$ver"
mkdir -p "$dirname"
local basename="doltgresql-${PLATFORM_TUPLE}"
local filename="${basename}.tar.gz"
local filepath="${dirname}/${filename}"
local url="https://github.com/dolthub/doltgresql/releases/download/${ver}/${filename}"
echo "Downloading doltgresql ${ver} for ${PLATFORM_TUPLE} ..." >&2
curl -L -o "$filepath" "$url"
tar -zxf "$filepath" -C "$dirname"
# Binary lives at doltgresql-<os>-<arch>/bin/doltgres
echo "${dirname}/${basename}/bin"
}
# ---------------------------------------------------------------------------
# Server management helpers used by the runner (not BATS)
# ---------------------------------------------------------------------------
_pick_port() {
for i in {0..99}; do
local port=$((RANDOM % 4096 + 2048))
if ! nc -z localhost "$port" 2>/dev/null; then
echo "$port"
return 0
fi
done
echo "ERROR: could not find a free port" >&2
return 1
}
_write_config() {
local dir="$1" port="$2"
cat > "$dir/runner-config.yaml" <<EOF
log_level: warning
behavior:
read_only: false
disable_client_multi_statements: false
listener:
host: localhost
port: $port
EOF
}
# start_server <binary> <datadir> <logfile> → sets RUNNER_SERVER_PID and RUNNER_SERVER_PORT
start_server() {
local binary="$1" datadir="$2" logfile="$3"
RUNNER_SERVER_PORT=$(_pick_port)
_write_config "$datadir" "$RUNNER_SERVER_PORT"
PGPASSWORD=password "$binary" -data-dir="$datadir" \
--config="$datadir/runner-config.yaml" > "$logfile" 2>&1 &
RUNNER_SERVER_PID=$!
local end=$((SECONDS + 20))
while [ $SECONDS -lt $end ]; do
if PGPASSWORD=password psql -U postgres -h localhost -p "$RUNNER_SERVER_PORT" \
-c "SELECT 1;" postgres >/dev/null 2>&1; then
return 0
fi
sleep 0.5
done
echo "ERROR: server failed to start on port $RUNNER_SERVER_PORT" >&2
cat "$logfile" >&2
kill "$RUNNER_SERVER_PID" 2>/dev/null
return 1
}
stop_server() {
if [ -n "$RUNNER_SERVER_PID" ]; then
kill "$RUNNER_SERVER_PID" 2>/dev/null || true
wait "$RUNNER_SERVER_PID" 2>/dev/null || true
RUNNER_SERVER_PID=""
RUNNER_SERVER_PORT=""
fi
}
RUNNER_SERVER_PID=""
RUNNER_SERVER_PORT=""
# ---------------------------------------------------------------------------
# Repository setup
# ---------------------------------------------------------------------------
# setup_repo <label> [<binary>]
# Creates a test repository under repos/<label>/ using the given binary
# (defaults to doltgres from PATH). Sets REPO_DIR to the resulting directory.
setup_repo() {
local label="$1"
local binary="${2:-doltgres}"
REPO_DIR="$(pwd)/repos/${label}"
mkdir -p "$REPO_DIR"
./test_files/setup_repo.sh "$REPO_DIR" "$binary"
}
# ---------------------------------------------------------------------------
# Version list helpers
# ---------------------------------------------------------------------------
list_backward_compatible_versions() {
grep -v '^ *#' < test_files/backward_compatible_versions.txt
}
list_forward_compatible_versions() {
grep -v '^ *#' < test_files/forward_compatible_versions.txt
}
# ---------------------------------------------------------------------------
# Test runners
# ---------------------------------------------------------------------------
test_backward_compatibility() {
local ver="$1"
local bin
bin=$(download_release "$ver")
echo "=== Backward compat: creating repo with doltgresql ${ver} ==="
setup_repo "$ver" "${bin}/doltgres"
echo "=== Backward compat: testing HEAD doltgresql against repo from ${ver} ==="
DOLTGRES_TEST_BIN="$(which doltgres)" \
REPO_DIR="$(pwd)/repos/${ver}" \
bats --print-output-on-failure ./test_files/bats/
# Backward-only workflows (test_files/bats/backwards/): tests that only make
# sense with the old binary writing first and HEAD continuing. Each test
# manages its own server lifecycle and needs both binaries.
if [ -d ./test_files/bats/backwards ]; then
local scratch="$(pwd)/repos/${ver}-backward-workflow"
mkdir -p "$scratch"
echo "=== Backward workflow: old=${ver}, new=HEAD ==="
DOLTGRES_LEGACY_BIN="${bin}/doltgres" \
DOLTGRES_NEW_BIN="$(which doltgres)" \
REPO_DIR="$scratch" \
bats --print-output-on-failure ./test_files/bats/backwards/
fi
}
test_forward_compatibility() {
if [ -z $1 ]; then
return
fi
local ver="$1"
local bin
bin=$(download_release "$ver")
echo "=== Forward compat: testing doltgresql ${ver} against repo from HEAD ==="
# repos/HEAD was already created by the main flow (see _main).
DOLTGRES_TEST_BIN="${bin}/doltgres" \
REPO_DIR="$(pwd)/repos/HEAD" \
bats --print-output-on-failure ./test_files/bats/
}
test_bidirectional_compatibility() {
if [ -z $1 ]; then
return
fi
local ver="$1"
local bin
bin=$(download_release "$ver")
local head_bin
head_bin="$(which doltgres)"
# Forward direction: old = released version, new = HEAD
local scratch_fwd="$(pwd)/repos/${ver}-bidir-forward"
mkdir -p "$scratch_fwd"
echo "=== Bidirectional (forward): old=${ver}, new=HEAD ==="
DOLTGRES_LEGACY_BIN="${bin}/doltgres" \
DOLTGRES_NEW_BIN="$head_bin" \
REPO_DIR="$scratch_fwd" \
bats --print-output-on-failure ./test_files/bats/bidirectional/bidirectional_compat.bats
# Reverse direction: old = HEAD, new = released version
local scratch_rev="$(pwd)/repos/${ver}-bidir-reverse"
mkdir -p "$scratch_rev"
echo "=== Bidirectional (reverse): old=HEAD, new=${ver} ==="
DOLTGRES_LEGACY_BIN="$head_bin" \
DOLTGRES_NEW_BIN="${bin}/doltgres" \
REPO_DIR="$scratch_rev" \
bats --print-output-on-failure ./test_files/bats/bidirectional/bidirectional_compat.bats
}
# ---------------------------------------------------------------------------
# Cleanup
# ---------------------------------------------------------------------------
cleanup() {
stop_server
rm -rf repos binaries
}
# ---------------------------------------------------------------------------
# Main
# ---------------------------------------------------------------------------
_main() {
PLATFORM_TUPLE=$(get_platform_tuple)
mkdir -p repos binaries
trap cleanup EXIT
# --- Backward compatibility ---
if [ -z "$DOLTGRES_SKIP_BACKWARD" ] && [ -s "test_files/backward_compatible_versions.txt" ]; then
echo "=== Running backward compatibility tests ==="
while IFS= read -r ver; do
test_backward_compatibility "$ver"
done < <(list_backward_compatible_versions)
fi
# --- Create HEAD repo (used for forward compat and the sanity check) ---
echo "=== Creating HEAD repo ==="
setup_repo HEAD
# --- Forward compatibility ---
if [ -z "$DOLTGRES_SKIP_FORWARD" ] && [ -s "test_files/forward_compatible_versions.txt" ]; then
echo "=== Running forward compatibility tests ==="
while IFS= read -r ver; do
test_forward_compatibility "$ver"
done < <(list_forward_compatible_versions)
fi
# --- Bidirectional compatibility (uses forward_compatible_versions list) ---
if [ -z "$DOLTGRES_SKIP_BIDIR" ] && [ -s "test_files/forward_compatible_versions.txt" ]; then
echo "=== Running bidirectional compatibility tests ==="
while IFS= read -r ver; do
test_bidirectional_compatibility "$ver"
done < <(list_forward_compatible_versions)
fi
# --- Sanity check: HEAD against HEAD ---
echo "=== Sanity check: HEAD doltgresql against HEAD repo ==="
DOLTGRES_TEST_BIN="$(which doltgres)" \
REPO_DIR="$(pwd)/repos/HEAD" \
bats --print-output-on-failure ./test_files/bats/compatibility.bats
DOLTGRES_TEST_BIN="$(which doltgres)" \
REPO_DIR="$(pwd)/repos/HEAD" \
bats --print-output-on-failure ./test_files/bats/types_compatibility.bats
}
_main
integration-tests/compatibility/test_files/backward_compatible_versions.txt
+7
View File
@@ -0,0 +1,7 @@
# Doltgresql versions to test backward compatibility against.
# Current HEAD must be able to read and write repos created by these older versions.
# Format: one version tag per line, e.g. v0.8.0
# Keep this list reasonably short; each entry downloads a binary and runs the full
# test suite, so CI time grows linearly.
v0.56.0
v0.56.2
integration-tests/compatibility/test_files/bats/backwards/backward_workflow_compat.bats
+420
View File
@@ -0,0 +1,420 @@
#!/usr/bin/env bats
# Backward-only compatibility tests for doltgresql.
#
# These tests exercise workflows where an older doltgresql release writes the
# initial state and the current HEAD build then continues operating on the same
# repo (schema changes, DML, branching, merging). Unlike bidirectional tests,
# they are only meaningful in one direction — old writes, then new operates —
# so we do not run them with the LEGACY / NEW roles swapped.
#
# Each test manages its own server lifecycle so that it can switch between the
# LEGACY and NEW binaries mid-test. setup() and teardown() only manage the
# scratch directory.
#
# Environment variables:
# DOLTGRES_LEGACY_BIN — path to the "old" doltgres binary
# DOLTGRES_NEW_BIN — path to the "new" (HEAD) doltgres binary
# REPO_DIR — scratch directory base (empty, just needs to exist)
load $BATS_TEST_DIRNAME/../helper/common.bash
BATS_REPO=""
setup() {
BATS_REPO="$BATS_TMPDIR/backward-$$-$RANDOM"
mkdir -p "$BATS_REPO"
}
teardown() {
stop_doltgres
rm -rf "$BATS_REPO"
}
# Convenience wrappers so test bodies read cleanly.
old_server_start() { start_doltgres "$DOLTGRES_LEGACY_BIN" "$BATS_REPO" "$BATS_REPO/old.log"; }
new_server_start() { start_doltgres "$DOLTGRES_NEW_BIN" "$BATS_REPO" "$BATS_REPO/new.log"; }
@test "backward_workflow: fk on varchar column from old repo, then merge" {
[ -n "$DOLTGRES_LEGACY_BIN" ] || skip "requires DOLTGRES_LEGACY_BIN"
[ -n "$DOLTGRES_NEW_BIN" ] || skip "requires DOLTGRES_NEW_BIN"
# --- Old: create parent with a VARCHAR(10) column and seed rows ---
old_server_start
sql <<SQL
CREATE TABLE parent (
id INT NOT NULL PRIMARY KEY,
val VARCHAR(10) NOT NULL UNIQUE
);
INSERT INTO parent VALUES (1, 'apple'), (2, 'banana'), (3, 'cherry');
SQL
sql -c "SELECT dolt_add('.'); SELECT dolt_commit('-m', 'old: create parent');"
stop_doltgres
# --- New: create child with FK on VARCHAR(10) referencing parent(val) ---
new_server_start
sql <<SQL
CREATE TABLE child (
id INT NOT NULL PRIMARY KEY,
ref VARCHAR(10) NOT NULL,
CONSTRAINT child_ref_fk FOREIGN KEY (ref) REFERENCES parent(val)
);
SQL
# Valid inserts — ref values that exist in parent.
sql -c "INSERT INTO child VALUES (10, 'apple'), (11, 'banana');"
run sql_csv -c "SELECT count(*) FROM child;"
[ "$status" -eq 0 ]
[[ "$output" =~ "2" ]] || false
# Invalid insert — 'grape' is not present in parent, FK must reject.
run sql -c "INSERT INTO child VALUES (12, 'grape');"
[ "$status" -ne 0 ]
# Row count unchanged after the rejected insert.
run sql_csv -c "SELECT count(*) FROM child;"
[ "$status" -eq 0 ]
[[ "$output" =~ "2" ]] || false
# dolt_verify_constraints reports no violations. Returns {0} on success.
run sql_csv -c "SELECT dolt_verify_constraints('--all');"
[ "$status" -eq 0 ]
[[ "$output" =~ "{0}" ]] || false
sql -c "SELECT dolt_add('.'); SELECT dolt_commit('-m', 'head: create child + fk');"
# --- Branch, write on both branches, then merge new into main ---
sql -c "SELECT dolt_branch('new');"
sql -c "INSERT INTO child VALUES (20, 'cherry');"
sql -c "SELECT dolt_add('.'); SELECT dolt_commit('-m', 'main: add cherry');"
sql <<SQL
SELECT dolt_checkout('new');
INSERT INTO child VALUES (30, 'banana');
SELECT dolt_add('.');
SELECT dolt_commit('-m', 'new: add banana on new branch');
SQL
run sql -c "SELECT dolt_merge('new');"
[ "$status" -eq 0 ]
# After merge, main should see rows from both branches.
run sql_csv -c "SELECT count(*) FROM child;"
[ "$status" -eq 0 ]
[[ "$output" =~ "4" ]] || false
run sql_csv -c "SELECT id, ref FROM child ORDER BY id;"
[ "$status" -eq 0 ]
[[ "$output" =~ "10,apple" ]] || false
[[ "$output" =~ "11,banana" ]] || false
[[ "$output" =~ "20,cherry" ]] || false
[[ "$output" =~ "30,banana" ]] || false
# FK still enforced after merge.
run sql -c "INSERT INTO child VALUES (99, 'grape');"
[ "$status" -ne 0 ]
# And still no reported constraint violations.
run sql_csv -c "SELECT dolt_verify_constraints('--all');"
[ "$status" -eq 0 ]
[[ "$output" =~ "{0}" ]] || false
stop_doltgres
}
@test "backward_workflow: dangling child violation surfaces on merge" {
[ -n "$DOLTGRES_LEGACY_BIN" ] || skip "requires DOLTGRES_LEGACY_BIN"
[ -n "$DOLTGRES_NEW_BIN" ] || skip "requires DOLTGRES_NEW_BIN"
# --- Old: create parent, seed rows ---
old_server_start
sql <<SQL
CREATE TABLE parent (
id INT NOT NULL PRIMARY KEY,
val VARCHAR(10) NOT NULL UNIQUE
);
INSERT INTO parent VALUES (1, 'apple'), (2, 'banana'), (3, 'cherry');
SQL
sql -c "SELECT dolt_add('.'); SELECT dolt_commit('-m', 'old: create parent');"
stop_doltgres
# --- New: create child + FK, seed a child that will become dangling ---
new_server_start
sql <<SQL
CREATE TABLE child (
id INT NOT NULL PRIMARY KEY,
ref VARCHAR(10) NOT NULL,
CONSTRAINT child_ref_fk FOREIGN KEY (ref) REFERENCES parent(val)
);
INSERT INTO child VALUES (10, 'apple');
SQL
sql -c "SELECT dolt_add('.'); SELECT dolt_commit('-m', 'head: create child + fk, seed');"
# No violations at this point.
run sql_csv -c "SELECT dolt_verify_constraints('--all');"
[ "$status" -eq 0 ]
[[ "$output" =~ "{0}" ]] || false
# Branch 'drop_banana': remove banana from parent (no child refs banana yet).
sql <<SQL
SELECT dolt_branch('drop_banana');
SELECT dolt_checkout('drop_banana');
DELETE FROM parent WHERE val='banana';
SELECT dolt_add('.');
SELECT dolt_commit('-m', 'drop_banana: remove banana');
SQL
# Back on main: add a child row that references banana (still valid on main).
sql <<SQL
SELECT dolt_checkout('main');
INSERT INTO child VALUES (20, 'banana');
SELECT dolt_add('.');
SELECT dolt_commit('-m', 'main: add child(20, banana)');
SQL
# merge should fail
run sql -c "SELECT dolt_merge('drop_banana');"
[ "$status" -ne 0 ]
# merge succeeds with constraint violation if forced
run sql -c "SET dolt_force_transaction_commit=1; SELECT dolt_merge('drop_banana');"
[ "$status" -eq 0 ]
# Verify the violation was recorded against child.
run sql_csv -c "SELECT count(*) FROM dolt_constraint_violations_child;"
[ "$status" -eq 0 ]
[[ "$output" =~ "1" ]] || false
# And that the violation row is the banana reference (id=20).
run sql_csv -c "SELECT id, ref FROM dolt_constraint_violations_child;"
[ "$status" -eq 0 ]
[[ "$output" =~ "20,banana" ]] || false
stop_doltgres
}
@test "backward_workflow: fk on text column from old repo, then merge" {
[ -n "$DOLTGRES_LEGACY_BIN" ] || skip "requires DOLTGRES_LEGACY_BIN"
[ -n "$DOLTGRES_NEW_BIN" ] || skip "requires DOLTGRES_NEW_BIN"
old_server_start
sql <<SQL
CREATE TABLE parent (
id INT NOT NULL PRIMARY KEY,
val TEXT NOT NULL UNIQUE
);
INSERT INTO parent VALUES (1, 'apple'), (2, 'banana'), (3, 'cherry');
SQL
sql -c "SELECT dolt_add('.'); SELECT dolt_commit('-m', 'old: create parent (text)');"
stop_doltgres
new_server_start
sql <<SQL
CREATE TABLE child (
id INT NOT NULL PRIMARY KEY,
ref TEXT NOT NULL,
CONSTRAINT child_ref_fk FOREIGN KEY (ref) REFERENCES parent(val)
);
SQL
sql -c "INSERT INTO child VALUES (10, 'apple'), (11, 'banana');"
run sql_csv -c "SELECT count(*) FROM child;"
[ "$status" -eq 0 ]
[[ "$output" =~ "2" ]] || false
run sql_csv -c "SELECT dolt_verify_constraints('--all');"
[ "$status" -eq 0 ]
[[ "$output" =~ "{0}" ]] || false
# Invalid insert — 'grape' is not present in parent, FK must reject.
run sql -c "INSERT INTO child VALUES (12, 'grape');"
[ "$status" -ne 0 ]
run sql_csv -c "SELECT count(*) FROM child;"
[ "$status" -eq 0 ]
[[ "$output" =~ "2" ]] || false
# No reported violations.
run sql_csv -c "SELECT dolt_verify_constraints('--all');"
[ "$status" -eq 0 ]
[[ "$output" =~ "{0}" ]] || false
sql -c "SELECT dolt_add('.'); SELECT dolt_commit('-m', 'head: create child + fk');"
# Branch, write on both, merge.
sql -c "SELECT dolt_branch('new');"
sql -c "INSERT INTO child VALUES (20, 'cherry');"
sql -c "SELECT dolt_add('.'); SELECT dolt_commit('-m', 'main: add cherry');"
sql <<SQL
SELECT dolt_checkout('new');
INSERT INTO child VALUES (30, 'banana');
SELECT dolt_add('.');
SELECT dolt_commit('-m', 'new: add banana on new branch');
SQL
run sql -c "SELECT dolt_merge('new');"
[ "$status" -eq 0 ]
run sql_csv -c "SELECT count(*) FROM child;"
[ "$status" -eq 0 ]
[[ "$output" =~ "4" ]] || false
# FK still enforced after merge.
run sql -c "INSERT INTO child VALUES (99, 'grape');"
[ "$status" -ne 0 ]
# Still no violations reported.
run sql_csv -c "SELECT dolt_verify_constraints('--all');"
[ "$status" -eq 0 ]
[[ "$output" =~ "{0}" ]] || false
stop_doltgres
}
@test "backward_workflow: fk from varchar child to text parent" {
[ -n "$DOLTGRES_LEGACY_BIN" ] || skip "requires DOLTGRES_LEGACY_BIN"
[ -n "$DOLTGRES_NEW_BIN" ] || skip "requires DOLTGRES_NEW_BIN"
# Cross-type FK: parent's TEXT column vs child's VARCHAR(10) column.
old_server_start
sql <<SQL
CREATE TABLE parent (
id INT NOT NULL PRIMARY KEY,
val TEXT NOT NULL UNIQUE
);
INSERT INTO parent VALUES (1, 'apple'), (2, 'banana'), (3, 'cherry');
SQL
sql -c "SELECT dolt_add('.'); SELECT dolt_commit('-m', 'old: create parent (text)');"
stop_doltgres
new_server_start
sql <<SQL
CREATE TABLE child (
id INT NOT NULL PRIMARY KEY,
ref VARCHAR(10) NOT NULL,
CONSTRAINT child_ref_fk FOREIGN KEY (ref) REFERENCES parent(val)
);
SQL
sql -c "INSERT INTO child VALUES (10, 'apple'), (11, 'banana');"
run sql_csv -c "SELECT count(*) FROM child;"
[ "$status" -eq 0 ]
[[ "$output" =~ "2" ]] || false
run sql -c "INSERT INTO child VALUES (12, 'grape');"
[ "$status" -ne 0 ]
run sql_csv -c "SELECT dolt_verify_constraints('--all');"
[ "$status" -eq 0 ]
[[ "$output" =~ "{0}" ]] || false
sql -c "SELECT dolt_add('.'); SELECT dolt_commit('-m', 'head: create child + fk (varchar → text)');"
stop_doltgres
}
@test "backward_workflow: fk from text child to varchar parent" {
[ -n "$DOLTGRES_LEGACY_BIN" ] || skip "requires DOLTGRES_LEGACY_BIN"
[ -n "$DOLTGRES_NEW_BIN" ] || skip "requires DOLTGRES_NEW_BIN"
old_server_start
sql <<SQL
CREATE TABLE parent (
id INT NOT NULL PRIMARY KEY,
val VARCHAR(10) NOT NULL UNIQUE
);
INSERT INTO parent VALUES (1, 'apple'), (2, 'banana'), (3, 'cherry');
SQL
sql -c "SELECT dolt_add('.'); SELECT dolt_commit('-m', 'old: create parent (varchar)');"
stop_doltgres
new_server_start
sql <<SQL
CREATE TABLE child (
id INT NOT NULL PRIMARY KEY,
ref TEXT NOT NULL,
CONSTRAINT child_ref_fk FOREIGN KEY (ref) REFERENCES parent(val)
);
SQL
sql -c "INSERT INTO child VALUES (10, 'apple'), (11, 'banana');"
run sql_csv -c "SELECT count(*) FROM child;"
[ "$status" -eq 0 ]
[[ "$output" =~ "2" ]] || false
run sql -c "INSERT INTO child VALUES (12, 'grape');"
[ "$status" -ne 0 ]
run sql_csv -c "SELECT dolt_verify_constraints('--all');"
[ "$status" -eq 0 ]
[[ "$output" =~ "{0}" ]] || false
sql -c "SELECT dolt_add('.'); SELECT dolt_commit('-m', 'head: create child + fk (text → varchar)');"
stop_doltgres
}
@test "backward_workflow: dangling child violation surfaces on merge (text)" {
[ -n "$DOLTGRES_LEGACY_BIN" ] || skip "requires DOLTGRES_LEGACY_BIN"
[ -n "$DOLTGRES_NEW_BIN" ] || skip "requires DOLTGRES_NEW_BIN"
old_server_start
sql <<SQL
CREATE TABLE parent (
id INT NOT NULL PRIMARY KEY,
val TEXT NOT NULL UNIQUE
);
INSERT INTO parent VALUES (1, 'apple'), (2, 'banana'), (3, 'cherry');
SQL
sql -c "SELECT dolt_add('.'); SELECT dolt_commit('-m', 'old: create parent (text)');"
stop_doltgres
new_server_start
sql <<SQL
CREATE TABLE child (
id INT NOT NULL PRIMARY KEY,
ref TEXT NOT NULL,
CONSTRAINT child_ref_fk FOREIGN KEY (ref) REFERENCES parent(val)
);
INSERT INTO child VALUES (10, 'apple');
SQL
sql -c "SELECT dolt_add('.'); SELECT dolt_commit('-m', 'head: create child + fk, seed');"
run sql_csv -c "SELECT dolt_verify_constraints('--all');"
[ "$status" -eq 0 ]
[[ "$output" =~ "{0}" ]] || false
# Branch 'drop_banana': remove banana from parent (no child refs banana yet).
sql <<SQL
SELECT dolt_branch('drop_banana');
SELECT dolt_checkout('drop_banana');
DELETE FROM parent WHERE val='banana';
SELECT dolt_add('.');
SELECT dolt_commit('-m', 'drop_banana: remove banana');
SQL
# Back on main: add a child row that references banana (still valid on main).
sql <<SQL
SELECT dolt_checkout('main');
INSERT INTO child VALUES (20, 'banana');
SELECT dolt_add('.');
SELECT dolt_commit('-m', 'main: add child(20, banana)');
SQL
# Merge should fail without forced commit due to constraint violation.
run sql -c "SELECT dolt_merge('drop_banana');"
[ "$status" -ne 0 ]
# Merge succeeds with constraint violation when forced.
run sql -c "SET dolt_force_transaction_commit=1; SELECT dolt_merge('drop_banana');"
[ "$status" -eq 0 ]
run sql_csv -c "SELECT count(*) FROM dolt_constraint_violations_child;"
[ "$status" -eq 0 ]
[[ "$output" =~ "1" ]] || false
run sql_csv -c "SELECT id, ref FROM dolt_constraint_violations_child;"
[ "$status" -eq 0 ]
[[ "$output" =~ "20,banana" ]] || false
stop_doltgres
}
integration-tests/compatibility/test_files/bats/bidirectional/bidirectional_compat.bats
+482
View File
@@ -0,0 +1,482 @@
#!/usr/bin/env bats
# Bidirectional compatibility tests for doltgresql.
#
# Each test creates an isolated repository and alternates reads/writes between
# two doltgres binaries — an older release (DOLTGRES_LEGACY_BIN) and the current
# HEAD build (DOLTGRES_NEW_BIN). The tests run in both directions: the runner
# swaps LEGACY and NEW to exercise forward and backward compatibility.
#
# Unlike the other compatibility tests, each test body manages its own server
# lifecycle; setup() and teardown() only manage the scratch directory.
#
# Environment variables:
# DOLTGRES_LEGACY_BIN — path to the "old" doltgres binary
# DOLTGRES_NEW_BIN — path to the "new" (HEAD) doltgres binary
# REPO_DIR — scratch directory base (empty, just needs to exist)
# Note: helper is one directory up from this file.
load $BATS_TEST_DIRNAME/../helper/common.bash
BATS_REPO=""
setup() {
BATS_REPO="$BATS_TMPDIR/bidir-$$-$RANDOM"
mkdir -p "$BATS_REPO"
}
teardown() {
stop_doltgres
rm -rf "$BATS_REPO"
}
# Convenience wrappers so test bodies read cleanly.
old_server_start() { start_doltgres "$DOLTGRES_LEGACY_BIN" "$BATS_REPO" "$BATS_REPO/old.log"; }
new_server_start() { start_doltgres "$DOLTGRES_NEW_BIN" "$BATS_REPO" "$BATS_REPO/new.log"; }
# ---------------------------------------------------------------------------
# Test 1: Scalar types DML — INT, VARCHAR, NUMERIC, TIMESTAMP round-trip.
# Four rounds: old → HEAD → old → HEAD.
# ---------------------------------------------------------------------------
@test "bidirectional: scalar types round-trip across versions" {
[ -n "$DOLTGRES_LEGACY_BIN" ] || skip "requires DOLTGRES_LEGACY_BIN"
[ -n "$DOLTGRES_NEW_BIN" ] || skip "requires DOLTGRES_NEW_BIN"
# --- Setup: old doltgres creates schema and seeds two rows ---
old_server_start
sql <<SQL
CREATE TABLE scalars (
pk INT NOT NULL PRIMARY KEY,
c_int INT,
c_varchar VARCHAR(255),
c_numeric NUMERIC(10,2),
c_ts TIMESTAMP
);
INSERT INTO scalars VALUES
(1, 100, 'old-row-1', 10.50, '2024-01-01 10:00:00'),
(2, 200, 'old-row-2', 20.75, '2024-06-15 12:30:00');
SQL
sql -c "SELECT dolt_add('.'); SELECT dolt_commit('-m', 'old: initial data');"
stop_doltgres
# --- Round 1: HEAD reads old's rows, inserts its own ---
new_server_start
run sql_csv -c "SELECT pk, c_varchar, c_numeric FROM scalars WHERE pk=1;"
[ "$status" -eq 0 ]
[[ "$output" =~ "1,old-row-1,10.50" ]] || false
run sql_csv -c "SELECT count(*) FROM scalars;"
[ "$status" -eq 0 ]
[[ "$output" =~ "2" ]] || false
sql -c "INSERT INTO scalars VALUES (3, 300, 'head-row-3', 30.25, '2025-01-15 08:00:00');"
sql -c "UPDATE scalars SET c_varchar='head-updated-1' WHERE pk=1;"
sql -c "SELECT dolt_add('.'); SELECT dolt_commit('-m', 'head: insert row 3, update row 1');"
stop_doltgres
# --- Round 2: old reads HEAD's changes ---
old_server_start
run sql_csv -c "SELECT pk, c_varchar FROM scalars WHERE pk=3;"
[ "$status" -eq 0 ]
[[ "$output" =~ "3,head-row-3" ]] || false
run sql_csv -c "SELECT pk, c_varchar FROM scalars WHERE pk=1;"
[ "$status" -eq 0 ]
[[ "$output" =~ "1,head-updated-1" ]] || false
sql -c "INSERT INTO scalars VALUES (4, 400, 'old-row-4', 40.00, '2025-03-01 09:00:00');"
sql -c "DELETE FROM scalars WHERE pk=2;"
sql -c "SELECT dolt_add('.'); SELECT dolt_commit('-m', 'old: insert row 4, delete row 2');"
stop_doltgres
# --- Round 3: HEAD reads old's changes ---
new_server_start
run sql_csv -c "SELECT pk, c_varchar, c_numeric FROM scalars WHERE pk=4;"
[ "$status" -eq 0 ]
[[ "$output" =~ "4,old-row-4,40.00" ]] || false
run sql_csv -c "SELECT count(*) FROM scalars WHERE pk=2;"
[ "$status" -eq 0 ]
[[ "$output" =~ "0" ]] || false
sql -c "INSERT INTO scalars VALUES (5, 500, 'head-row-5', 50.50, '2025-06-01 14:00:00');"
sql -c "UPDATE scalars SET c_numeric=99.99 WHERE pk=4;"
sql -c "SELECT dolt_add('.'); SELECT dolt_commit('-m', 'head: insert row 5, update row 4 numeric');"
stop_doltgres
# --- Round 4: old reads final state ---
old_server_start
run sql_csv -c "SELECT count(*) FROM scalars;"
[ "$status" -eq 0 ]
[[ "$output" =~ "4" ]] || false # pks 1, 3, 4, 5
run sql_csv -c "SELECT pk, c_numeric FROM scalars WHERE pk=4;"
[ "$status" -eq 0 ]
[[ "$output" =~ "4,99.99" ]] || false
run sql_csv -c "SELECT pk, c_varchar FROM scalars WHERE pk=5;"
[ "$status" -eq 0 ]
[[ "$output" =~ "5,head-row-5" ]] || false
stop_doltgres
}
# ---------------------------------------------------------------------------
# Test 2: Large TEXT values — out-of-band storage round-trip.
# ---------------------------------------------------------------------------
@test "bidirectional: large text values round-trip" {
[ -n "$DOLTGRES_LEGACY_BIN" ] || skip "requires DOLTGRES_LEGACY_BIN"
[ -n "$DOLTGRES_NEW_BIN" ] || skip "requires DOLTGRES_NEW_BIN"
# Setup: old creates table, inserts small and large text
old_server_start
sql <<SQL
CREATE TABLE texts (
pk INT NOT NULL PRIMARY KEY,
c_text TEXT
);
INSERT INTO texts VALUES
(1, 'old-small'),
(2, repeat('A', 70000));
SQL
sql -c "SELECT dolt_add('.'); SELECT dolt_commit('-m', 'old: initial texts');"
stop_doltgres
# Round 1: HEAD reads both rows, inserts its own large text
new_server_start
run sql_csv -c "SELECT pk, c_text FROM texts WHERE pk=1;"
[ "$status" -eq 0 ]
[[ "$output" =~ "old-small" ]] || false
run sql_csv -c "SELECT pk, length(c_text) FROM texts WHERE pk=2;"
[ "$status" -eq 0 ]
[[ "$output" =~ "2,70000" ]] || false
sql -c "INSERT INTO texts VALUES (3, repeat('H', 80000));"
sql -c "UPDATE texts SET c_text=repeat('U', 75000) WHERE pk=1;"
sql -c "SELECT dolt_add('.'); SELECT dolt_commit('-m', 'head: large insert, large update');"
stop_doltgres
# Round 2: old reads HEAD's large values
old_server_start
run sql_csv -c "SELECT pk, length(c_text) FROM texts WHERE pk=3;"
[ "$status" -eq 0 ]
[[ "$output" =~ "3,80000" ]] || false
run sql_csv -c "SELECT pk, length(c_text) FROM texts WHERE pk=1;"
[ "$status" -eq 0 ]
[[ "$output" =~ "1,75000" ]] || false
sql -c "UPDATE texts SET c_text=repeat('V', 90000) WHERE pk=2;"
sql -c "SELECT dolt_add('.'); SELECT dolt_commit('-m', 'old: update row 2 to even larger');"
stop_doltgres
# Round 3: HEAD reads old's in-place update
new_server_start
run sql_csv -c "SELECT pk, length(c_text) FROM texts WHERE pk=2;"
[ "$status" -eq 0 ]
[[ "$output" =~ "2,90000" ]] || false
run sql_csv -c "SELECT count(*) FROM texts;"
[ "$status" -eq 0 ]
[[ "$output" =~ "3" ]] || false
stop_doltgres
}
# ---------------------------------------------------------------------------
# Test 3: ADD COLUMN DDL — both versions add columns to the same table.
# ---------------------------------------------------------------------------
@test "bidirectional: add columns from both versions" {
[ -n "$DOLTGRES_LEGACY_BIN" ] || skip "requires DOLTGRES_LEGACY_BIN"
[ -n "$DOLTGRES_NEW_BIN" ] || skip "requires DOLTGRES_NEW_BIN"
# Setup: old creates minimal table
old_server_start
sql <<SQL
CREATE TABLE evolving (
pk INT NOT NULL PRIMARY KEY,
c_base INT
);
INSERT INTO evolving VALUES (1, 10), (2, 20), (3, 30);
SQL
sql -c "SELECT dolt_add('.'); SELECT dolt_commit('-m', 'old: base schema');"
stop_doltgres
# Round 1: HEAD adds TEXT and DATE columns
new_server_start
sql -c "ALTER TABLE evolving ADD COLUMN c_text TEXT;"
sql -c "ALTER TABLE evolving ADD COLUMN c_date DATE;"
sql -c "UPDATE evolving SET c_text='text-' || pk::text, c_date='2025-01-01';"
sql -c "INSERT INTO evolving VALUES (4, 40, 'text-4', '2025-02-01');"
sql -c "SELECT dolt_add('.'); SELECT dolt_commit('-m', 'head: add text and date columns');"
stop_doltgres
# Round 2: old reads HEAD's new columns, adds its own
old_server_start
run sql_csv -c "SELECT pk, c_text, c_date FROM evolving WHERE pk=1;"
[ "$status" -eq 0 ]
[[ "$output" =~ "1,text-1,2025-01-01" ]] || false
run sql_csv -c "SELECT pk, c_text FROM evolving WHERE pk=4;"
[ "$status" -eq 0 ]
[[ "$output" =~ "4,text-4" ]] || false
sql -c "ALTER TABLE evolving ADD COLUMN c_int2 INT;"
sql -c "ALTER TABLE evolving ADD COLUMN c_numeric NUMERIC(8,2);"
sql -c "UPDATE evolving SET c_int2=pk*100, c_numeric=pk*1.5;"
sql -c "INSERT INTO evolving VALUES (5, 50, 'text-5', '2025-03-01', 500, 7.50);"
sql -c "SELECT dolt_add('.'); SELECT dolt_commit('-m', 'old: add int2 and numeric columns');"
stop_doltgres
# Round 3: HEAD reads all 4 added columns
new_server_start
run sql_csv -c "SELECT pk, c_text, c_date, c_int2, c_numeric FROM evolving WHERE pk=1;"
[ "$status" -eq 0 ]
[[ "$output" =~ "1,text-1,2025-01-01,100,1.50" ]] || false
run sql_csv -c "SELECT pk, c_text, c_int2, c_numeric FROM evolving WHERE pk=5;"
[ "$status" -eq 0 ]
[[ "$output" =~ "5,text-5,500,7.50" ]] || false
stop_doltgres
}
# ---------------------------------------------------------------------------
# Test 4: Branch and merge — both versions create branches, merge across
# version boundaries.
# ---------------------------------------------------------------------------
@test "bidirectional: branch and merge across versions" {
[ -n "$DOLTGRES_LEGACY_BIN" ] || skip "requires DOLTGRES_LEGACY_BIN"
[ -n "$DOLTGRES_NEW_BIN" ] || skip "requires DOLTGRES_NEW_BIN"
# Setup: old creates repo with base table
old_server_start
sql <<SQL
CREATE TABLE shared (
pk INT NOT NULL PRIMARY KEY,
val VARCHAR(100),
src VARCHAR(20)
);
INSERT INTO shared VALUES (1, 'base-1', 'old'), (2, 'base-2', 'old');
SQL
sql -c "SELECT dolt_add('.'); SELECT dolt_commit('-m', 'old: base data');"
stop_doltgres
# Round 1: HEAD creates a feature branch
new_server_start
sql <<SQL
SELECT dolt_branch('head_feature');
SELECT dolt_checkout('head_feature');
INSERT INTO shared VALUES (10, 'head-feature-10', 'head'), (11, 'head-feature-11', 'head');
SELECT dolt_add('.');
SELECT dolt_commit('-m', 'head: feature branch inserts');
SQL
stop_doltgres
# Round 2: old creates its own branch, merges HEAD's feature branch
old_server_start
sql <<SQL
SELECT dolt_branch('old_branch');
SELECT dolt_checkout('old_branch');
INSERT INTO shared VALUES (20, 'old-branch-20', 'old');
SELECT dolt_add('.');
SELECT dolt_commit('-m', 'old: old_branch insert');
SQL
sql <<SQL
INSERT INTO shared VALUES (3, 'base-3', 'old');
SQL
sql -c "SELECT dolt_add('.'); SELECT dolt_commit('-m', 'old: main insert');"
sql -c "SELECT dolt_merge('head_feature');"
run sql_csv -c "SELECT count(*) FROM shared;"
[ "$status" -eq 0 ]
[[ "$output" =~ "5" ]] || false # 1,2,3,10,11
run sql_csv -c "SELECT pk, val FROM shared WHERE pk=10;"
[ "$status" -eq 0 ]
[[ "$output" =~ "10,head-feature-10" ]] || false
stop_doltgres
# Round 3: HEAD reads merged state, merges old_branch
new_server_start
run sql_csv -c "SELECT count(*) FROM shared;"
[ "$status" -eq 0 ]
[[ "$output" =~ "5" ]] || false
run sql_csv -c "SELECT pk, val FROM shared WHERE pk=3;"
[ "$status" -eq 0 ]
[[ "$output" =~ "3,base-3" ]] || false
sql -c "SELECT dolt_merge('old_branch');"
run sql_csv -c "SELECT count(*) FROM shared;"
[ "$status" -eq 0 ]
[[ "$output" =~ "6" ]] || false # 1,2,3,10,11,20
run sql_csv -c "SELECT pk, val FROM shared WHERE pk=20;"
[ "$status" -eq 0 ]
[[ "$output" =~ "20,old-branch-20" ]] || false
stop_doltgres
}
# ---------------------------------------------------------------------------
# Test 5: Comprehensive type coverage — both versions add columns of different
# type families across rounds.
# ---------------------------------------------------------------------------
@test "bidirectional: comprehensive type coverage across versions" {
[ -n "$DOLTGRES_LEGACY_BIN" ] || skip "requires DOLTGRES_LEGACY_BIN"
[ -n "$DOLTGRES_NEW_BIN" ] || skip "requires DOLTGRES_NEW_BIN"
# Setup: old creates minimal table
old_server_start
sql <<SQL
CREATE TABLE typed (
pk INT NOT NULL PRIMARY KEY
);
INSERT INTO typed (pk) VALUES (1), (2), (3);
SQL
sql -c "SELECT dolt_add('.'); SELECT dolt_commit('-m', 'old: pk-only base');"
stop_doltgres
# Round 1: HEAD adds numeric columns
new_server_start
sql -c "ALTER TABLE typed ADD COLUMN c_smallint SMALLINT;"
sql -c "ALTER TABLE typed ADD COLUMN c_bigint BIGINT;"
sql -c "ALTER TABLE typed ADD COLUMN c_real REAL;"
sql -c "ALTER TABLE typed ADD COLUMN c_double DOUBLE PRECISION;"
sql -c "UPDATE typed SET c_smallint=pk*10, c_bigint=pk*1000000, c_real=pk*1.5, c_double=pk*2.5;"
sql -c "INSERT INTO typed (pk, c_smallint, c_bigint, c_real, c_double) VALUES (4, 40, 4000000, 6.0, 10.0);"
sql -c "SELECT dolt_add('.'); SELECT dolt_commit('-m', 'head: add numeric columns');"
stop_doltgres
# Round 2: old reads HEAD's numeric columns, adds text/binary columns
old_server_start
run sql_csv -c "SELECT pk, c_smallint, c_bigint FROM typed WHERE pk=1;"
[ "$status" -eq 0 ]
[[ "$output" =~ "1,10,1000000" ]] || false
sql -c "ALTER TABLE typed ADD COLUMN c_varchar VARCHAR(255);"
sql -c "ALTER TABLE typed ADD COLUMN c_bytea BYTEA;"
sql -c "UPDATE typed SET c_varchar='varchar-' || pk::text, c_bytea='\xDEAD';"
sql -c "INSERT INTO typed (pk, c_varchar, c_bytea) VALUES (5, 'varchar-5', '\xBEEF');"
sql -c "SELECT dolt_add('.'); SELECT dolt_commit('-m', 'old: add varchar and bytea columns');"
stop_doltgres
# Round 3: HEAD reads old's string/binary columns, adds temporal/numeric
new_server_start
run sql_csv -c "SELECT pk, c_varchar FROM typed WHERE pk=1;"
[ "$status" -eq 0 ]
[[ "$output" =~ "1,varchar-1" ]] || false
run sql_csv -c "SELECT pk, c_varchar FROM typed WHERE pk=5;"
[ "$status" -eq 0 ]
[[ "$output" =~ "5,varchar-5" ]] || false
sql -c "ALTER TABLE typed ADD COLUMN c_date DATE;"
sql -c "ALTER TABLE typed ADD COLUMN c_numeric NUMERIC(10,3);"
sql -c "UPDATE typed SET c_date='2025-01-01', c_numeric=pk*3.141;"
sql -c "SELECT dolt_add('.'); SELECT dolt_commit('-m', 'head: add temporal and numeric columns');"
stop_doltgres
# Round 4: old reads HEAD's temporal/numeric columns, adds boolean/jsonb
old_server_start
run sql_csv -c "SELECT pk, c_date, c_numeric FROM typed WHERE pk=1;"
[ "$status" -eq 0 ]
[[ "$output" =~ "1,2025-01-01,3.141" ]] || false
sql -c "ALTER TABLE typed ADD COLUMN c_boolean BOOLEAN;"
sql -c "ALTER TABLE typed ADD COLUMN c_jsonb JSONB;"
sql -c "UPDATE typed SET c_boolean=(pk % 2 = 1), c_jsonb=json_build_object('pk', pk);"
sql -c "INSERT INTO typed (pk, c_varchar, c_date, c_numeric, c_boolean, c_jsonb) VALUES (6, 'varchar-6', '2025-06-01', 18.847, false, '{\"old\":true}');"
sql -c "SELECT dolt_add('.'); SELECT dolt_commit('-m', 'old: add boolean and jsonb columns');"
stop_doltgres
# Round 5: HEAD reads old's boolean/jsonb, does a final insert using all columns
new_server_start
run sql_csv -c "SELECT pk, c_boolean FROM typed WHERE pk=1;"
[ "$status" -eq 0 ]
[[ "$output" =~ "1,t" ]] || false
run sql_csv -c "SELECT pk, c_boolean FROM typed WHERE pk=2;"
[ "$status" -eq 0 ]
[[ "$output" =~ "2,f" ]] || false
run sql_csv -c "SELECT pk, c_jsonb->>'old' FROM typed WHERE pk=6;"
[ "$status" -eq 0 ]
[[ "$output" =~ "true" ]] || false
sql -c "INSERT INTO typed (pk, c_smallint, c_bigint, c_varchar, c_date, c_numeric, c_boolean, c_jsonb) VALUES (7, 70, 7000000, 'varchar-7', '2025-07-01', 21.988, true, '{\"head\":true}');"
sql -c "SELECT dolt_add('.'); SELECT dolt_commit('-m', 'head: final insert using all columns');"
stop_doltgres
# Round 6: old reads HEAD's final insert — all columns visible
old_server_start
run sql_csv -c "SELECT pk, c_smallint, c_bigint, c_varchar, c_numeric FROM typed WHERE pk=7;"
[ "$status" -eq 0 ]
[[ "$output" =~ "7,70,7000000,varchar-7,21.988" ]] || false
run sql_csv -c "SELECT count(*) FROM typed;"
[ "$status" -eq 0 ]
[[ "$output" =~ "7" ]] || false
stop_doltgres
}
# ---------------------------------------------------------------------------
# Test 6: JSONB round-trip — inline and large JSONB documents across versions.
# ---------------------------------------------------------------------------
@test "bidirectional: jsonb round-trip across versions" {
[ -n "$DOLTGRES_LEGACY_BIN" ] || skip "requires DOLTGRES_LEGACY_BIN"
[ -n "$DOLTGRES_NEW_BIN" ] || skip "requires DOLTGRES_NEW_BIN"
# Setup: old creates table with small and large JSONB
old_server_start
sql <<SQL
CREATE TABLE jsondocs (
pk INT NOT NULL PRIMARY KEY,
c_small JSONB,
c_big JSONB
);
INSERT INTO jsondocs VALUES
(1, '{"key":"val1","num":100}', '{"meta":"small"}'),
(2, '{"key":"val2"}', json_build_object('big', repeat('x', 5000))::jsonb);
SQL
sql -c "SELECT dolt_add('.'); SELECT dolt_commit('-m', 'old: initial jsonb');"
stop_doltgres
# Round 1: HEAD reads both rows, inserts its own
new_server_start
run sql_csv -c "SELECT pk, c_small->>'key', c_small->>'meta' FROM jsondocs WHERE pk=1;"
[ "$status" -eq 0 ]
[[ "$output" =~ "val1" ]] || false
run sql_csv -c "SELECT pk, length(c_big->>'big') FROM jsondocs WHERE pk=2;"
[ "$status" -eq 0 ]
[[ "$output" =~ "2,5000" ]] || false
sql -c "INSERT INTO jsondocs VALUES (3, '{\"head\":true}', json_build_object('hbig', repeat('H', 6000))::jsonb);"
sql -c "UPDATE jsondocs SET c_small=c_small || '{\"updated\":true}' WHERE pk=1;"
sql -c "SELECT dolt_add('.'); SELECT dolt_commit('-m', 'head: insert row 3, update row 1');"
stop_doltgres
# Round 2: old reads HEAD's changes
old_server_start
run sql_csv -c "SELECT pk, c_small->>'head' FROM jsondocs WHERE pk=3;"
[ "$status" -eq 0 ]
[[ "$output" =~ "true" ]] || false
run sql_csv -c "SELECT pk, c_small->>'updated' FROM jsondocs WHERE pk=1;"
[ "$status" -eq 0 ]
[[ "$output" =~ "true" ]] || false
run sql_csv -c "SELECT pk, length(c_big->>'hbig') FROM jsondocs WHERE pk=3;"
[ "$status" -eq 0 ]
[[ "$output" =~ "3,6000" ]] || false
run sql_csv -c "SELECT count(*) FROM jsondocs;"
[ "$status" -eq 0 ]
[[ "$output" =~ "3" ]] || false
stop_doltgres
}
integration-tests/compatibility/test_files/bats/compatibility.bats
+270
View File
@@ -0,0 +1,270 @@
#!/usr/bin/env bats
# Backward / forward compatibility tests for doltgresql.
#
# Each test starts a fresh doltgres server against a copy of the pre-created repo
# (REPO_DIR) and verifies that the binary under test can correctly read and write
# the data created by the other version.
#
# Environment variables consumed by this file:
# REPO_DIR — path to the data directory created by setup_repo.sh
# DOLTGRES_TEST_BIN — doltgres binary to use (default: doltgres from PATH)
load $BATS_TEST_DIRNAME/helper/common.bash
BATS_REPO=""
setup() {
BATS_REPO="$BATS_TMPDIR/compat-repo-$$-$RANDOM"
copy_repo "$REPO_DIR" "$BATS_REPO"
start_doltgres "${DOLTGRES_TEST_BIN:-doltgres}" "$BATS_REPO" "$BATS_REPO/server.log"
}
teardown() {
stop_doltgres
rm -rf "$BATS_REPO"
}
# ---------------------------------------------------------------------------
# Sanity / version checks
# ---------------------------------------------------------------------------
@test "compatibility: server is accessible" {
run sql -c "SELECT 1;"
[ "$status" -eq 0 ]
}
@test "compatibility: dolt_version returns a version string" {
run sql -c "SELECT dolt_version();"
[ "$status" -eq 0 ]
[[ "$output" =~ [0-9]+\.[0-9]+\.[0-9]+ ]] || false
}
# ---------------------------------------------------------------------------
# Branch enumeration
# ---------------------------------------------------------------------------
@test "compatibility: expected branches exist" {
run sql_csv -c "SELECT name FROM dolt_branches ORDER BY name;"
[ "$status" -eq 0 ]
[[ "$output" =~ "check_merge" ]] || false
[[ "$output" =~ "init" ]] || false
[[ "$output" =~ "main" ]] || false
[[ "$output" =~ "other" ]] || false
}
# ---------------------------------------------------------------------------
# Working set is clean on main
# ---------------------------------------------------------------------------
@test "compatibility: working set is clean on main" {
run sql -c "SELECT * FROM dolt_status;"
[ "$status" -eq 0 ]
[[ "$output" =~ "(0 rows)" ]] || false
}
# ---------------------------------------------------------------------------
# Schema on branch init (original schema before alterations)
# ---------------------------------------------------------------------------
@test "compatibility: schema on branch init has original abc columns" {
run sql <<SQL
SELECT dolt_checkout('init');
SELECT column_name FROM information_schema.columns
WHERE table_name = 'abc' ORDER BY ordinal_position;
SQL
[ "$status" -eq 0 ]
[[ "$output" =~ "pk" ]] || false
[[ "$output" =~ " a" ]] || false
[[ "$output" =~ " b" ]] || false
[[ "$output" =~ " w" ]] || false
[[ "$output" =~ " x" ]] || false
}
# ---------------------------------------------------------------------------
# Data on branch init
# ---------------------------------------------------------------------------
@test "compatibility: data on branch init matches initial insert" {
run sql_csv <<SQL
SELECT dolt_checkout('init');
SELECT pk, a, b, w, x FROM abc ORDER BY pk;
SQL
[ "$status" -eq 0 ]
[[ "$output" =~ "0,asdf,1.1,0,0" ]] || false
[[ "$output" =~ "1,asdf,1.1,0,0" ]] || false
[[ "$output" =~ "2,asdf,1.1,0,0" ]] || false
}
# ---------------------------------------------------------------------------
# Schema on main (after ALTER TABLE DROP/ADD)
# ---------------------------------------------------------------------------
@test "compatibility: schema on main has altered abc columns" {
run sql <<SQL
SELECT column_name FROM information_schema.columns
WHERE table_name = 'abc' ORDER BY ordinal_position;
SQL
[ "$status" -eq 0 ]
[[ "$output" =~ "pk" ]] || false
[[ "$output" =~ " a" ]] || false
[[ "$output" =~ " b" ]] || false
[[ "$output" =~ " x" ]] || false
[[ "$output" =~ " y" ]] || false
# 'w' was dropped on main
[[ ! "$output" =~ " w" ]] || false
}
# ---------------------------------------------------------------------------
# Data on main
# ---------------------------------------------------------------------------
@test "compatibility: data on main matches expected changes" {
run sql_csv -c "SELECT pk, a, x, y FROM abc ORDER BY pk;"
[ "$status" -eq 0 ]
[[ "$output" =~ "0,asdf,1,121" ]] || false
[[ "$output" =~ "2,asdf,0,121" ]] || false
[[ "$output" =~ "3,data,0,121" ]] || false
}
# ---------------------------------------------------------------------------
# Schema on branch other (different ALTER TABLE than main)
# ---------------------------------------------------------------------------
@test "compatibility: schema on branch other has w and z columns" {
run sql <<SQL
SELECT dolt_checkout('other');
SELECT column_name FROM information_schema.columns
WHERE table_name = 'abc' ORDER BY ordinal_position;
SQL
[ "$status" -eq 0 ]
[[ "$output" =~ "pk" ]] || false
[[ "$output" =~ " w" ]] || false
[[ "$output" =~ " z" ]] || false
# 'x' was dropped on other
[[ ! "$output" =~ " x" ]] || false
}
# ---------------------------------------------------------------------------
# Data on branch other
# ---------------------------------------------------------------------------
@test "compatibility: data on branch other matches expected changes" {
run sql_csv <<SQL
SELECT dolt_checkout('other');
SELECT pk, a, w, z FROM abc ORDER BY pk;
SQL
[ "$status" -eq 0 ]
[[ "$output" =~ "0,asdf,1,122" ]] || false
[[ "$output" =~ "1,asdf,0,122" ]] || false
[[ "$output" =~ "4,data,0,122" ]] || false
# pk=2 was deleted on other
[[ ! "$output" =~ ",2," ]] || false
}
# ---------------------------------------------------------------------------
# big table
# ---------------------------------------------------------------------------
@test "compatibility: big table has 1000 rows" {
run sql_csv -c "SELECT count(*) FROM big;"
[ "$status" -eq 0 ]
[[ "$output" =~ "1000" ]] || false
}
@test "compatibility: big table supports delete and insert" {
sql -c "DELETE FROM big WHERE pk IN (71, 331, 881);"
run sql_csv -c "SELECT count(*) FROM big;"
[ "$status" -eq 0 ]
[[ "$output" =~ "997" ]] || false
sql -c "INSERT INTO big VALUES (1001, 'foo'), (1002, 'bar'), (1003, 'baz');"
run sql_csv -c "SELECT count(*) FROM big;"
[ "$status" -eq 0 ]
[[ "$output" =~ "1000" ]] || false
sql -c "SELECT dolt_add('.'); SELECT dolt_commit('-m', 'modified big');"
}
# ---------------------------------------------------------------------------
# View
# ---------------------------------------------------------------------------
@test "compatibility: view1 is queryable" {
run sql -c "SELECT * FROM view1;"
[ "$status" -eq 0 ]
[[ "$output" =~ "4" ]] || false
}
@test "compatibility: all_types_view has 3 rows" {
run sql_csv -c "SELECT count(*) FROM all_types_view;"
[ "$status" -eq 0 ]
[[ "$output" =~ "3" ]] || false
}
# ---------------------------------------------------------------------------
# DML on existing tables (write test)
# ---------------------------------------------------------------------------
@test "compatibility: can insert and read back on main" {
sql -c "INSERT INTO abc (pk, a, b, x, y) VALUES (99, 'new', 9.9, 9, 99);"
run sql_csv -c "SELECT pk, a, x, y FROM abc WHERE pk=99;"
[ "$status" -eq 0 ]
[[ "$output" =~ "99,new,9,99" ]] || false
sql -c "DELETE FROM abc WHERE pk=99;"
}
@test "compatibility: can update a row on main" {
sql -c "UPDATE abc SET a='updated' WHERE pk=0;"
run sql_csv -c "SELECT pk, a FROM abc WHERE pk=0;"
[ "$status" -eq 0 ]
[[ "$output" =~ "0,updated" ]] || false
}
@test "compatibility: dml is committable" {
sql -c "INSERT INTO abc (pk, a, b, x, y) VALUES (200, 'commit-test', 1.0, 1, 1);"
sql -c "SELECT dolt_add('.');"
run sql -c "SELECT length(dolt_commit('-m', 'compat dml commit')::text);"
[ "$status" -eq 0 ]
[[ "$output" =~ "34" ]] || false
}
# ---------------------------------------------------------------------------
# Check constraint (def table)
# ---------------------------------------------------------------------------
@test "compatibility: check constraint is enforced" {
run sql -c "INSERT INTO def VALUES (-1);"
[ "$status" -ne 0 ]
}
@test "compatibility: valid insert into def succeeds" {
run sql -c "INSERT INTO def VALUES (100);"
[ "$status" -eq 0 ]
}
# ---------------------------------------------------------------------------
# Merge: check_merge into main (should succeed cleanly)
# ---------------------------------------------------------------------------
@test "compatibility: clean merge of check_merge into main succeeds" {
run sql <<SQL
SELECT dolt_merge('check_merge');
SQL
[ "$status" -eq 0 ]
}
# ---------------------------------------------------------------------------
# Merge: other into main (should produce a conflict on abc)
# ---------------------------------------------------------------------------
@test "compatibility: conflicting merge of other into main is detected" {
run sql -c "SELECT dolt_merge('other');"
# The merge should either return a non-zero status or report conflicts
# in dolt_conflicts — either outcome confirms conflict detection works.
if [ "$status" -eq 0 ]; then
run sql_csv -c "SELECT count(*) FROM dolt_conflicts;"
[ "$status" -eq 0 ]
[[ "$output" =~ [1-9] ]] || false
fi
}
integration-tests/compatibility/test_files/bats/helper/common.bash
+114
View File
@@ -0,0 +1,114 @@
# Common helpers for doltgresql compatibility BATS tests.
# Loaded by all test files in this directory tree via:
# load $BATS_TEST_DIRNAME/../helper/common.bash (from subdirs)
# or
# load $BATS_TEST_DIRNAME/helper/common.bash (from top level)
# ---------------------------------------------------------------------------
# Server lifecycle
# ---------------------------------------------------------------------------
COMPAT_SERVER_PID=""
COMPAT_SERVER_PORT=""
# pick_port — find an unused TCP port in the range 2048–6144.
pick_port() {
for i in {0..99}; do
port=$((RANDOM % 4096 + 2048))
if ! nc -z localhost "$port" 2>/dev/null; then
echo "$port"
return 0
fi
done
echo "ERROR: could not find a free port" >&2
return 1
}
# write_config <dir> <port> — write a minimal doltgres config.yaml into <dir>.
write_config() {
local dir="$1"
local port="$2"
cat > "$dir/compat-config.yaml" <<EOF
log_level: warning
behavior:
read_only: false
disable_client_multi_statements: false
listener:
host: localhost
port: $port
EOF
}
# start_doltgres <binary> <datadir> [logfile]
# Starts a doltgres server and waits for it to accept connections.
# Sets COMPAT_SERVER_PID and COMPAT_SERVER_PORT.
start_doltgres() {
local binary="${1:?start_doltgres: binary required}"
local datadir="${2:?start_doltgres: datadir required}"
local logfile="${3:-/dev/null}"
COMPAT_SERVER_PORT=$(pick_port)
write_config "$datadir" "$COMPAT_SERVER_PORT"
PGPASSWORD=password "$binary" -data-dir="$datadir" --config="$datadir/compat-config.yaml" \
> "$logfile" 2>&1 &
COMPAT_SERVER_PID=$!
# Wait up to 15 s
local end=$((SECONDS + 15))
while [ $SECONDS -lt $end ]; do
if PGPASSWORD=password psql -U postgres -h localhost -p "$COMPAT_SERVER_PORT" \
-c "SELECT 1;" postgres >/dev/null 2>&1; then
return 0
fi
sleep 0.5
done
echo "ERROR: doltgres server failed to start on port $COMPAT_SERVER_PORT" >&2
if [ -f "$logfile" ]; then cat "$logfile" >&2; fi
kill "$COMPAT_SERVER_PID" 2>/dev/null
COMPAT_SERVER_PID=""
return 1
}
# stop_doltgres — kill the server started by start_doltgres.
stop_doltgres() {
if [ -n "$COMPAT_SERVER_PID" ]; then
kill "$COMPAT_SERVER_PID" 2>/dev/null || true
wait "$COMPAT_SERVER_PID" 2>/dev/null || true
COMPAT_SERVER_PID=""
COMPAT_SERVER_PORT=""
fi
}
# ---------------------------------------------------------------------------
# Query helpers
# ---------------------------------------------------------------------------
# sql [psql-args...] — run psql against the current server's postgres database.
# Use with heredoc for multi-statement sessions:
# sql <<SQL
# SELECT dolt_checkout('other');
# SELECT * FROM abc;
# SQL
# Or with -c for a single statement:
# run sql -c "SELECT count(*) FROM abc;"
sql() {
PGPASSWORD=password psql -U postgres -h localhost -p "$COMPAT_SERVER_PORT" \
-v ON_ERROR_STOP=1 "$@" postgres
}
# sql_csv [psql-args...] — same as sql but outputs CSV (--csv flag).
sql_csv() {
PGPASSWORD=password psql --csv -U postgres -h localhost -p "$COMPAT_SERVER_PORT" \
-v ON_ERROR_STOP=1 "$@" postgres
}
# ---------------------------------------------------------------------------
# Repo isolation
# ---------------------------------------------------------------------------
# copy_repo <src> <dst> — copy a data directory for test isolation.
copy_repo() {
cp -Rpf "$1" "$2"
}
integration-tests/compatibility/test_files/bats/types_compatibility.bats
+367
View File
@@ -0,0 +1,367 @@
#!/usr/bin/env bats
# Types compatibility tests for doltgresql.
#
# Verifies that the current doltgres build can correctly read, write, and alter
# all_types rows that were originally written by an older doltgres version (or
# by the current version in a forward-compatibility run).
#
# Depends on the all_types table created by setup_repo.sh.
load $BATS_TEST_DIRNAME/helper/common.bash
BATS_REPO=""
setup() {
BATS_REPO="$BATS_TMPDIR/types-compat-repo-$$-$RANDOM"
copy_repo "$REPO_DIR" "$BATS_REPO"
start_doltgres "${DOLTGRES_TEST_BIN:-doltgres}" "$BATS_REPO" "$BATS_REPO/server.log"
}
teardown() {
stop_doltgres
rm -rf "$BATS_REPO"
}
# ---------------------------------------------------------------------------
# Row counts
# ---------------------------------------------------------------------------
@test "types_compat: all_types has 3 rows" {
run sql_csv -c "SELECT count(*) FROM all_types;"
[ "$status" -eq 0 ]
[[ "$output" =~ "3" ]] || false
}
# ---------------------------------------------------------------------------
# Numeric types
# ---------------------------------------------------------------------------
@test "types_compat: smallint column readable" {
run sql_csv -c "SELECT pk, c_smallint FROM all_types WHERE pk=1;"
[ "$status" -eq 0 ]
[[ "$output" =~ "1,100" ]] || false
}
@test "types_compat: negative smallint readable" {
run sql_csv -c "SELECT pk, c_smallint FROM all_types WHERE pk=2;"
[ "$status" -eq 0 ]
[[ "$output" =~ "2,-100" ]] || false
}
@test "types_compat: int and bigint columns readable" {
run sql_csv -c "SELECT pk, c_int, c_bigint FROM all_types WHERE pk=1;"
[ "$status" -eq 0 ]
[[ "$output" =~ "1,2000000,9223372036854775807" ]] || false
}
@test "types_compat: negative int and bigint readable" {
run sql_csv -c "SELECT pk, c_int, c_bigint FROM all_types WHERE pk=2;"
[ "$status" -eq 0 ]
[[ "$output" =~ "2,-2000000,-9223372036854775807" ]] || false
}
@test "types_compat: real and double precision readable" {
run sql_csv -c "SELECT pk, c_real, c_double FROM all_types WHERE pk=1;"
[ "$status" -eq 0 ]
[[ "$output" =~ "1,1.5,2.5" ]] || false
}
@test "types_compat: numeric column readable" {
run sql_csv -c "SELECT pk, c_numeric FROM all_types WHERE pk=1;"
[ "$status" -eq 0 ]
[[ "$output" =~ "1,12345.67" ]] || false
}
@test "types_compat: negative numeric readable" {
run sql_csv -c "SELECT pk, c_numeric FROM all_types WHERE pk=2;"
[ "$status" -eq 0 ]
[[ "$output" =~ "2,-12345.67" ]] || false
}
# ---------------------------------------------------------------------------
# Text types
# ---------------------------------------------------------------------------
@test "types_compat: char column readable" {
run sql_csv -c "SELECT pk, trim(c_char) FROM all_types WHERE pk=1;"
[ "$status" -eq 0 ]
[[ "$output" =~ "hello" ]] || false
}
@test "types_compat: varchar column readable" {
run sql_csv -c "SELECT pk, c_varchar FROM all_types WHERE pk=1;"
[ "$status" -eq 0 ]
[[ "$output" =~ "1,hello world" ]] || false
}
@test "types_compat: text column readable" {
run sql_csv -c "SELECT pk, c_text FROM all_types WHERE pk=1;"
[ "$status" -eq 0 ]
[[ "$output" =~ "1,text val" ]] || false
}
@test "types_compat: large text value (500 chars) readable" {
run sql_csv -c "SELECT pk, length(c_text) FROM all_types WHERE pk=3;"
[ "$status" -eq 0 ]
[[ "$output" =~ "3,500" ]] || false
}
# ---------------------------------------------------------------------------
# Temporal types
# ---------------------------------------------------------------------------
@test "types_compat: date column readable" {
run sql_csv -c "SELECT pk, c_date FROM all_types WHERE pk=1;"
[ "$status" -eq 0 ]
[[ "$output" =~ "1,2024-01-15" ]] || false
}
@test "types_compat: time column readable" {
skip "broken functionality"
run sql_csv -c "SELECT pk, c_time FROM all_types WHERE pk=1;"
[ "$status" -eq 0 ]
[[ "$output" =~ "13:30:45" ]] || false
}
@test "types_compat: timestamp column readable" {
run sql_csv -c "SELECT pk, c_timestamp FROM all_types WHERE pk=1;"
[ "$status" -eq 0 ]
[[ "$output" =~ "2024-01-15" ]] || false
[[ "$output" =~ "13:30:45" ]] || false
}
@test "types_compat: timestamptz not null for pk=1" {
run sql_csv -c "SELECT pk, (c_timestamptz IS NOT NULL) FROM all_types WHERE pk=1;"
[ "$status" -eq 0 ]
[[ "$output" =~ "t" ]] || false
}
@test "types_compat: timestamptz is null for pk=2" {
run sql_csv -c "SELECT pk, (c_timestamptz IS NULL) FROM all_types WHERE pk=2;"
[ "$status" -eq 0 ]
[[ "$output" =~ "t" ]] || false
}
# ---------------------------------------------------------------------------
# Boolean type
# ---------------------------------------------------------------------------
@test "types_compat: boolean column readable" {
run sql_csv -c "SELECT pk, c_boolean FROM all_types WHERE pk=1;"
[ "$status" -eq 0 ]
[[ "$output" =~ "1,t" ]] || false
}
@test "types_compat: false boolean readable" {
run sql_csv -c "SELECT pk, c_boolean FROM all_types WHERE pk=2;"
[ "$status" -eq 0 ]
[[ "$output" =~ "2,f" ]] || false
}
# ---------------------------------------------------------------------------
# JSONB type
# ---------------------------------------------------------------------------
@test "types_compat: jsonb column readable" {
run sql_csv -c "SELECT pk, c_jsonb->'k' FROM all_types WHERE pk=1;"
[ "$status" -eq 0 ]
[[ "$output" =~ "v" ]] || false
}
@test "types_compat: jsonb array readable" {
skip "jsonb_array_length not implemented"
run sql_csv -c "SELECT pk, jsonb_array_length(c_jsonb) FROM all_types WHERE pk=2;"
[ "$status" -eq 0 ]
[[ "$output" =~ "3" ]] || false
}
# ---------------------------------------------------------------------------
# DML: insert and read back all column types
# ---------------------------------------------------------------------------
@test "types_compat: full-column-set insert round-trips correctly" {
skip "spurious error message for numeric type"
sql <<SQL
INSERT INTO all_types (pk, c_smallint, c_int, c_bigint,
c_real, c_double, c_numeric,
c_char, c_varchar, c_text, c_bytea,
c_date, c_time, c_timestamp, c_timestamptz,
c_boolean, c_jsonb)
VALUES (50, 55, 555555, 5555555555,
0.5, 0.25, 55.55,
'round', 'round trip', 'rt text', '\x526F756E64',
'2025-03-18', '09:00:00', '2025-03-18 09:00:00', '2025-03-18 09:00:00+00',
true, '{"round":true}');
SQL
run sql_csv -c "SELECT pk, c_smallint, c_int FROM all_types WHERE pk=50;"
[ "$status" -eq 0 ]
[[ "$output" =~ "50,55,555555" ]] || false
run sql_csv -c "SELECT pk, c_varchar, c_text FROM all_types WHERE pk=50;"
[ "$status" -eq 0 ]
[[ "$output" =~ "50,round trip,rt text" ]] || false
run sql_csv -c "SELECT pk, c_boolean FROM all_types WHERE pk=50;"
[ "$status" -eq 0 ]
[[ "$output" =~ "50,t" ]] || false
}
@test "types_compat: update all_types row written by older version" {
sql -c "UPDATE all_types SET c_varchar='updated', c_text='updated text' WHERE pk=1;"
run sql_csv -c "SELECT pk, c_varchar, c_text FROM all_types WHERE pk=1;"
[ "$status" -eq 0 ]
[[ "$output" =~ "1,updated,updated text" ]] || false
}
@test "types_compat: update text to large value" {
sql -c "UPDATE all_types SET c_text=repeat('u', 2000) WHERE pk=1;"
run sql_csv -c "SELECT pk, length(c_text) FROM all_types WHERE pk=1;"
[ "$status" -eq 0 ]
[[ "$output" =~ "1,2000" ]] || false
}
@test "types_compat: delete a row from old table" {
sql -c "DELETE FROM all_types WHERE pk=2;"
run sql_csv -c "SELECT count(*) FROM all_types;"
[ "$status" -eq 0 ]
[[ "$output" =~ "2" ]] || false
}
# ---------------------------------------------------------------------------
# ALTER TABLE: add columns of various types to a table written by older version
# ---------------------------------------------------------------------------
@test "types_compat: add text column to abc and use dml" {
sql -c "ALTER TABLE abc ADD COLUMN new_text TEXT;"
sql -c "UPDATE abc SET new_text='text for row';"
sql -c "INSERT INTO abc (pk, a, b, x, y, new_text) VALUES (99, 'test', 1.0, 0, 0, 'inserted');"
run sql_csv -c "SELECT pk, new_text FROM abc WHERE pk=99;"
[ "$status" -eq 0 ]
[[ "$output" =~ "99,inserted" ]] || false
sql -c "DELETE FROM abc WHERE pk=99;"
}
@test "types_compat: add integer columns to abc and use dml" {
sql -c "ALTER TABLE abc ADD COLUMN new_smallint SMALLINT, ADD COLUMN new_bigint BIGINT;"
sql -c "UPDATE abc SET new_smallint=42, new_bigint=9999999999;"
sql -c "INSERT INTO abc (pk, a, b, x, y, new_smallint, new_bigint) VALUES (99, 'test', 1.0, 0, 0, -1, -9999999999);"
run sql_csv -c "SELECT pk, new_smallint, new_bigint FROM abc WHERE pk=99;"
[ "$status" -eq 0 ]
[[ "$output" =~ "99,-1,-9999999999" ]] || false
sql -c "DELETE FROM abc WHERE pk=99;"
}
@test "types_compat: add numeric column to abc and use dml" {
sql -c "ALTER TABLE abc ADD COLUMN new_numeric NUMERIC(12,4);"
sql -c "UPDATE abc SET new_numeric=9876.5432;"
sql -c "INSERT INTO abc (pk, a, b, x, y, new_numeric) VALUES (99, 'test', 1.0, 0, 0, -9876.5432);"
run sql_csv -c "SELECT pk, new_numeric FROM abc WHERE pk=99;"
[ "$status" -eq 0 ]
[[ "$output" =~ "99,-9876.5432" ]] || false
sql -c "DELETE FROM abc WHERE pk=99;"
}
@test "types_compat: add date/timestamp columns to abc and use dml" {
sql -c "ALTER TABLE abc ADD COLUMN new_date DATE, ADD COLUMN new_ts TIMESTAMP;"
sql -c "UPDATE abc SET new_date='2025-03-18', new_ts='2025-03-18 10:00:00';"
sql -c "INSERT INTO abc (pk, a, b, x, y, new_date, new_ts) VALUES (99, 'test', 1.0, 0, 0, '2025-06-01', '2025-06-01 12:00:00');"
run sql_csv -c "SELECT pk, new_date FROM abc WHERE pk=99;"
[ "$status" -eq 0 ]
[[ "$output" =~ "99,2025-06-01" ]] || false
sql -c "DELETE FROM abc WHERE pk=99;"
}
@test "types_compat: add boolean column to abc and use dml" {
sql -c "ALTER TABLE abc ADD COLUMN new_bool BOOLEAN;"
sql -c "UPDATE abc SET new_bool=true;"
sql -c "INSERT INTO abc (pk, a, b, x, y, new_bool) VALUES (99, 'test', 1.0, 0, 0, false);"
run sql_csv -c "SELECT pk, new_bool FROM abc WHERE pk=99;"
[ "$status" -eq 0 ]
[[ "$output" =~ "99,f" ]] || false
sql -c "DELETE FROM abc WHERE pk=99;"
}
@test "types_compat: add jsonb column to abc and use dml" {
sql -c "ALTER TABLE abc ADD COLUMN new_jsonb JSONB;"
sql -c "UPDATE abc SET new_jsonb='{\"updated\":true}';"
sql -c "INSERT INTO abc (pk, a, b, x, y, new_jsonb) VALUES (99, 'test', 1.0, 0, 0, '{\"inserted\":1}');"
run sql_csv -c "SELECT pk, new_jsonb->>'inserted' FROM abc WHERE pk=99;"
[ "$status" -eq 0 ]
[[ "$output" =~ "1" ]] || false
sql -c "DELETE FROM abc WHERE pk=99;"
}
@test "types_compat: add bytea column to abc and use dml" {
skip "encode not implemented"
sql -c "ALTER TABLE abc ADD COLUMN new_bytea BYTEA;"
sql -c "UPDATE abc SET new_bytea='\xDEAD';"
sql -c "INSERT INTO abc (pk, a, b, x, y, new_bytea) VALUES (99, 'test', 1.0, 0, 0, '\xBEEF');"
run sql_csv -c "SELECT pk, encode(new_bytea, 'hex') FROM abc WHERE pk=99;"
[ "$status" -eq 0 ]
[[ "$output" =~ "beef" ]] || false
sql -c "DELETE FROM abc WHERE pk=99;"
}
# ---------------------------------------------------------------------------
# Schema inspection
# ---------------------------------------------------------------------------
@test "types_compat: all_types column types visible in information_schema" {
run sql -c "SELECT column_name, data_type FROM information_schema.columns WHERE table_name='all_types' ORDER BY ordinal_position;"
[ "$status" -eq 0 ]
[[ "$output" =~ "smallint" ]] || false
[[ "$output" =~ "integer" ]] || false
[[ "$output" =~ "bigint" ]] || false
[[ "$output" =~ "real" ]] || false
[[ "$output" =~ "double precision" ]] || false
[[ "$output" =~ "numeric" ]] || false
[[ "$output" =~ "character" ]] || false
[[ "$output" =~ "text" ]] || false
[[ "$output" =~ "bytea" ]] || false
[[ "$output" =~ "date" ]] || false
[[ "$output" =~ "timestamp" ]] || false
[[ "$output" =~ "boolean" ]] || false
[[ "$output" =~ "jsonb" ]] || false
}
# ---------------------------------------------------------------------------
# Commit works after DML on old table
# ---------------------------------------------------------------------------
@test "types_compat: commit works after dml on all_types" {
sql -c "INSERT INTO all_types (pk, c_text) VALUES (98, 'commit test');"
sql -c "SELECT dolt_add('.');"
run sql -c "SELECT length(dolt_commit('-m', 'types compat commit')::text);"
[ "$status" -eq 0 ]
[[ "$output" =~ "34" ]] || false
}
# ---------------------------------------------------------------------------
# View over all_types
# ---------------------------------------------------------------------------
@test "types_compat: all_types_view returns same rows as underlying table" {
run sql_csv -c "SELECT pk, c_smallint, c_varchar FROM all_types_view WHERE pk=1;"
[ "$status" -eq 0 ]
[[ "$output" =~ "1,100,hello world" ]] || false
}
@test "types_compat: all_types_view supports filtering" {
run sql_csv -c "SELECT count(*) FROM all_types_view WHERE pk < 3;"
[ "$status" -eq 0 ]
[[ "$output" =~ "2" ]] || false
}
integration-tests/compatibility/test_files/forward_compatible_versions.txt
+6
View File
@@ -0,0 +1,6 @@
# Doltgresql versions to test forward compatibility against.
# These older versions must be able to read repos created by the current HEAD.
# Typically a smaller set than backward_compatible_versions because forward
# compatibility windows are narrower — old clients may lack support for newer
# storage features written by HEAD.
v0.56.5
integration-tests/compatibility/test_files/setup_repo.sh
Executable
+247
View File
@@ -0,0 +1,247 @@
#!/bin/bash
# Creates a doltgresql repository populated with test data for compatibility testing.
# Usage: setup_repo.sh <datadir> [<doltgres-binary>]
#
# Starts a temporary doltgres server, creates tables / views / branches expected
# by compatibility.bats and types_compatibility.bats, then stops the server.
# The resulting data directory is the "repo" passed to BATS tests via REPO_DIR.
set -eo pipefail
DATADIR="$1"
DOLTGRES_BIN="${2:-doltgres}"
if [ -z "$DATADIR" ]; then
echo "Usage: setup_repo.sh <datadir> [<doltgres-binary>]" >&2
exit 1
fi
mkdir -p "$DATADIR"
# ---------------------------------------------------------------------------
# Server helpers
# ---------------------------------------------------------------------------
pick_port() {
for i in {0..99}; do
local port=$((RANDOM % 4096 + 2048))
if ! nc -z localhost "$port" 2>/dev/null; then
echo "$port"
return 0
fi
done
echo "ERROR: could not find a free port" >&2
return 1
}
PORT=$(pick_port)
CONFIGFILE="$DATADIR/setup-config.yaml"
cat > "$CONFIGFILE" <<EOF
log_level: warning
behavior:
read_only: false
disable_client_multi_statements: false
listener:
host: localhost
port: $PORT
EOF
"$DOLTGRES_BIN" -data-dir="$DATADIR" --config="$CONFIGFILE" \
> "$DATADIR/setup.log" 2>&1 &
SERVER_PID=$!
trap 'kill "$SERVER_PID" 2>/dev/null; wait "$SERVER_PID" 2>/dev/null; exit' EXIT INT TERM
echo "Waiting for doltgres server on port $PORT ..."
end=$((SECONDS + 20))
while [ $SECONDS -lt $end ]; do
if PGPASSWORD=password psql -U postgres -h localhost -p "$PORT" \
-c "SELECT 1;" postgres >/dev/null 2>&1; then
echo "Server ready."
break
fi
sleep 0.5
done
if ! PGPASSWORD=password psql -U postgres -h localhost -p "$PORT" \
-c "SELECT 1;" postgres >/dev/null 2>&1; then
echo "ERROR: server failed to start. Log:" >&2
cat "$DATADIR/setup.log" >&2
exit 1
fi
# Q — run SQL against the server (heredoc-friendly, single session).
Q() {
PGPASSWORD=password psql -U postgres -h localhost -p "$PORT" \
-v ON_ERROR_STOP=1 "$@" postgres
}
# ---------------------------------------------------------------------------
# Step 1: Create initial schema and data on main, then branch from there.
#
# Branch layout:
# init — snapshot of initial data (abc w/o alterations)
# other — diverges from init; drops x, adds z to abc
# check_merge — diverges from main after its alterations; adds rows to def
# main — drops w, adds y to abc; clean-merges check_merge
# ---------------------------------------------------------------------------
echo "Step 1: creating initial schema on main ..."
Q <<'SQL'
CREATE TABLE abc (
pk BIGINT NOT NULL,
a TEXT,
b DOUBLE PRECISION,
w BIGINT,
x BIGINT,
PRIMARY KEY (pk)
);
INSERT INTO abc VALUES
(0, 'asdf', 1.1, 0, 0),
(1, 'asdf', 1.1, 0, 0),
(2, 'asdf', 1.1, 0, 0);
CREATE VIEW view1 AS SELECT 2+2;
CREATE TABLE def (
i INT CHECK (i > 0)
);
INSERT INTO def VALUES (1), (2), (3);
CREATE TABLE all_types (
pk INT NOT NULL PRIMARY KEY,
c_smallint SMALLINT,
c_int INT,
c_bigint BIGINT,
c_real REAL,
c_double DOUBLE PRECISION,
c_numeric NUMERIC(10,2),
c_char CHAR(10),
c_varchar VARCHAR(255),
c_text TEXT,
c_bytea BYTEA,
c_date DATE,
c_time TIME,
c_timestamp TIMESTAMP,
c_timestamptz TIMESTAMPTZ,
c_boolean BOOLEAN,
c_jsonb JSONB
);
INSERT INTO all_types VALUES (
1,
100, 2000000, 9223372036854775807,
1.5, 2.5, 12345.67,
'hello', 'hello world', 'text val', E'\\xDEADBEEF',
'2024-01-15', '13:30:45', '2024-01-15 13:30:45', '2024-01-15 13:30:45+00',
true, '{"k":"v"}'
);
INSERT INTO all_types VALUES (
2,
-100, -2000000, -9223372036854775807,
-1.5, -2.5, -12345.67,
'hi', 'hi there', 'text val2', E'\\xC0FFEE',
'2023-12-31', '23:59:59', '2023-12-31 23:59:59', NULL,
false, '[1,2,3]'
);
INSERT INTO all_types (pk, c_text) VALUES (3, repeat('t', 500));
CREATE VIEW all_types_view AS SELECT * FROM all_types;
SQL
Q -c "SELECT dolt_add('.');"
Q -c "SELECT dolt_commit('-m', 'initialized data');"
Q <<'EOF'
CREATE TABLE big (
pk INT PRIMARY KEY,
str TEXT
);
EOF
for i in $(seq 0 9); do
start=$((i * 100 + 1))
end=$((start + 99))
stmt="INSERT INTO big VALUES "
for j in $(seq $start $end); do
stmt+=$(printf "(%d, 'row %d')" $j $j)
[ $j -lt $end ] && stmt+=", "
done
stmt+=";"
Q -c "$stmt"
done
# ---------------------------------------------------------------------------
# Step 2: Branch 'init' and 'other' from this initial-data commit.
# ---------------------------------------------------------------------------
echo "Step 2: branching init and other from initial data commit ..."
Q -c "SELECT dolt_branch('init');"
Q -c "SELECT dolt_branch('other');"
# ---------------------------------------------------------------------------
# Step 3: Advance main — alter abc (drop w, add y).
# ---------------------------------------------------------------------------
echo "Step 3: advancing main with abc alterations ..."
Q <<'SQL'
DELETE FROM abc WHERE pk=1;
UPDATE abc SET x = 1 WHERE pk = 0;
INSERT INTO abc VALUES (3, 'data', 1.1, 0, 0);
ALTER TABLE abc DROP COLUMN w;
ALTER TABLE abc ADD COLUMN y BIGINT;
UPDATE abc SET y = 121;
SQL
Q -c "SELECT dolt_add('.');"
Q -c "SELECT dolt_commit('-m', 'made changes to main');"
# Branch 'check_merge' from main's current commit (has abc alterations).
Q -c "SELECT dolt_branch('check_merge');"
# ---------------------------------------------------------------------------
# Step 4: Populate 'other' — different alterations to abc.
# ---------------------------------------------------------------------------
echo "Step 4: populating other branch ..."
Q <<'SQL'
SELECT dolt_checkout('other');
DELETE FROM abc WHERE pk=2;
UPDATE abc SET w = 1 WHERE pk = 0;
INSERT INTO abc VALUES (4, 'data', 1.1, 0, 0);
ALTER TABLE abc DROP COLUMN x;
ALTER TABLE abc ADD COLUMN z BIGINT;
UPDATE abc SET z = 122;
SELECT dolt_add('.');
SELECT dolt_commit('-m', 'made changes to other');
SQL
# ---------------------------------------------------------------------------
# Step 5: Populate 'check_merge' — add rows to def only.
# ---------------------------------------------------------------------------
echo "Step 5: populating check_merge branch ..."
Q <<'SQL'
SELECT dolt_checkout('check_merge');
INSERT INTO def VALUES (5), (6), (7);
SELECT dolt_add('.');
SELECT dolt_commit('-m', 'made changes to check_merge');
SQL
# ---------------------------------------------------------------------------
# Done
# ---------------------------------------------------------------------------
echo ""
echo "Repository setup complete."
echo " Data dir : $DATADIR"
echo " Branches : main, init, other, check_merge"
kill "$SERVER_PID"
wait "$SERVER_PID" 2>/dev/null || true
trap - EXIT INT TERM
integration-tests/go-sql-server-driver/concurrent_gc_test.go
+334
View File
@@ -0,0 +1,334 @@
// Copyright 2023 Dolthub, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package main
import (
"context"
"database/sql"
sqldriver "database/sql/driver"
"fmt"
"strings"
"testing"
"time"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"golang.org/x/sync/errgroup"
driver "github.com/dolthub/doltgresql/integration-tests/go-sql-server-driver/driver"
)
func TestConcurrentGC(t *testing.T) {
t.Skip("Doltgres does not yet handle dolt_gc() concurrently with active write connections the way this test expects: writers fail with 'unexpected EOF' / connection resets rather than the retriable safepoint errors Dolt produces (behavioral difference)")
t.Parallel()
type dimension struct {
names []string
factors func(gcTest) []gcTest
}
commits := dimension{
names: []string{"NoCommits", "WithCommits"},
factors: func(base gcTest) []gcTest {
no, yes := base, base
no.commit = false
yes.commit = true
return []gcTest{no, yes}
},
}
full := dimension{
names: []string{"NotFull", "Full"},
factors: func(base gcTest) []gcTest {
no, yes := base, base
no.full = false
yes.full = true
return []gcTest{no, yes}
},
}
safepoint := dimension{
names: []string{"KillConnections", "SessionAware"},
factors: func(base gcTest) []gcTest {
no, yes := base, base
no.sessionAware = false
yes.sessionAware = true
return []gcTest{no, yes}
},
}
var doDimensions func(t *testing.T, base gcTest, dims []dimension)
doDimensions = func(t *testing.T, base gcTest, dims []dimension) {
if len(dims) == 0 {
base.run(t)
return
}
dim, dims := dims[0], dims[1:]
dimf := dim.factors(base)
for i := range dim.names {
t.Run(dim.names[i], func(t *testing.T) {
t.Parallel()
doDimensions(t, dimf[i], dims)
})
}
}
dimensions := []dimension{commits, full, safepoint}
doDimensions(t, gcTest{
numThreads: 8,
duration: 10 * time.Second,
}, dimensions)
}
type gcTest struct {
numThreads int
duration time.Duration
commit bool
full bool
sessionAware bool
}
func (gct gcTest) createDB(t *testing.T, ctx context.Context, db *sql.DB) {
conn, err := db.Conn(ctx)
require.NoError(t, err)
defer conn.Close()
// We're going to bootstrap the database with a table which has id, val, id == [0,7*1024], val == 0.
_, err = conn.ExecContext(ctx, "create table vals (id int primary key, val int)")
require.NoError(t, err)
vals := []string{}
for i := 0; i <= (gct.numThreads-1)*1024; i++ {
vals = append(vals, fmt.Sprintf("(%d,0)", i))
}
_, err = conn.ExecContext(ctx, "insert into vals values "+strings.Join(vals, ","))
require.NoError(t, err)
_, err = conn.ExecContext(ctx, "select dolt_commit('-Am', 'create vals table')")
require.NoError(t, err)
}
// When running with kill_connections GC safepoints, asserts that the
// error we got is not an error that was not allowed.
func assertExpectedGCError(t *testing.T, err error) bool {
if !assert.NotContains(t, err.Error(), "dangling ref") {
return false
}
if !assert.NotContains(t, err.Error(), "is unexpected noms value") {
return false
}
if !assert.NotContains(t, err.Error(), "interface conversion: types.Value is nil") {
return false
}
return true
}
func (gct gcTest) doUpdate(t *testing.T, ctx context.Context, db *sql.DB, i int) error {
conn, err := db.Conn(ctx)
if gct.sessionAware {
if !assert.NoError(t, err) {
return nil
}
} else if err != nil {
if !assert.NotContains(t, err.Error(), "connection refused") {
return err
}
t.Logf("err in Conn: %v", err)
return nil
}
defer conn.Close()
tx, err := conn.BeginTx(ctx, nil)
if gct.sessionAware {
assert.NoError(t, err)
}
if err != nil {
// Ignore and try with a different connection...
return nil
}
defer tx.Rollback()
_, err = tx.ExecContext(ctx, "update vals set val = val+1 where id = $1", i)
if gct.sessionAware {
assert.NoError(t, err)
} else if err != nil {
if !assertExpectedGCError(t, err) {
return err
}
t.Logf("err in Exec update: %v", err)
}
if err != nil {
// Early return so we do not try to continue using the
// broken connection or attempt to commit something
// with no changes.
return nil
}
if gct.commit {
_, err = tx.ExecContext(ctx, fmt.Sprintf("select dolt_commit('-am', 'increment vals id = %d')", i))
if gct.sessionAware {
assert.NoError(t, err)
} else if err != nil {
if !assertExpectedGCError(t, err) {
return err
}
t.Logf("err in Exec select dolt_commit: %v", err)
}
} else {
err = tx.Commit()
if gct.sessionAware {
assert.NoError(t, err)
} else if err != nil {
if !assertExpectedGCError(t, err) {
return err
}
t.Logf("err in tx commit: %v", err)
}
}
return nil
}
func (gct gcTest) doGC(t *testing.T, ctx context.Context, db *sql.DB) error {
conn, err := db.Conn(ctx)
if gct.sessionAware {
if !assert.NoError(t, err) {
return nil
}
} else if err != nil {
if !assert.NotContains(t, err.Error(), "connection refused") {
return err
}
t.Logf("err in Conn for dolt_gc: %v", err)
return nil
}
if !gct.sessionAware {
defer func() {
// After calling dolt_gc, the connection is bad. Remove it from the connection pool.
conn.Raw(func(_ any) error {
return sqldriver.ErrBadConn
})
}()
} else {
defer conn.Close()
}
b := time.Now()
if !gct.full {
_, err = conn.ExecContext(ctx, "select dolt_gc()")
} else {
_, err = conn.ExecContext(ctx, `select dolt_gc('--full')`)
}
if assert.NoError(t, err) {
t.Logf("successful dolt_gc took %v", time.Since(b))
}
return nil
}
func (gct gcTest) finalize(t *testing.T, ctx context.Context, db *sql.DB) {
conn, err := db.Conn(ctx)
require.NoError(t, err)
var ids []any
var qmarks []string
n := 1
for i := 0; i < gct.numThreads*1024; i += 1024 {
ids = append(ids, i)
qmarks = append(qmarks, fmt.Sprintf("$%d", n))
n++
}
rows, err := conn.QueryContext(ctx, fmt.Sprintf("select val from vals where id in (%s)", strings.Join(qmarks, ",")), ids...)
require.NoError(t, err)
i := 0
cnt := 0
for rows.Next() {
var val int
i += 1
require.NoError(t, rows.Scan(&val))
cnt += val
}
require.Equal(t, len(ids), i)
t.Logf("successfully updated val %d times", cnt)
require.NoError(t, rows.Close())
require.NoError(t, rows.Err())
rows, err = conn.QueryContext(ctx, "select count(*) from dolt_log")
require.NoError(t, err)
require.True(t, rows.Next())
require.NoError(t, rows.Scan(&cnt))
t.Logf("database has %d commit(s)", cnt)
require.False(t, rows.Next())
require.NoError(t, rows.Close())
require.NoError(t, rows.Err())
require.NoError(t, conn.Close())
}
func (gct gcTest) run(t *testing.T) {
ports := newPorts(t)
u, err := driver.NewDoltUser()
require.NoError(t, err)
t.Cleanup(func() {
u.Cleanup()
})
rs, err := u.MakeRepoStore()
require.NoError(t, err)
_, err = rs.MakeRepo("concurrent_gc_test")
require.NoError(t, err)
srvSettings := &driver.Server{
Args: []string{},
DynamicPort: "server_port",
}
if gct.sessionAware {
srvSettings.Envs = append(srvSettings.Envs, "DOLT_GC_SAFEPOINT_CONTROLLER_CHOICE=session_aware")
} else {
srvSettings.Envs = append(srvSettings.Envs, "DOLT_GC_SAFEPOINT_CONTROLLER_CHOICE=kill_connections")
}
server := StartServer(t, rs, "concurrent_gc_test", srvSettings, ports)
db, err := server.DB(driver.Connection{})
require.NoError(t, err)
defer db.Close()
gct.createDB(t, context.Background(), db)
start := time.Now()
eg, egCtx := errgroup.WithContext(context.Background())
// We're going to spawn 8 threads, each running mutations on their own part of the table...
for i := 0; i < gct.numThreads; i++ {
i := i * 1024
eg.Go(func() error {
for j := 0; time.Since(start) < gct.duration && egCtx.Err() == nil; j++ {
if err := gct.doUpdate(t, egCtx, db, i); err != nil {
return err
}
}
return nil
})
}
// We spawn a thread which calls dolt_gc() periodically
eg.Go(func() error {
for time.Since(start) < gct.duration && egCtx.Err() == nil {
if err := gct.doGC(t, egCtx, db); err != nil {
return err
}
time.Sleep(100 * time.Millisecond)
}
return nil
})
require.NoError(t, eg.Wait())
// Recreate the connection pool here, since idle connections in the
// connection pool may be stale.
db.Close()
db, err = server.DB(driver.Connection{})
require.NoError(t, err)
gct.finalize(t, context.Background(), db)
db.Close()
}
integration-tests/go-sql-server-driver/concurrent_writes_test.go
+148
View File
@@ -0,0 +1,148 @@
// Copyright 2024 Dolthub, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package main
import (
"fmt"
"strings"
"sync/atomic"
"testing"
"time"
"github.com/stretchr/testify/require"
"golang.org/x/sync/errgroup"
driver "github.com/dolthub/doltgresql/integration-tests/go-sql-server-driver/driver"
)
// TestConcurrentWrites verifies concurrent write behavior and transaction locking in the SQL server driver.
func TestConcurrentWrites(t *testing.T) {
t.Parallel()
ports := newPorts(t)
u, err := driver.NewDoltUser()
require.NoError(t, err)
t.Cleanup(func() {
u.Cleanup()
})
rs, err := u.MakeRepoStore()
require.NoError(t, err)
_, err = rs.MakeRepo("concurrent_writes_test")
require.NoError(t, err)
srvSettings := &driver.Server{
Args: []string{},
DynamicPort: "server_port",
}
server := StartServer(t, rs, "concurrent_writes_test", srvSettings, ports)
db, err := server.DB(driver.Connection{})
require.NoError(t, err)
db.SetMaxIdleConns(0)
defer func() {
require.NoError(t, db.Close())
}()
ctx := t.Context()
func() {
conn, err := db.Conn(ctx)
require.NoError(t, err)
defer conn.Close()
// Create table and initial data.
_, err = conn.ExecContext(ctx, "CREATE TABLE data (id VARCHAR(64) PRIMARY KEY, worker INT, data TEXT, created_at TIMESTAMP)")
require.NoError(t, err)
_, err = conn.ExecContext(ctx, "SELECT DOLT_COMMIT('-Am', 'init with table')")
require.NoError(t, err)
}()
eg, ctx := errgroup.WithContext(ctx)
nextInt := uint32(0)
const numWriters = 32
startCh := make(chan struct{})
readyCh := make(chan struct{})
for i := range numWriters {
eg.Go(func() error {
db, err := server.DB(driver.Connection{})
if err != nil {
return err
}
defer db.Close()
db.SetMaxOpenConns(1)
conn, err := db.Conn(ctx)
if err != nil {
return err
}
defer conn.Close()
select {
case readyCh <- struct{}{}:
case <-ctx.Done():
return nil
}
select {
case <-startCh:
case <-ctx.Done():
return nil
}
// Each thread writes 16.
for j := range 16 {
if ctx.Err() != nil {
return nil
}
key := fmt.Sprintf("main-%d-%d", i, j)
_, err := conn.ExecContext(ctx, "INSERT INTO data VALUES ($1,$2,$3,$4)", key, i, key, time.Now())
if err != nil {
return err
}
atomic.AddUint32(&nextInt, 1)
_, err = conn.ExecContext(ctx, fmt.Sprintf("SELECT DOLT_COMMIT('-Am', 'insert %s')", key))
if err != nil && !strings.Contains(err.Error(), "nothing to commit") {
// Technically we can get "nothing to commit" because of how DOLT_COMMIT works.
// It first commits this transaction to the working set and then commits the
// merged working set as a dolt commit.
//
// If we ever fix DOLT_COMMIT, we should fix this exception here as well.
return err
}
}
return nil
})
}
for range numWriters {
select {
case <-readyCh:
case <-ctx.Done():
// This will fail.
require.NoError(t, eg.Wait())
t.FailNow()
}
}
close(startCh)
require.NoError(t, eg.Wait())
require.Equal(t, 512, int(nextInt))
t.Logf("wrote %d", nextInt)
ctx = t.Context()
conn, err := db.Conn(ctx)
require.NoError(t, err)
defer func() {
require.NoError(t, conn.Close())
}()
var i int
err = conn.QueryRowContext(ctx, "SELECT COUNT(*) FROM data").Scan(&i)
require.NoError(t, err)
require.Equal(t, int(nextInt), i)
err = conn.QueryRowContext(ctx, "SELECT COUNT(*) FROM dolt_log").Scan(&i)
require.NoError(t, err)
t.Logf("ended with %d commits", i)
}
integration-tests/go-sql-server-driver/driver/cmd.go
+579
View File
@@ -0,0 +1,579 @@
// Copyright 2024 Dolthub, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package driver
import (
"bufio"
"bytes"
"context"
"database/sql"
"database/sql/driver"
"fmt"
"io"
"log"
"net"
"net/url"
"os"
"os/exec"
"path/filepath"
"time"
"github.com/jackc/pgx/v5"
"github.com/jackc/pgx/v5/stdlib"
)
var DoltgresPath string
var DelvePath string
const TestUserName = "Bats Tests"
const TestEmailAddress = "bats@email.fake"
const ConnectAttempts = 50
const RetrySleepDuration = 50 * time.Millisecond
// EnvDoltgresBinPath is the environment variable used to locate the doltgres
// binary used by these tests. If unset, "doltgres" on the PATH is used.
const EnvDoltgresBinPath = "DOLTGRES_BIN_PATH"
func init() {
path := os.Getenv(EnvDoltgresBinPath)
if path == "" {
path = "doltgres"
}
path = filepath.Clean(path)
var err error
DoltgresPath, err = exec.LookPath(path)
if err != nil {
log.Printf("did not find doltgres binary: %v\n", err.Error())
}
DelvePath, _ = exec.LookPath("dlv")
}
// DoltUser is an abstraction for a user account that calls the `doltgres`
// server. All of our doltgres binary invocations are done through DoltUser.
//
// For our purposes, it does the following:
//
// - owns a tmpdir, to which it sets DOLT_ROOT_PATH when invoking doltgres.
//
// - writes some initial dolt global config (user.name, user.email,
// metrics.disabled = true) into that root path. Doltgres has no `config`
// CLI command, so unlike Dolt we write the config file directly.
//
// - can create repo stores, which will be a tmpdir to store a data-dir
// containing one or more databases.
type DoltUser struct {
tmpdir string
}
var _ DoltCmdable = DoltUser{}
var _ DoltDebuggable = DoltUser{}
func NewDoltUser() (DoltUser, error) {
tmpdir, err := os.MkdirTemp("", "go-sql-server-driver-")
if err != nil {
return DoltUser{}, err
}
res := DoltUser{tmpdir}
// Doltgres has no `config --global` CLI command, so we write the global
// config file directly. Doltgres uses Dolt's env loading, which reads the
// global config from $DOLT_ROOT_PATH/.dolt/config_global.json.
cfgDir := filepath.Join(tmpdir, ".dolt")
if err := os.MkdirAll(cfgDir, 0750); err != nil {
return DoltUser{}, err
}
contents := fmt.Sprintf(`{"metrics.disabled":"true","user.name":%q,"user.email":%q}`+"\n", TestUserName, TestEmailAddress)
if err := os.WriteFile(filepath.Join(cfgDir, "config_global.json"), []byte(contents), 0640); err != nil {
return DoltUser{}, err
}
return res, nil
}
func (u DoltUser) DoltCmd(args ...string) *exec.Cmd {
cmd := exec.Command(DoltgresPath, args...)
cmd.Dir = u.tmpdir
cmd.Env = append(os.Environ(), "DOLT_ROOT_PATH="+u.tmpdir)
ApplyCmdAttributes(cmd)
return cmd
}
func (u DoltUser) DoltDebug(debuggerPort int, args ...string) *exec.Cmd {
if DelvePath != "" {
dlvArgs := []string{
fmt.Sprintf("--listen=:%d", debuggerPort),
"--headless",
"--api-version=2",
"--accept-multiclient",
"exec",
DoltgresPath,
"--",
}
cmd := exec.Command(DelvePath, append(dlvArgs, args...)...)
cmd.Dir = u.tmpdir
cmd.Env = append(os.Environ(), "DOLT_ROOT_PATH="+u.tmpdir)
ApplyCmdAttributes(cmd)
return cmd
} else {
panic("dlv not found")
}
}
func (u DoltUser) MakeRepoStore() (RepoStore, error) {
tmpdir, err := os.MkdirTemp(u.tmpdir, "repo-store-")
if err != nil {
return RepoStore{}, err
}
return RepoStore{u, tmpdir}, nil
}
func (u DoltUser) Cleanup() error {
return os.RemoveAll(u.tmpdir)
}
type RepoStore struct {
user DoltUser
Dir string
}
var _ DoltCmdable = RepoStore{}
var _ DoltDebuggable = RepoStore{}
// MakeRepo creates a new doltgres database named |name| within this store's
// data-dir. Because doltgres has no `init` CLI command, the database is
// initialized by briefly running a doltgres server with DOLTGRES_DB set to the
// database name and waiting for the on-disk database to be created.
func (rs RepoStore) MakeRepo(name string) (Repo, error) {
ret := Repo{rs.user, rs, filepath.Join(rs.Dir, name), name}
err := rs.initDatabase(name, nil)
if err != nil {
return Repo{}, err
}
return ret, nil
}
func (rs RepoStore) DoltCmd(args ...string) *exec.Cmd {
cmd := rs.user.DoltCmd(args...)
cmd.Dir = rs.Dir
return cmd
}
func (rs RepoStore) DoltDebug(debuggerPort int, args ...string) *exec.Cmd {
cmd := rs.user.DoltDebug(debuggerPort, args...)
cmd.Dir = rs.Dir
return cmd
}
// initDatabase starts a short-lived doltgres server bound to a free port with
// the data-dir set to this store, creates the database |name| if it does not
// already exist, runs the optional |fn| against a connection to that database,
// and then shuts the server down. This is the doltgres equivalent of
// `dolt init` plus any pre-server setup such as adding remotes.
func (rs RepoStore) initDatabase(name string, fn func(db *sql.DB) error) error {
port, err := freePort()
if err != nil {
return err
}
cfgPath := filepath.Join(rs.Dir, fmt.Sprintf(".init-%s-config.yaml", sanitize(name)))
// The unix socket path must stay well under the OS limit (~108 chars), so
// it is placed directly in the temp dir keyed by the (unique) port rather
// than inside the potentially-long data-dir path.
sockPath := filepath.Join(os.TempDir(), fmt.Sprintf("dg-init-%d.sock", port))
cfgContents := fmt.Sprintf("log_level: warn\nlistener:\n host: 127.0.0.1\n port: %d\n socket: %s\n", port, sockPath)
if err := os.WriteFile(cfgPath, []byte(cfgContents), 0640); err != nil {
return err
}
defer os.Remove(cfgPath)
cmd := rs.DoltCmd("--data-dir=.", "--config="+cfgPath)
cmd.Env = append(cmd.Env, "DOLTGRES_DB="+name)
output := new(bytes.Buffer)
cmd.Stdout = output
cmd.Stderr = output
if err := cmd.Start(); err != nil {
return err
}
defer func() {
_ = cmd.Process.Signal(os.Interrupt)
_, _ = cmd.Process.Wait()
}()
db, err := ConnectDB("postgres", "password", name, "127.0.0.1", port, nil)
if err != nil {
return fmt.Errorf("could not connect to init server for %s: %w (output: %s)", name, err, output.String())
}
defer db.Close()
if _, err := db.Exec(fmt.Sprintf(`CREATE DATABASE IF NOT EXISTS %s`, name)); err != nil {
return err
}
if fn != nil {
if err := fn(db); err != nil {
return err
}
}
return nil
}
func sanitize(s string) string {
out := make([]rune, 0, len(s))
for _, r := range s {
if r == '/' || r == '\\' || r == ' ' {
out = append(out, '_')
} else {
out = append(out, r)
}
}
return string(out)
}
// freePort asks the OS for an available TCP port on the loopback interface.
func freePort() (int, error) {
l, err := net.Listen("tcp", "127.0.0.1:0")
if err != nil {
return 0, err
}
defer l.Close()
return l.Addr().(*net.TCPAddr).Port, nil
}
type Repo struct {
user DoltUser
rs RepoStore
Dir string
Name string
}
func (r Repo) DoltCmd(args ...string) *exec.Cmd {
cmd := r.user.DoltCmd(args...)
cmd.Dir = r.Dir
return cmd
}
// CreateRemote adds a remote named |name| with the given |url| to this
// database. Doltgres has no `remote` CLI command, so this is done by briefly
// running a server and calling the dolt_remote() stored function.
func (r Repo) CreateRemote(name, url string) error {
return r.rs.initDatabase(r.Name, func(db *sql.DB) error {
_, err := db.Exec(`SELECT dolt_remote('add', $1, $2)`, name, url)
return err
})
}
type SqlServer struct {
Name string
Done chan struct{}
Cmd *exec.Cmd
CmdWaitErr error
Port int
DebugPort int
Output *bytes.Buffer
DBName string
RecreateCmd func(args ...string) *exec.Cmd
// Where to write server log output for display. If nil,
// defaults to os.Stdout.
LogWriter io.Writer
// If non-nil, called with each complete line of server output.
OutputVisitor func(string)
}
type SqlServerOpt func(s *SqlServer)
func WithArgs(args ...string) SqlServerOpt {
return func(s *SqlServer) {
s.Cmd.Args = append(s.Cmd.Args, args...)
}
}
func WithName(name string) SqlServerOpt {
return func(s *SqlServer) {
s.Name = name
}
}
func WithEnvs(envs ...string) SqlServerOpt {
return func(s *SqlServer) {
s.Cmd.Env = append(s.Cmd.Env, envs...)
}
}
func WithPort(port int) SqlServerOpt {
return func(s *SqlServer) {
s.Port = port
}
}
func WithOutputVisitor(f func(string)) SqlServerOpt {
return func(s *SqlServer) {
s.OutputVisitor = f
}
}
func WithLogWriter(w io.Writer) SqlServerOpt {
return func(s *SqlServer) {
s.LogWriter = w
}
}
func WithDebugPort(port int) SqlServerOpt {
return func(s *SqlServer) {
s.DebugPort = port
}
}
type DoltCmdable interface {
DoltCmd(args ...string) *exec.Cmd
}
type DoltDebuggable interface {
DoltDebug(debuggerPort int, args ...string) *exec.Cmd
}
// StartSqlServer starts a doltgres server. Unlike Dolt, doltgres has no
// `sql-server` subcommand; running the binary itself starts the server.
func StartSqlServer(dc DoltCmdable, opts ...SqlServerOpt) (*SqlServer, error) {
cmd := dc.DoltCmd()
return runSqlServerCommand(dc, opts, cmd)
}
func DebugSqlServer(dc DoltCmdable, debuggerPort int, opts ...SqlServerOpt) (*SqlServer, error) {
ddb, ok := dc.(DoltDebuggable)
if !ok {
return nil, fmt.Errorf("%T does not implement DoltDebuggable", dc)
}
cmd := ddb.DoltDebug(debuggerPort)
return runSqlServerCommand(dc, append(opts, WithDebugPort(debuggerPort)), cmd)
}
func runSqlServerCommand(dc DoltCmdable, opts []SqlServerOpt, cmd *exec.Cmd) (*SqlServer, error) {
stdout, err := cmd.StdoutPipe()
if err != nil {
return nil, err
}
cmd.Stderr = cmd.Stdout
output := new(bytes.Buffer)
done := make(chan struct{})
server := &SqlServer{
Done: done,
Cmd: cmd,
Port: 5432,
Output: output,
}
for _, o := range opts {
o(server)
}
go func() {
defer func() {
server.CmdWaitErr = server.Cmd.Wait()
close(done)
}()
logw := server.LogWriter
if logw == nil {
logw = os.Stdout
}
multiCopyWithNamePrefix(logw, output, stdout, server.Name, server.OutputVisitor)
}()
server.RecreateCmd = func(args ...string) *exec.Cmd {
if server.DebugPort > 0 {
ddb, ok := dc.(DoltDebuggable)
if !ok {
panic(fmt.Sprintf("%T does not implement DoltDebuggable", dc))
}
return ddb.DoltDebug(server.DebugPort, args...)
} else {
return dc.DoltCmd(args...)
}
}
err = server.Cmd.Start()
if err != nil {
return nil, err
}
return server, nil
}
func (s *SqlServer) ErrorStop() error {
<-s.Done
return s.CmdWaitErr
}
func multiCopyWithNamePrefix(stdout, captured io.Writer, in io.Reader, name string, visitor func(string)) {
reader := bufio.NewReader(in)
multiOut := io.MultiWriter(stdout, captured)
var lineBuf []byte
wantsPrefix := true
for {
line, isPrefix, err := reader.ReadLine()
if err != nil {
return
}
if wantsPrefix && name != "" {
stdout.Write([]byte("["))
stdout.Write([]byte(name))
stdout.Write([]byte("] "))
}
multiOut.Write(line)
if isPrefix {
if visitor != nil {
lineBuf = append(lineBuf, line...)
}
wantsPrefix = false
} else {
multiOut.Write([]byte("\n"))
if visitor != nil {
if lineBuf != nil {
lineBuf = append(lineBuf, line...)
visitor(string(lineBuf))
lineBuf = nil
} else {
visitor(string(line))
}
}
wantsPrefix = true
}
}
}
func (s *SqlServer) Restart(newargs *[]string, newenvs *[]string) error {
err := s.GracefulStop()
if err != nil {
return err
}
args := s.Cmd.Args[1:]
if newargs != nil {
args = *newargs
}
s.Cmd = s.RecreateCmd(args...)
if newenvs != nil {
s.Cmd.Env = append(s.Cmd.Env, (*newenvs)...)
}
stdout, err := s.Cmd.StdoutPipe()
if err != nil {
return err
}
s.CmdWaitErr = nil
s.Cmd.Stderr = s.Cmd.Stdout
s.Done = make(chan struct{})
go func() {
defer func() {
s.CmdWaitErr = s.Cmd.Wait()
close(s.Done)
}()
logw := s.LogWriter
if logw == nil {
logw = os.Stdout
}
multiCopyWithNamePrefix(logw, s.Output, stdout, s.Name, s.OutputVisitor)
}()
return s.Cmd.Start()
}
func (s *SqlServer) DB(c Connection) (*sql.DB, error) {
connector, err := s.Connector(c)
if err != nil {
return nil, err
}
return OpenDB(connector)
}
// Connector returns a database/sql driver.Connector for a connection to this
// server using the pgx stdlib driver.
func (s *SqlServer) Connector(c Connection) (driver.Connector, error) {
pass, err := c.Password()
if err != nil {
return nil, err
}
user := c.User
if user == "" {
user = "postgres"
}
dsn := GetDSN(user, pass, s.DBName, "127.0.0.1", s.Port, c.DriverParams)
cfg, err := pgx.ParseConfig(dsn)
if err != nil {
return nil, err
}
return stdlib.GetConnector(*cfg), nil
}
// GetDSN builds a postgres connection URL.
func GetDSN(user, password, name, host string, port int, driverParams map[string]string) string {
params := make(url.Values)
params.Set("sslmode", "prefer")
for k, v := range driverParams {
params.Set(k, v)
}
u := url.URL{
Scheme: "postgres",
User: url.UserPassword(user, password),
Host: fmt.Sprintf("%s:%d", host, port),
Path: "/" + name,
RawQuery: params.Encode(),
}
return u.String()
}
func OpenDB(connector driver.Connector) (*sql.DB, error) {
db := sql.OpenDB(connector)
var err error
for i := 0; i < ConnectAttempts; i++ {
err = db.Ping()
if err == nil {
return db, nil
}
time.Sleep(RetrySleepDuration)
}
if err != nil {
return nil, err
}
return db, nil
}
func ConnectDB(user, password, name, host string, port int, params map[string]string) (*sql.DB, error) {
dsn := GetDSN(user, password, name, host, port, params)
db, err := sql.Open("pgx", dsn)
if err != nil {
return nil, err
}
for i := 0; i < ConnectAttempts; i++ {
err = db.Ping()
if err == nil {
return db, nil
}
time.Sleep(RetrySleepDuration)
}
if err != nil {
return nil, err
}
return db, nil
}
// WithConnectRetriesDisabled is retained for API-compatibility with the Dolt
// test driver. The mysql driver performed automatic connection retries that
// some tests needed to circumvent; the pgx stdlib driver does not perform the
// same retries, so this currently returns the context unchanged.
//
// TODO: If we add tests that require precise control over connection-retry
// behavior against doltgres (e.g. max-connection tests), implement an
// equivalent hook here.
func WithConnectRetriesDisabled(ctx context.Context) context.Context {
return ctx
}
integration-tests/go-sql-server-driver/driver/cmd_unix.go
+41
View File
@@ -0,0 +1,41 @@
// Copyright 2024 Dolthub, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//go:build darwin || dragonfly || freebsd || linux || netbsd || openbsd
// +build darwin dragonfly freebsd linux netbsd openbsd
package driver
import (
"os/exec"
"syscall"
)
func ApplyCmdAttributes(cmd *exec.Cmd) {
// nothing to do on unix / darwin
}
func (s *SqlServer) GracefulStop() error {
select {
case <-s.Done:
return s.CmdWaitErr
default:
}
err := s.Cmd.Process.Signal(syscall.SIGTERM)
if err != nil {
return err
}
<-s.Done
return s.CmdWaitErr
}
integration-tests/go-sql-server-driver/driver/cmd_windows.go
+46
View File
@@ -0,0 +1,46 @@
// Copyright 2024 Dolthub, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package driver
import (
"os/exec"
"syscall"
"golang.org/x/sys/windows"
)
func ApplyCmdAttributes(cmd *exec.Cmd) {
// Creating a new process group for the process will allow GracefulStop to send the break signal to that process
// without also killing the parent process
cmd.SysProcAttr = &syscall.SysProcAttr{
CreationFlags: syscall.CREATE_NEW_PROCESS_GROUP,
}
}
func (s *SqlServer) GracefulStop() error {
err := windows.GenerateConsoleCtrlEvent(windows.CTRL_BREAK_EVENT, uint32(s.Cmd.Process.Pid))
if err != nil {
return err
}
<-s.Done
_, err = s.Cmd.Process.Wait()
if err != nil {
return err
}
return nil
}
integration-tests/go-sql-server-driver/driver/server.go
+292
View File
@@ -0,0 +1,292 @@
// Copyright 2024 Dolthub, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package driver
import (
"io"
"os"
"path/filepath"
"strings"
"github.com/creasty/defaults"
"gopkg.in/yaml.v3"
)
// |Connection| represents a single connection to a sql-server instance defined
// in the test. The connection will be established and every |Query| in
// |Queries| will be run against it. At the end, the connection will be torn down.
// If |RestartServer| is non-nil, the server which the connection targets will
// be restarted after the connection is terminated.
type Connection struct {
On string `yaml:"on"`
Queries []Query `yaml:"queries"`
RestartServer *RestartArgs `yaml:"restart_server"`
// Rarely needed, allows the entire connection assertion to be retried
// on an assertion failure. Use this is only for idempotent connection
// interactions and only if the sql-server is prone to tear down the
// connection based on things that are happening, such as cluster role
// transitions.
RetryAttempts int `yaml:"retry_attempts"`
// The user to connect as. For Doltgres this defaults to the Postgres
// superuser, "postgres".
User string `default:"postgres" yaml:"user"`
// The password to connect with.
Pass string `yaml:"password"`
PassFile string `yaml:"password_file"`
// Any driver params to pass in the connection string.
DriverParams map[string]string `yaml:"driver_params"`
}
func (c *Connection) UnmarshalYAML(unmarshal func(interface{}) error) error {
defaults.Set(c)
type plain Connection
if err := unmarshal((*plain)(c)); err != nil {
return err
}
return nil
}
func (c Connection) Password() (string, error) {
if c.PassFile != "" {
passFile := c.PassFile
if v := os.Getenv("TESTGENDIR"); v != "" {
passFile = strings.ReplaceAll(passFile, "$TESTGENDIR", v)
}
bs, err := os.ReadFile(passFile)
if err != nil {
return "", err
}
return strings.TrimSpace(string(bs)), nil
}
// Doltgres connections default to the well-known development password
// "password" when none is specified, matching DefaultPass in the
// doltgres server config.
if c.Pass == "" {
return "password", nil
}
return c.Pass, nil
}
// |RestartArgs| are possible arguments, to change the arguments which are
// provided to the sql-server process when it is restarted. This is used, for
// example, to change server config on a restart.
type RestartArgs struct {
Args *[]string `yaml:"args"`
Envs *[]string `yaml:"envs"`
}
// |TestRepo| represents an init'd doltgres database that is available to a
// server instance. It can be created with some files and with remotes defined.
// |Name| can include path components separated by `/`, which will create the
// repository in a subdirectory.
type TestRepo struct {
Name string `yaml:"name"`
WithFiles []WithFile `yaml:"with_files"`
WithRemotes []WithRemote `yaml:"with_remotes"`
// Only valid on Test.Repos, not in Test.MultiRepos.Repos. If set, a
// sql-server process will be run against this TestRepo. It will be
// available as TestRepo.Name.
Server *Server `yaml:"server"`
ExternalServer *ExternalServer `yaml:"external-server"`
}
// |MultiRepo| is a subdirectory where many |TestRepo|s can be defined. You can
// start a sql-server on a |MultiRepo|, in which case there will be no default
// database to connect to.
type MultiRepo struct {
Name string `yaml:"name"`
Repos []TestRepo `yaml:"repos"`
WithFiles []WithFile `yaml:"with_files"`
// If set, a sql-server process will be run against this TestRepo. It
// will be available as MultiRepo.Name.
Server *Server `yaml:"server"`
}
// |WithRemote| defines remotes which should be defined on the repository
// before the sql-server is started.
type WithRemote struct {
Name string `yaml:"name"`
URL string `yaml:"url"`
}
// |WithFile| defines a file and its contents to be created in a |Repo| or
// |MultiRepo| before the servers are started.
type WithFile struct {
Name string `yaml:"name"`
// The contents of the file, provided inline in the YAML.
Contents string `yaml:"contents"`
// A source file path to copy to |Name|. Mutually exclusive with
// Contents.
SourcePath string `yaml:"source_path"`
// If this is non-nil, the template will be applied to the
// contents of the file as they are written through |WriteAtDir|.
Template func(string) string
}
func (f WithFile) WriteAtDir(dir string) error {
path := filepath.Join(dir, f.Name)
d := filepath.Dir(path)
err := os.MkdirAll(d, 0750)
if err != nil {
return err
}
if f.SourcePath != "" {
sourcePath := f.SourcePath
if v := os.Getenv("TESTGENDIR"); v != "" {
sourcePath = strings.ReplaceAll(sourcePath, "$TESTGENDIR", v)
}
source, err := os.Open(sourcePath)
if err != nil {
return err
}
defer source.Close()
dest, err := os.OpenFile(path, os.O_RDWR|os.O_CREATE, 0550)
if err != nil {
return err
}
contents, err := io.ReadAll(source)
if err != nil {
return err
}
if f.Template != nil {
str := f.Template(string(contents))
contents = []byte(str)
}
_, err = dest.Write(contents)
return err
} else {
contents := f.Contents
if f.Template != nil {
contents = f.Template(contents)
}
return os.WriteFile(path, []byte(contents), 0550)
}
}
// |Server| defines a sql-server process to start. |Name| must match the
// top-level |Name| of a |TestRepo| or |MultiRepo|.
type Server struct {
Name string `yaml:"name"`
Args []string `yaml:"args"`
Envs []string `yaml:"envs"`
// The |Port| which the server will be running on. For now, it is up to
// the |Args| or config file to make sure this is true. Defaults to 5432.
Port int `yaml:"port"`
// This can be used with templating of dynamic ports to
// specify the SQL listener port which will have been filled
// in by a call to `{{get_port "server_name"}}` within the
// config or args of the server.
//
// A |Port| != 0 with a |DynamicPort| != "" is an error.
DynamicPort string `yaml:"dynamic_port"`
// DebugPort if set to a non-zero value will cause this server to be started with |dlv| listening for a debugger
// connection on the port given.
DebugPort int `yaml:"debug_port"`
// Assertions to be run against the log output of the server process
// after the server process successfully terminates.
LogMatches []string `yaml:"log_matches"`
// Assertions to be run against the log output of the server process
// after the server process successfully terminates. Each entry is a
// regex that must NOT match. The test fails if any pattern matches.
LogNotMatches []string `yaml:"log_not_matches"`
// Assertions to be run against the log output of the server process
// after the server process exits with an error. If |ErrorMatches| is
// defined, then the server process must exit with a non-0 exit code
// after it is launched. This will be asserted before any |Connections|
// interactions are performed.
ErrorMatches []string `yaml:"error_matches"`
}
type ExternalServer struct {
Name string `yaml:"name"`
Host string `yaml:"host"`
User string `yaml:"user"`
Password string `yaml:"password"`
// The |Port| which the server will be running on. For now, it is up to
// the |Args| to make sure this is true. Defaults to 5432.
Port int `yaml:"port"`
}
// The primary interaction of a |Connection|. Either |Query| or |Exec| should
// be set, not both.
type Query struct {
// Run a query against the connection.
Query string `yaml:"query"`
// Run a command against the connection.
Exec string `yaml:"exec"`
// Args to be passed as query parameters to either Query or Exec.
Args []string `yaml:"args"`
// This can only be non-empty for a |Query|. Asserts the results of the
// |Query|.
Result QueryResult `yaml:"result"`
// If this is non-empty, asserts the |Query| or the |Exec|
// generates an error that matches this string.
ErrorMatch string `yaml:"error_match"`
// If this is non-zero, it represents the number of times to try the
// |Query| or the |Exec| and to check its assertions before we fail the
// test as a result of failed assertions. When interacting with queries
// that introspect things like replication state, this can be used to
// wait for quiescence in an inherently racey process. Interactions
// will be delayed slightly between each failure.
RetryAttempts int `yaml:"retry_attempts"`
}
// |QueryResult| specifies assertions on the results of a |Query|. Columns must
// be specified for a |Query| and the query results must fully match. If Rows
// are omitted, anything is allowed as long as all rows are read successfully.
// All assertions here are string equality.
type QueryResult struct {
Columns []string `yaml:"columns"`
Rows ResultRows `yaml:"rows"`
}
type ResultRows struct {
Or *[][][]string
}
func (r *ResultRows) UnmarshalYAML(value *yaml.Node) error {
if value.Kind == yaml.SequenceNode {
res := make([][][]string, 1)
r.Or = &res
return value.Decode(&(*r.Or)[0])
}
var or struct {
Or *[][][]string `yaml:"or"`
}
err := value.Decode(&or)
if err != nil {
return err
}
r.Or = or.Or
return nil
}
integration-tests/go-sql-server-driver/full_gc_no_conjoin_test.go
+240
View File
@@ -0,0 +1,240 @@
// Copyright 2026 Dolthub, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package main
import (
"fmt"
"io/fs"
"path/filepath"
"strings"
"sync/atomic"
"testing"
"time"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
driver "github.com/dolthub/doltgresql/integration-tests/go-sql-server-driver/driver"
)
// This test asserts that running a full gc into the old gen at the
// point that it is at its conjoin limit will not generate a conjoin
// in the middle of the GC.
//
// It first generates empty commits and runs GC in a loop. After every
// GC, it checks how many table files are in the oldgen. When it sees
// them get conjoined, it assumes max table files is the observed high
// water mark.
//
// It then creates exactly that many table files in oldgen by creating
// empty commits and running gc however many times it needs to.
//
// Then it needs to create one more commit and run a
// dolt_gc(--full). Currently the sql-server logs whenever it begins
// a conjoin, and that determination is made synchronously on the
// write path. So, no real attempt is made to get a theoretically
// started conjoin to win the race against the newgen collection, for
// example.
//
// As described above, this test is tightly coupled with
// GenerationalNBS, NomsBlockStore and the file persisters, with
// conjoin strategy behavior. Doltgres uses the same Dolt storage
// engine, so the conjoin log lines (pkg=store.noms) and the on-disk
// oldgen table-file layout are identical.
//
// At the end of the test, there should be one file in the oldgen, the
// results of the --full. There should be exactly one "beginning
// conjoin of database" in the server logs and it should correspond to
// when we measured the high water mark. After shutting down the
// server should happily start again.
//
// NOTE: a successful online dolt_gc() invalidates every open connection
// to the doltgres server, so the helpers below open a fresh connection
// for each commit/gc iteration (db.SetMaxIdleConns(0) ensures each
// connection is a new one rather than a pooled, now-invalid connection).
func TestFullGCNoOldgenConjoin(t *testing.T) {
t.Parallel()
ports := newPorts(t)
u, err := driver.NewDoltUser()
require.NoError(t, err)
t.Cleanup(func() {
u.Cleanup()
})
dbname := "full_gc_no_oldgen_conjoin_test"
rs, err := u.MakeRepoStore()
require.NoError(t, err)
repo, err := rs.MakeRepo(dbname)
require.NoError(t, err)
srvSettings := &driver.Server{
Args: []string{},
DynamicPort: "server",
}
var conjoinStarted, conjoinFinished atomic.Bool
upstreamLenOnConjoin := 0
server := MakeServer(t, rs, rs.Dir, srvSettings, ports, driver.WithOutputVisitor(func(out string) {
// Matching a line like:
// time="..." level=info msg="beginning conjoin of database" database=full_gc_no_oldgen_conjoin_test generation=old pkg=store.noms upstream_len=257
// Extracting its upstream_len to see exactly when the conjoin was triggered.
if upstreamLenOnConjoin <= 0 && strings.Contains(out, "beginning conjoin of database") {
if i := strings.Index(out, "upstream_len="); i != -1 {
i += len("upstream_len=")
if _, err := fmt.Sscanf(out[i:], "%d", &upstreamLenOnConjoin); err != nil {
upstreamLenOnConjoin = -1
}
}
conjoinStarted.Store(true)
}
// Matching the line:
// time="..." level=info msg="conjoin completed successfully" database=full_gc_no_oldgen_conjoin_test generation=old new_upstream_len=2 pkg=store.noms
// So we can block on further operations until conjoin is done.
if strings.Contains(out, "conjoin completed successfully") {
conjoinFinished.Store(true)
}
}))
server.DBName = dbname
oldgendir := filepath.Join(repo.Dir, "/.dolt/noms/oldgen")
CommitAndGCUntilConjoin(t, server, &conjoinStarted, oldgendir)
require.Greater(t, upstreamLenOnConjoin, 0)
require.Eventually(t, func() bool {
return conjoinFinished.Load()
}, 5*time.Second, 32*time.Millisecond)
CreateUpToNumFiles(t, server, oldgendir, upstreamLenOnConjoin)
cnt := CountTableFiles(t, oldgendir)
t.Logf("now there are %d", cnt)
RunGCFull(t, server, oldgendir)
require.NoError(t, server.GracefulStop())
output := server.Output.String()
assert.Equal(t, 1, strings.Count(output, "beginning conjoin of database"))
// The line for triggering a conjoin on policy but not proceeding because
// conjoin is dynamically disabled looks like:
// time="..." level=info msg="conjoin dynamically disabled. not conjoining." database=full_gc_no_oldgen_conjoin_test generation=old pkg=store.noms
assert.Equal(t, 1, strings.Count(output, "conjoin dynamically disabled"))
cnt = CountTableFiles(t, oldgendir)
assert.Equal(t, 1, cnt)
newServer := MakeServer(t, rs, rs.Dir, srvSettings, ports)
newServer.DBName = dbname
db, err := newServer.DB(driver.Connection{})
require.NoError(t, err)
defer db.Close()
require.NoError(t, db.PingContext(t.Context()))
}
// isTableFileName reports whether n is the name of a Dolt/Noms table file: a
// 32-character string in Noms' base32 hash alphabet. This mirrors
// hash.MaybeParse from Dolt's store/hash package without importing it.
func isTableFileName(n string) bool {
const alphabet = "0123456789abcdefghijklmnopqrstuv"
if len(n) != 32 {
return false
}
for _, c := range n {
if !strings.ContainsRune(alphabet, c) {
return false
}
}
return true
}
func CountTableFiles(t *testing.T, dir string) int {
var count int
err := filepath.WalkDir(dir, func(path string, d fs.DirEntry, err error) error {
if err != nil {
return err
}
n := d.Name()
// Archive table files carry a .darc suffix.
n = strings.TrimSuffix(n, ".darc")
if isTableFileName(n) {
count += 1
}
return nil
})
require.NoError(t, err)
return count
}
func CommitAndGCUntilConjoin(t *testing.T, srv *driver.SqlServer, conjoinStarted *atomic.Bool, path string) {
ctx := t.Context()
db, err := srv.DB(driver.Connection{})
require.NoError(t, err)
defer db.Close()
// Ensure each db.Conn() hands out a new connection rather than a pooled
// one that a prior online GC has invalidated.
db.SetMaxIdleConns(0)
for {
func() {
conn, err := db.Conn(ctx)
require.NoError(t, err)
defer conn.Close()
_, err = conn.ExecContext(ctx, "SELECT DOLT_COMMIT('-A', '--allow-empty', '-m', 'creating a new commit')")
require.NoError(t, err)
_, err = conn.ExecContext(ctx, "SELECT DOLT_GC()")
require.NoError(t, err)
}()
if conjoinStarted.Load() {
return
}
}
}
func CreateUpToNumFiles(t *testing.T, srv *driver.SqlServer, path string, numFiles int) {
ctx := t.Context()
db, err := srv.DB(driver.Connection{})
require.NoError(t, err)
defer db.Close()
db.SetMaxIdleConns(0)
for {
cnt := CountTableFiles(t, path)
if cnt == numFiles {
return
}
func() {
conn, err := db.Conn(ctx)
require.NoError(t, err)
defer conn.Close()
_, err = conn.ExecContext(ctx, "SELECT DOLT_COMMIT('-A', '--allow-empty', '-m', 'creating a new commit')")
require.NoError(t, err)
_, err = conn.ExecContext(ctx, "SELECT DOLT_GC()")
require.NoError(t, err)
}()
}
}
func RunGCFull(t *testing.T, srv *driver.SqlServer, path string) {
ctx := t.Context()
db, err := srv.DB(driver.Connection{})
require.NoError(t, err)
defer db.Close()
db.SetMaxIdleConns(0)
conn, err := db.Conn(ctx)
require.NoError(t, err)
defer conn.Close()
_ = CountTableFiles(t, path)
_, err = conn.ExecContext(ctx, "SELECT DOLT_GC('--full')")
require.NoError(t, err)
cnt := CountTableFiles(t, path)
require.Equal(t, 1, cnt)
}
integration-tests/go-sql-server-driver/gc_incremental_test.go
+316
View File
@@ -0,0 +1,316 @@
// Copyright 2026 Dolthub, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package main
import (
"context"
"database/sql"
"fmt"
"os"
"path/filepath"
"strings"
"testing"
"github.com/stretchr/testify/require"
driver "github.com/dolthub/doltgresql/integration-tests/go-sql-server-driver/driver"
)
// Archive levels accepted by dolt_gc's --archive-level option. These mirror
// chunks.GCArchiveLevel in Dolt (NoArchive=0, SimpleArchive=1); we use plain
// ints here to avoid depending on Dolt's internal store/chunks package.
const (
noArchive = 0
simpleArchive = 1
)
func TestGCIncremental(t *testing.T) {
t.Parallel()
archiveLevels := []int{noArchive, simpleArchive}
// smallFileSize is chosen so that every leaf chunk gets put in its own table file. In this test, we know
// exactly how many table files to expect after GC.
smallFileSize := 1
// mediumFileSize is chosen so that multiple chunks are put into each table file, and ensures that we correctly handle
// an in-progress table file at the end of the mark-and-sweep.
mediumFileSize := 5_000
// largeFileSize is arbitrarily large and tests the case where all leaf chunks fit in a single table file.
largeFileSize := 1_000_0000
fileSizes := []int{smallFileSize, mediumFileSize, largeFileSize}
for _, archiveLevel := range archiveLevels {
t.Run(fmt.Sprintf("archiveLevel=%d", archiveLevel), func(t *testing.T) {
t.Parallel()
for _, fileSize := range fileSizes {
t.Run(fmt.Sprintf("fileSize=%d", fileSize), func(t *testing.T) {
t.Parallel()
for _, full := range []bool{true, false} {
t.Run(fmt.Sprintf("full=%v", full), func(t *testing.T) {
t.Parallel()
runGCIncrementalTest(t, archiveLevel, fileSize, full)
})
}
})
}
})
}
}
func runGCIncrementalTest(t *testing.T, archiveLevel int, fileSize int, full bool) {
ports := newPorts(t)
u, err := driver.NewDoltUser()
require.NoError(t, err)
t.Cleanup(func() { u.Cleanup() })
rs, err := u.MakeRepoStore()
require.NoError(t, err)
repoName := fmt.Sprintf("incremental_gc_test_archivelevel_%d_filesize_%d_full_%v", archiveLevel, fileSize, full)
repo, err := rs.MakeRepo(repoName)
require.NoError(t, err)
ctx := context.Background()
server := StartServer(t, rs, repoName, &driver.Server{
Args: []string{},
DynamicPort: "server",
}, ports)
// Create the database and run GC
func() {
db, err := server.DB(driver.Connection{})
require.NoError(t, err)
defer db.Close()
conn, err := db.Conn(ctx)
require.NoError(t, err)
defer conn.Close()
populateDB(t, conn)
// dolt_gc takes its options as string arguments; build the statement
// with literals rather than bind parameters so the string-typed args
// are passed exactly as the procedure expects.
gcSQL := fmt.Sprintf("select dolt_gc('--archive-level','%d','--incremental-file-size','%d')", archiveLevel, fileSize)
if full {
gcSQL = fmt.Sprintf("select dolt_gc('--archive-level','%d','--incremental-file-size','%d','--full')", archiveLevel, fileSize)
}
_, err = conn.ExecContext(ctx, gcSQL)
require.NoError(t, err)
}()
// Verify that the DB is in a valid state after GC. Doltgres does not expose
// an fsck command (Dolt's `dolt fsck` CLI / dolt_fsck() are unavailable
// here), so as a stand-in we reconnect (the previous connection is
// invalidated by the online GC) and confirm the database is still
// queryable and the committed data survived.
requireDatabaseQueryable(t, server)
repoSize, err := GetRepoSize(repo.Dir)
require.NoError(t, err)
// Both newgen and oldgen should contain multiple chunk files
require.Greater(t, repoSize.NewGenC, 1)
require.Greater(t, repoSize.OldGenC, 1)
}
// requireDatabaseQueryable opens a fresh connection to the server and confirms
// the committed table is readable. A successful online GC invalidates prior
// connections, so callers must not reuse a connection held across a GC.
func requireDatabaseQueryable(t *testing.T, server *driver.SqlServer) {
ctx := context.Background()
db, err := server.DB(driver.Connection{})
require.NoError(t, err)
defer db.Close()
var n int
err = db.QueryRowContext(ctx, "select count(*) from vals").Scan(&n)
require.NoError(t, err)
require.Greater(t, n, 0)
}
func populateDB(t *testing.T, conn *sql.Conn) {
ctx := context.Background()
// Create some chunks that will be collected into oldgen because they're referenced by a commit
_, err := conn.ExecContext(ctx, "create table vals (id bigint primary key, val bigint)")
require.NoError(t, err)
vals := []string{}
for i := 0; i <= 1024; i++ {
vals = append(vals, fmt.Sprintf("(%d,0)", i))
}
_, err = conn.ExecContext(ctx, "insert into vals values "+strings.Join(vals, ","))
require.NoError(t, err)
_, err = conn.ExecContext(ctx, "select dolt_commit('-Am', 'create vals table')")
require.NoError(t, err)
// Create some chunks that will be collected into newgen because they aren't referenced by a commit
_, err = conn.ExecContext(ctx, "create table vals2 (id bigint primary key, val bigint)")
require.NoError(t, err)
_, err = conn.ExecContext(ctx, "insert into vals2 values "+strings.Join(vals, ","))
require.NoError(t, err)
}
type RepoSize struct {
Journal int64
NewGen int64
NewGenC int
OldGen int64
OldGenC int
}
func GetRepoSize(dir string) (RepoSize, error) {
skipFile := func(filename string) bool {
return filename == "manifest" || filename == "LOCK"
}
var ret RepoSize
entries, err := os.ReadDir(filepath.Join(dir, ".dolt/noms"))
if err != nil {
return ret, err
}
for _, e := range entries {
if skipFile(e.Name()) {
continue
}
stat, err := e.Info()
if err != nil {
return ret, err
}
if stat.IsDir() {
continue
}
ret.NewGen += stat.Size()
ret.NewGenC += 1
if e.Name() == "vvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvv" {
ret.Journal += stat.Size()
}
}
entries, err = os.ReadDir(filepath.Join(dir, ".dolt/noms/oldgen"))
if err != nil {
return ret, err
}
for _, e := range entries {
if skipFile(e.Name()) {
continue
}
stat, err := e.Info()
if err != nil {
return ret, err
}
if stat.IsDir() {
continue
}
ret.OldGen += stat.Size()
ret.OldGenC += 1
}
return ret, nil
}
func TestResumableGC(t *testing.T) {
t.Parallel()
ports := newPorts(t)
u, err := driver.NewDoltUser()
require.NoError(t, err)
t.Cleanup(func() { u.Cleanup() })
rs, err := u.MakeRepoStore()
require.NoError(t, err)
repoName := "resumable_gc_test"
repo, err := rs.MakeRepo(repoName)
require.NoError(t, err)
ctx := context.Background()
server := StartServer(t, rs, repoName, &driver.Server{
Args: []string{},
DynamicPort: "server",
Envs: []string{"DOLT_TEST_ABORT_GC_AFTER_INCREMENTAL_FILE_WRITE=true"},
}, ports)
// Create the database and run GC
func() {
db, err := server.DB(driver.Connection{})
require.NoError(t, err)
defer db.Close()
conn, err := db.Conn(ctx)
require.NoError(t, err)
defer conn.Close()
populateDB(t, conn)
// Test 1: Oldgen is not amended when --full is set
_, err = conn.ExecContext(ctx, "select dolt_gc('--archive-level','0','--incremental-file-size','1','--full')")
require.Errorf(t, err, "GC aborting after writing incremental table file")
requireFileAddedToManifest(t, filepath.Join(repo.Dir, ".dolt/noms/oldgen"), false)
// Test 2: Oldgen is amended when --full is not set
_, err = conn.ExecContext(ctx, "select dolt_gc('--archive-level','0','--incremental-file-size','1')")
require.Errorf(t, err, "GC aborting after writing incremental table file")
requireFileAddedToManifest(t, filepath.Join(repo.Dir, ".dolt/noms/oldgen"), true)
_, err = conn.ExecContext(ctx, "select dolt_gc('--archive-level','0','--incremental-file-size','1')")
// Test 3: Newgen is not amended even when --full is not set (but there are no changes to oldgen)
_, err = conn.ExecContext(ctx, "insert into vals2 values (1025, 0)")
require.NoError(t, err)
_, err = conn.ExecContext(ctx, "select dolt_gc('--archive-level','0','--incremental-file-size','1')")
require.Errorf(t, err, "GC aborting after writing incremental table file")
requireFileAddedToManifest(t, filepath.Join(repo.Dir, ".dolt/noms/oldgen"), false)
}()
}
func requireFileAddedToManifest(t *testing.T, path string, expectedResult bool) {
// Verify that the manifest contains the new chunk file
manifest, err := os.Open(filepath.Join(path, "manifest"))
if os.IsNotExist(err) {
// A nonexistent manifest contains no files
require.False(t, expectedResult)
return
}
require.NoError(t, err)
entries, err := os.ReadDir(path)
require.NoError(t, err)
var tablefileName string
for _, e := range entries {
if e.Name() == "manifest" || e.Name() == "LOCK" {
continue
}
if e.IsDir() {
continue
}
tablefileName = e.Name()
break
}
require.NotEmpty(t, tablefileName)
manifestBuffer := make([]byte, 200)
_, err = manifest.Read(manifestBuffer)
require.NoError(t, err)
if expectedResult {
require.Contains(t, string(manifestBuffer), tablefileName)
} else {
require.NotContains(t, string(manifestBuffer), tablefileName)
}
}
integration-tests/go-sql-server-driver/gencerts_test.go
+337
View File
@@ -0,0 +1,337 @@
// Copyright 2022 Dolthub, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package main
import (
"crypto/ed25519"
"crypto/rand"
"crypto/rsa"
"crypto/x509"
"crypto/x509/pkix"
"encoding/pem"
"fmt"
"math/big"
"net/url"
"os"
"path/filepath"
"time"
)
// Generates a 4096-bit RSA chain and an ed25519 chain.
// Each chain includes a root, an intermediate, a leaf with DNS and URI SANs.
// Root and intermediate have isCA=true and key usage CertSign.
// Leaf has isCA=false and key usage digitalSignature and extKeyUsage ServerAuth.
//
// Generates separate expired leafs for each key type.
//
// Emits private keys of the leafs. RSA keys are emitted PEM encoded PKCS1.
// ed25519 keys are emitted PEM encoded PKCS8.
//
// These certificates and private keys are used by
// tests/sql-server-cluster-tls.yaml and tests/sql-server-tls.yaml, for
// example.
//
// TODO: Further tests which should not verify? (SHA-1 signatures, expired
// roots or intermediates, wrong isCA, wrong key usage, etc.)
func GenerateX509Certs(dir string) error {
rsacerts, err := MakeRSACerts()
if err != nil {
return fmt.Errorf("could not make rsa certs: %w", err)
}
err = WriteRSACerts(dir, rsacerts)
if err != nil {
return fmt.Errorf("could not write rsa certs: %w", err)
}
edcerts, err := MakeEd25519Certs()
if err != nil {
return fmt.Errorf("could not make ed25519 certs: %w", err)
}
err = WriteEd25519Certs(dir, edcerts)
if err != nil {
return fmt.Errorf("could not write ed25519 certs: %w", err)
}
return nil
}
func WriteRSACerts(dir string, rsacerts TestCerts) error {
err := os.WriteFile(filepath.Join(dir, "rsa_root.pem"), pem.EncodeToMemory(&pem.Block{
Type: "CERTIFICATE",
Bytes: rsacerts.Root.Raw,
}), 0664)
if err != nil {
return err
}
err = os.WriteFile(filepath.Join(dir, "rsa_chain.pem"), append(pem.EncodeToMemory(&pem.Block{
Type: "CERTIFICATE",
Bytes: rsacerts.Leaf.Raw,
}), pem.EncodeToMemory(&pem.Block{
Type: "CERTIFICATE",
Bytes: rsacerts.Intermediate.Raw,
})...), 0664)
if err != nil {
return err
}
err = os.WriteFile(filepath.Join(dir, "rsa_key.pem"), pem.EncodeToMemory(&pem.Block{
Type: "RSA PRIVATE KEY",
Bytes: x509.MarshalPKCS1PrivateKey(rsacerts.LeafKey.(*rsa.PrivateKey)),
}), 0664)
if err != nil {
return err
}
err = os.WriteFile(filepath.Join(dir, "rsa_exp_chain.pem"), append(pem.EncodeToMemory(&pem.Block{
Type: "CERTIFICATE",
Bytes: rsacerts.ExpiredLeaf.Raw,
}), pem.EncodeToMemory(&pem.Block{
Type: "CERTIFICATE",
Bytes: rsacerts.Intermediate.Raw,
})...), 0664)
if err != nil {
return err
}
err = os.WriteFile(filepath.Join(dir, "rsa_exp_key.pem"), pem.EncodeToMemory(&pem.Block{
Type: "RSA PRIVATE KEY",
Bytes: x509.MarshalPKCS1PrivateKey(rsacerts.ExpiredLeafKey.(*rsa.PrivateKey)),
}), 0664)
if err != nil {
return err
}
return nil
}
func WriteEd25519Certs(dir string, edcerts TestCerts) error {
err := os.WriteFile(filepath.Join(dir, "ed25519_root.pem"), pem.EncodeToMemory(&pem.Block{
Type: "CERTIFICATE",
Bytes: edcerts.Root.Raw,
}), 0664)
if err != nil {
return err
}
err = os.WriteFile(filepath.Join(dir, "ed25519_chain.pem"), append(pem.EncodeToMemory(&pem.Block{
Type: "CERTIFICATE",
Bytes: edcerts.Leaf.Raw,
}), pem.EncodeToMemory(&pem.Block{
Type: "CERTIFICATE",
Bytes: edcerts.Intermediate.Raw,
})...), 0664)
if err != nil {
return err
}
keybytes, err := x509.MarshalPKCS8PrivateKey(edcerts.LeafKey)
if err != nil {
return err
}
err = os.WriteFile(filepath.Join(dir, "ed25519_key.pem"), pem.EncodeToMemory(&pem.Block{
Type: "PRIVATE KEY",
Bytes: keybytes,
}), 0664)
if err != nil {
return err
}
err = os.WriteFile(filepath.Join(dir, "ed25519_exp_chain.pem"), append(pem.EncodeToMemory(&pem.Block{
Type: "CERTIFICATE",
Bytes: edcerts.ExpiredLeaf.Raw,
}), pem.EncodeToMemory(&pem.Block{
Type: "CERTIFICATE",
Bytes: edcerts.Intermediate.Raw,
})...), 0664)
if err != nil {
return err
}
keybytes, err = x509.MarshalPKCS8PrivateKey(edcerts.ExpiredLeafKey)
if err != nil {
return err
}
err = os.WriteFile(filepath.Join(dir, "edcerts_exp_key.pem"), pem.EncodeToMemory(&pem.Block{
Type: "PRIVATE KEY",
Bytes: keybytes,
}), 0664)
if err != nil {
return err
}
return nil
}
type TestCerts struct {
Root *x509.Certificate
Intermediate *x509.Certificate
Leaf *x509.Certificate
LeafKey any
ExpiredLeaf *x509.Certificate
ExpiredLeafKey any
}
func MakeRSACerts() (TestCerts, error) {
genKey := func() (any, any, error) {
key, err := rsa.GenerateKey(rand.Reader, 4096)
if err != nil {
return nil, nil, err
}
return key.Public(), key, nil
}
return MakeCerts("RSA 4096-bit", genKey)
}
func MakeEd25519Certs() (TestCerts, error) {
genKey := func() (any, any, error) {
return ed25519.GenerateKey(rand.Reader)
}
return MakeCerts("ed25519", genKey)
}
func MakeCerts(desc string, genKey func() (any, any, error)) (TestCerts, error) {
nbf := time.Now().Add(-24 * time.Hour)
exp := nbf.Add(24 * 365 * 10 * time.Hour)
badExp := nbf.Add(12 * time.Hour)
rootpub, rootpriv, err := genKey()
if err != nil {
return TestCerts{}, err
}
intpub, intpriv, err := genKey()
if err != nil {
return TestCerts{}, err
}
leafpub, leafpriv, err := genKey()
if err != nil {
return TestCerts{}, err
}
exppub, exppriv, err := genKey()
if err != nil {
return TestCerts{}, err
}
signer, err := NewRootSigner(&x509.Certificate{
SerialNumber: big.NewInt(1),
Subject: pkix.Name{
Country: []string{"US"},
Organization: []string{"DoltHub, Inc."},
CommonName: "dolt integration tests " + desc + " Root",
},
BasicConstraintsValid: true,
IsCA: true,
KeyUsage: x509.KeyUsageCertSign,
NotBefore: nbf,
NotAfter: exp,
}, rootpub, rootpriv)
if err != nil {
return TestCerts{}, err
}
intcert, err := signer.Sign(&x509.Certificate{
SerialNumber: big.NewInt(2),
Subject: pkix.Name{
Country: []string{"US"},
Organization: []string{"DoltHub, Inc."},
CommonName: "dolt integration tests " + desc + " Intermediate",
},
BasicConstraintsValid: true,
IsCA: true,
KeyUsage: x509.KeyUsageCertSign,
NotBefore: nbf,
NotAfter: exp,
}, intpub)
if err != nil {
return TestCerts{}, err
}
intsigner := Signer{intcert, intpriv}
leafdns := "dolt-instance.dolt-integration-test.example"
leafurl, err := url.Parse("spiffe://dolt-integration-tests.dev.trust.dolthub.com.example/dolt-instance")
if err != nil {
return TestCerts{}, err
}
leafcert, err := intsigner.Sign(&x509.Certificate{
SerialNumber: big.NewInt(3),
Subject: pkix.Name{
Country: []string{"US"},
Organization: []string{"DoltHub, Inc."},
CommonName: "dolt integration tests " + desc + " Leaf",
},
BasicConstraintsValid: true,
IsCA: false,
KeyUsage: x509.KeyUsageDigitalSignature,
ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth},
NotBefore: nbf,
NotAfter: exp,
DNSNames: []string{leafdns},
URIs: []*url.URL{leafurl},
}, leafpub)
if err != nil {
return TestCerts{}, err
}
expcert, err := intsigner.Sign(&x509.Certificate{
SerialNumber: big.NewInt(4),
Subject: pkix.Name{
Country: []string{"US"},
Organization: []string{"DoltHub, Inc."},
CommonName: "dolt integration tests " + desc + " Expired Leaf",
},
BasicConstraintsValid: true,
IsCA: false,
KeyUsage: x509.KeyUsageDigitalSignature,
ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth},
NotBefore: nbf,
NotAfter: badExp,
DNSNames: []string{leafdns},
URIs: []*url.URL{leafurl},
}, exppub)
if err != nil {
return TestCerts{}, err
}
return TestCerts{
Root: signer.Cert,
Intermediate: intsigner.Cert,
Leaf: leafcert,
ExpiredLeaf: expcert,
LeafKey: leafpriv,
ExpiredLeafKey: exppriv,
}, nil
}
type Signer struct {
Cert *x509.Certificate
Key interface{}
}
func (s Signer) Sign(cert *x509.Certificate, pub any) (*x509.Certificate, error) {
der, err := x509.CreateCertificate(rand.Reader, cert, s.Cert, pub, s.Key)
if err != nil {
return nil, err
}
return x509.ParseCertificate(der)
}
func NewRootSigner(cert *x509.Certificate, pub, priv any) (Signer, error) {
der, err := x509.CreateCertificate(rand.Reader, cert, cert, pub, priv)
if err != nil {
return Signer{}, err
}
cert, err = x509.ParseCertificate(der)
if err != nil {
return Signer{}, err
}
return Signer{cert, priv}, nil
}
integration-tests/go-sql-server-driver/genjwt_test.go
+134
View File
@@ -0,0 +1,134 @@
// Copyright 2022 Dolthub, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package main
import (
"crypto"
"crypto/rand"
"crypto/rsa"
"encoding/json"
"fmt"
"os"
"path/filepath"
"time"
"github.com/google/uuid"
"gopkg.in/go-jose/go-jose.v2"
"gopkg.in/go-jose/go-jose.v2/jwt"
)
var sub = "test_user"
var iss = "dolthub.com"
var aud = "my_resource"
var onBehalfOf = "my_user"
// Generates a JWKS and a JWT for authenticating against it. Outputs it into
// files `|dir|/token.jwt` and `|dir|/test_jwks.json`.
//
// These files are used by sql-server-jwt-auth.yaml, for example.
func GenerateTestJWTs(dir string) error {
privKey, err := rsa.GenerateKey(rand.Reader, 2048)
if err != nil {
return fmt.Errorf("could not generate rsa key: %w", err)
}
pubKey := privKey.Public()
kid, err := uuid.NewRandom()
if err != nil {
return fmt.Errorf("could not generate random uuid: %w", err)
}
err = writeJWKSToFile(dir, pubKey, kid.String())
if err != nil {
return fmt.Errorf("could not write jwks to file: %w", err)
}
jwt, err := generateJWT(privKey, kid.String())
if err != nil {
return fmt.Errorf("could not generate jwt: %w", err)
}
err = os.WriteFile(filepath.Join(dir, "token.jwt"), []byte(jwt), 0644)
if err != nil {
return fmt.Errorf("could not write jwt to file: %w", err)
}
return nil
}
func writeJWKSToFile(dir string, pubKey crypto.PublicKey, kid string) error {
jwk := jose.JSONWebKey{
KeyID: kid,
Key: pubKey,
Use: "sig",
Algorithm: "RS256",
}
jwks := jose.JSONWebKeySet{Keys: []jose.JSONWebKey{jwk}}
jwksjson, err := json.Marshal(jwks)
if err != nil {
return err
}
err = os.WriteFile(filepath.Join(dir, "test_jwks.json"), jwksjson, 0644)
if err != nil {
return err
}
return nil
}
func generateJWT(privKey *rsa.PrivateKey, kid string) (string, error) {
id, err := uuid.NewRandom()
if err != nil {
return "", err
}
now := time.Now()
claims := jwt.Claims{
ID: id.String(),
Audience: []string{aud},
Issuer: iss,
Subject: sub,
IssuedAt: jwt.NewNumericDate(now),
Expiry: jwt.NewNumericDate(now.Add(364 * 24 * time.Hour)),
}
privClaims := struct {
OnBehalfOf string `json:"on_behalf_of"`
}{
onBehalfOf,
}
sig := jose.SigningKey{Algorithm: jose.RS256, Key: privKey}
opts := (&jose.SignerOptions{ExtraHeaders: map[jose.HeaderKey]interface{}{
"kid": kid,
}}).WithType("JWT")
signer, err := jose.NewSigner(sig, opts)
if err != nil {
return "", err
}
jwtBuilder := jwt.Signed(signer)
jwtBuilder = jwtBuilder.Claims(claims)
jwtBuilder = jwtBuilder.Claims(privClaims)
com, err := jwtBuilder.CompactSerialize()
if err != nil {
return "", err
}
return com, nil
}
integration-tests/go-sql-server-driver/go.mod
+27
View File
@@ -0,0 +1,27 @@
module github.com/dolthub/doltgresql/integration-tests/go-sql-server-driver
go 1.26.2
require (
github.com/creasty/defaults v1.6.0
github.com/google/uuid v1.6.0
github.com/jackc/pgx/v5 v5.9.2
github.com/stretchr/testify v1.11.1
golang.org/x/sync v0.20.0
golang.org/x/sys v0.38.0
gopkg.in/go-jose/go-jose.v2 v2.6.3
gopkg.in/yaml.v3 v3.0.1
)
require (
github.com/davecgh/go-spew v1.1.1 // indirect
github.com/google/go-cmp v0.7.0 // indirect
github.com/jackc/pgpassfile v1.0.0 // indirect
github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect
github.com/jackc/puddle/v2 v2.2.2 // indirect
github.com/kr/text v0.2.0 // indirect
github.com/pmezard/go-difflib v1.0.0 // indirect
github.com/rogpeppe/go-internal v1.15.0 // indirect
golang.org/x/crypto v0.45.0 // indirect
golang.org/x/text v0.31.0 // indirect
)
integration-tests/go-sql-server-driver/go.sum
+47
View File
@@ -0,0 +1,47 @@
github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
github.com/creasty/defaults v1.6.0 h1:ltuE9cfphUtlrBeomuu8PEyISTXnxqkBIoQfXgv7BSc=
github.com/creasty/defaults v1.6.0/go.mod h1:iGzKe6pbEHnpMPtfDXZEr0NVxWnPTjb1bbDy08fPzYM=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8=
github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU=
github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM=
github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg=
github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 h1:iCEnooe7UlwOQYpKFhBabPMi4aNAfoODPEFNiAnClxo=
github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM=
github.com/jackc/pgx/v5 v5.9.2 h1:3ZhOzMWnR4yJ+RW1XImIPsD1aNSz4T4fyP7zlQb56hw=
github.com/jackc/pgx/v5 v5.9.2/go.mod h1:mal1tBGAFfLHvZzaYh77YS/eC6IX9OWbRV1QIIM0Jn4=
github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo=
github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4=
github.com/kr/pretty v0.3.0 h1:WgNl7dwNpEZ6jJ9k1snq4pZsg7DOEN8hP9Xw0Tsjwk0=
github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk=
github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/rogpeppe/go-internal v1.15.0 h1:D0RCU5rMAp+SpgkiNdrjfJ+LX4J1M32V2NeCY7EJ6hc=
github.com/rogpeppe/go-internal v1.15.0/go.mod h1:DrUVZyrJU+txYW5/1kwtXQSMFio52ZOxX7yM1VHvnxs=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U=
github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U=
golang.org/x/crypto v0.45.0 h1:jMBrvKuj23MTlT0bQEOBcAE0mjg8mK9RXFhRH6nyF3Q=
golang.org/x/crypto v0.45.0/go.mod h1:XTGrrkGJve7CYK7J8PEww4aY7gM3qMCElcJQ8n8JdX4=
golang.org/x/sync v0.20.0 h1:e0PTpb7pjO8GAtTs2dQ6jYa5BWYlMuX047Dco/pItO4=
golang.org/x/sync v0.20.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0=
golang.org/x/sys v0.38.0 h1:3yZWxaJjBmCWXqhN1qh02AkOnCQ1poK6oF+a7xWL6Gc=
golang.org/x/sys v0.38.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks=
golang.org/x/text v0.31.0 h1:aC8ghyu4JhP8VojJ2lEHBnochRno1sgL6nEi9WGFGMM=
golang.org/x/text v0.31.0/go.mod h1:tKRAlv61yKIjGGHX/4tP1LTbc13YSec1pxVEWXzfoeM=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
gopkg.in/go-jose/go-jose.v2 v2.6.3 h1:nt80fvSDlhKWQgSWyHyy5CfmlQr+asih51R8PTWNKKs=
gopkg.in/go-jose/go-jose.v2 v2.6.3/go.mod h1:zzZDPkNNw/c9IE7Z9jr11mBZQhKQTMzoEEIoEdZlFBI=
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
integration-tests/go-sql-server-driver/helpers_test.go
+57
View File
@@ -0,0 +1,57 @@
// Copyright 2024 Dolthub, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package main
import (
"testing"
"github.com/stretchr/testify/require"
driver "github.com/dolthub/doltgresql/integration-tests/go-sql-server-driver/driver"
)
// newPorts returns a DynamicResources bound to the global port pool for use in
// standalone Go tests.
func newPorts(t *testing.T) *DynamicResources {
ports := &DynamicResources{}
ports.global = &GlobalPorts
ports.t = t
return ports
}
// StartServer is the doltgres convenience wrapper used by standalone Go tests
// to start a server for the database |dbName| in the store |rs|. The doltgres
// server runs from the store directory (the data-dir) and serves |dbName| as a
// database. The returned server's DBName is set to |dbName| so connections
// target it by default.
func StartServer(t *testing.T, rs driver.RepoStore, dbName string, s *driver.Server, ports *DynamicResources) *driver.SqlServer {
server := MakeServer(t, rs, rs.Dir, s, ports)
if server != nil {
server.DBName = dbName
}
return server
}
// RunServerUntilEndOfTest runs a server until the end of the test. Because we
// do not return the server for doing things like making connections to it,
// this is only useful for asserting the behavior of other commands which
// interact with the server.
func RunServerUntilEndOfTest(t *testing.T, rs driver.RepoStore, s *driver.Server, ports *DynamicResources) {
server := MakeServer(t, rs, rs.Dir, s, ports)
require.NotNil(t, server)
db, err := server.DB(driver.Connection{})
require.NoError(t, err)
require.NoError(t, db.Close())
}
integration-tests/go-sql-server-driver/large_values_test.go
+1506
View File
File diff suppressed because it is too large Load Diff
integration-tests/go-sql-server-driver/main_test.go
+165
View File
@@ -0,0 +1,165 @@
// Copyright 2024 Dolthub, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package main
import (
"flag"
"log"
"os"
"testing"
)
// We generate various TLS keys and certificates and some JWKS/JWT material
// which the tests reference. We do this once for the test run, because it can
// be expensive, and we expose the location of the generated files through an
// environment variable. The test definitions interpolate that environment
// variable into a few fields.
//
// It's good enough for now, and it keeps us from checking in certificates or
// JWT which will expire at some point in the future.
func TestMain(m *testing.M) {
old := os.Getenv("TESTGENDIR")
defer func() {
os.Setenv("TESTGENDIR", old)
}()
gendir, err := os.MkdirTemp(os.TempDir(), "go-sql-server-driver-gen-*")
if err != nil {
log.Fatalf("could not create temp dir: %v", err)
}
defer os.RemoveAll(gendir)
err = GenerateTestJWTs(gendir)
if err != nil {
log.Fatalf("%v", err)
}
err = GenerateX509Certs(gendir)
if err != nil {
log.Fatalf("%v", err)
}
os.Setenv("TESTGENDIR", gendir)
flag.Parse()
InitGlobalDynamicPorts()
os.Exit(m.Run())
}
func InitGlobalDynamicPorts() {
// XXX: Max and min here could be supplied by flags. Currently
// tests use at most 6 ports, so this may run out of ports if
// we have more than ~40 concurrent processes.
for i := 0; i < 256; i++ {
GlobalPorts.available = append(GlobalPorts.available, 5432+i)
}
}
func TestConfig(t *testing.T) {
t.Parallel()
RunTestsFile(t, "tests/sql-server-config.yaml")
}
func TestCluster(t *testing.T) {
t.Parallel()
RunTestsFile(t, "tests/sql-server-cluster.yaml")
}
func TestClusterUsersAndGrants(t *testing.T) {
t.Parallel()
RunTestsFile(t, "tests/sql-server-cluster-users-and-grants.yaml")
}
func TestRemotesAPI(t *testing.T) {
t.Parallel()
RunTestsFile(t, "tests/sql-server-remotesapi.yaml")
}
// TestSingle is a convenience method for running a single test from within an IDE. Unskip and set to the file and name
// of the test you want to debug. See README.md in the `tests` directory for more debugging info.
func TestSingle(t *testing.T) {
t.Skip()
RunSingleTest(t, "tests/sql-server-cluster.yaml", "primary comes up and replicates to standby")
}
func TestClusterTLS(t *testing.T) {
t.Parallel()
RunTestsFile(t, "tests/sql-server-cluster-tls.yaml")
}
func TestOriginal(t *testing.T) {
t.Parallel()
RunTestsFile(t, "tests/sql-server-orig.yaml")
}
func TestTLS(t *testing.T) {
t.Parallel()
RunTestsFile(t, "tests/sql-server-tls.yaml")
}
func TestClusterReadOnly(t *testing.T) {
t.Parallel()
RunTestsFile(t, "tests/sql-server-cluster-read-only.yaml")
}
func TestLargeTextReplication(t *testing.T) {
t.Parallel()
RunTestsFile(t, "tests/sql-server-large-text-replication.yaml")
}
func TestLargeBlobReplication(t *testing.T) {
t.Parallel()
RunTestsFile(t, "tests/sql-server-large-blob-replication.yaml")
}
func TestLargeJSONReplication(t *testing.T) {
t.Parallel()
RunTestsFile(t, "tests/sql-server-large-json-replication.yaml")
}
func TestLargeMultiColumnReplication(t *testing.T) {
t.Parallel()
RunTestsFile(t, "tests/sql-server-large-multi-column-replication.yaml")
}
func TestLargeValuesGC(t *testing.T) {
t.Parallel()
RunTestsFile(t, "tests/sql-server-large-values-gc.yaml")
}
func TestTypeDiversityCluster(t *testing.T) {
t.Parallel()
RunTestsFile(t, "tests/sql-server-type-diversity.yaml")
}
func TestWideIntTable(t *testing.T) {
t.Parallel()
RunTestsFile(t, "tests/sql-server-wide-int-table.yaml")
}
func TestWideVarcharTable(t *testing.T) {
t.Parallel()
RunTestsFile(t, "tests/sql-server-wide-varchar-table.yaml")
}
func TestWideTextTable(t *testing.T) {
t.Parallel()
RunTestsFile(t, "tests/sql-server-wide-text-table.yaml")
}
func TestLargeValuesFailover(t *testing.T) {
t.Parallel()
RunTestsFile(t, "tests/sql-server-large-values-failover.yaml")
}
func TestWideTableFailover(t *testing.T) {
t.Parallel()
RunTestsFile(t, "tests/sql-server-wide-table-failover.yaml")
}
integration-tests/go-sql-server-driver/metrics_auth_test.go
+385
View File
@@ -0,0 +1,385 @@
// Copyright 2025 Dolthub, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package main
import (
"context"
"encoding/json"
"fmt"
"net"
"net/http"
"os"
"path/filepath"
"testing"
"time"
driver "github.com/dolthub/doltgresql/integration-tests/go-sql-server-driver/driver"
"github.com/stretchr/testify/require"
"gopkg.in/go-jose/go-jose.v2"
)
type getConfigFunc func(serverPort, metricsPort int) string
// runJWKSServer starts a local HTTP server that serves the JWKS file at /jwks.json.
// The server is started in a goroutine and will be shut down via t.Cleanup.
func runJWKSServer(t *testing.T, jwksFilePath string, port int) {
data, err := os.ReadFile(jwksFilePath)
require.NoError(t, err)
mux := http.NewServeMux()
mux.HandleFunc("/jwks.json", func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/json")
_, _ = w.Write(data)
})
addr := fmt.Sprintf("127.0.0.1:%d", port)
server := &http.Server{
Addr: addr,
Handler: mux,
}
ln, err := net.Listen("tcp", addr)
require.NoError(t, err)
go func() {
if err := server.Serve(ln); err != nil && err != http.ErrServerClosed {
// Use t.Log instead of t.Fatal because Serve runs in goroutine
t.Logf("runJWKSServer: server error: %v", err)
}
}()
t.Logf("Started test JWKS server on %s serving %s", addr, jwksFilePath)
t.Cleanup(func() {
ctx, cancel := context.WithTimeout(context.Background(), 1*time.Second)
defer cancel()
if err := server.Shutdown(ctx); err != nil {
t.Logf("runJWKSServer: shutdown error: %v", err)
}
})
}
func createJWT(t *testing.T, issuer, audience, subject string) string {
const kid = "749df841-6e38-48f1-a178-20ecdd0b09f7"
// load jwks from testdata
data, err := os.ReadFile("testdata/test_jwks_private.json")
require.NoError(t, err)
var jwks jose.JSONWebKeySet
err = json.Unmarshal(data, &jwks)
require.NoError(t, err)
require.NotEmpty(t, jwks.Keys)
// choose key by kid or default to first
var jwk *jose.JSONWebKey
if kid == "" {
jwk = &jwks.Keys[0]
} else {
for i := range jwks.Keys {
if jwks.Keys[i].KeyID == kid {
jwk = &jwks.Keys[i]
break
}
}
require.NotNil(t, jwk)
}
// ensure we have a private key
require.False(t, jwk.IsPublic())
// create signer with kid header (if present)
opts := (&jose.SignerOptions{}).WithType("JWT")
if jwk.KeyID != "" {
opts = opts.WithHeader("kid", jwk.KeyID)
}
signer, err := jose.NewSigner(jose.SigningKey{Algorithm: jose.RS256, Key: jwk.Key}, opts)
require.NoError(t, err)
// build claims
now := time.Now().UTC()
claims := map[string]interface{}{
"iss": issuer,
"aud": audience,
"sub": subject,
"iat": now.Unix(),
"exp": now.Add(time.Hour).Unix(),
}
payload, err := json.Marshal(claims)
require.NoError(t, err)
jws, err := signer.Sign(payload)
require.NoError(t, err)
compact, err := jws.CompactSerialize()
require.NoError(t, err)
return compact
}
func makeMetricsCall(t *testing.T, metricsPort int, bearerToken string) *http.Response {
url := fmt.Sprintf("http://127.0.0.1:%d/metrics", metricsPort)
req, err := http.NewRequest("GET", url, nil)
require.NoError(t, err)
if bearerToken != "" {
req.Header.Add("Authorization", "Bearer "+bearerToken)
}
client := &http.Client{}
resp, err := client.Do(req)
require.NoError(t, err)
defer resp.Body.Close()
return resp
}
var jwksPort int
func TestMetricsAuth(t *testing.T) {
// Doltgres serves a /metrics HTTP endpoint, but its metrics configuration
// (DoltgesMetricsConfig) does not support the `jwks` /
// `jwt_required_for_localhost` options this test relies on, and the metrics
// server does not enforce JWT auth (MetricsJwksConfig() returns nil). The
// full logic below is a faithful port and should be enabled once Doltgres
// supports JWT-protected metrics.
t.Skip("metrics JWKS / JWT-auth configuration (metrics.jwks, jwt_required_for_localhost) is unsupported in Doltgres")
jwksPort = GlobalPorts.GetPort(t)
absPath, err := filepath.Abs("./testdata/test_jwks.json")
require.NoError(t, err)
runJWKSServer(t, absPath, jwksPort)
t.Parallel()
t.Run("No Metrics Auth", testNoMetricsAuth)
t.Run("Missing Metrics Auth", testMissingMetricsAuth)
t.Run("Valid Metrics Auth", testValidMetricsAuth)
t.Run("Bad Audience Claim", testBadAudienceClaim)
t.Run("Bad Issuer Claim", testBadIssuerClaim)
t.Run("Bad Subject Claim", testBadSubjectClaim)
}
func startServerWithMetrics(t *testing.T, getConfig getConfigFunc) int {
ports := newPorts(t)
serverPort := ports.GetOrAllocatePort("server")
metricsPort := ports.GetOrAllocatePort("metrics")
config := getConfig(serverPort, metricsPort)
u, err := driver.NewDoltUser()
require.NoError(t, err)
t.Cleanup(func() {
u.Cleanup()
})
rs, err := u.MakeRepoStore()
require.NoError(t, err)
_, err = rs.MakeRepo("metrics_auth_test")
require.NoError(t, err)
f, err := os.CreateTemp("", "config-*.yaml")
require.NoError(t, err)
t.Cleanup(func() {
os.Remove(f.Name())
})
_, err = f.WriteString(config)
require.NoError(t, err)
args := []string{"--config", f.Name()}
srvSettings := &driver.Server{
Args: args,
DynamicPort: "server",
}
t.Log("Starting server with config:\n" + config)
MakeServer(t, rs, rs.Dir, srvSettings, ports)
// Wait for the metrics HTTP listener to be ready.
require.Eventually(t, func() bool {
resp, err := http.Get(fmt.Sprintf("http://127.0.0.1:%d/metrics", metricsPort))
if err != nil {
return false
}
resp.Body.Close()
return true
}, 5*time.Second, 50*time.Millisecond)
return metricsPort
}
func testNoMetricsAuth(t *testing.T) {
t.Parallel()
getConfig := func(serverPort, metricsPort int) string {
return fmt.Sprintf(`
listener:
host: localhost
port: %d
metrics:
host: localhost
port: %d
jwt_required_for_localhost: true
`, serverPort, metricsPort)
}
metricsPort := startServerWithMetrics(t, getConfig)
resp := makeMetricsCall(t, metricsPort, "")
require.Equal(t, http.StatusOK, resp.StatusCode)
}
func testMissingMetricsAuth(t *testing.T) {
t.Parallel()
getConfig := func(serverPort, metricsPort int) string {
return fmt.Sprintf(`listener:
host: localhost
port: %d
metrics:
host: localhost
port: %d
jwks:
name: jwksname
location_url: http://127.0.0.1:%d/jwks.json
claims:
alg: RS256
iss: dolthub.com
sub: test_sub
aud: test_aud
jwt_required_for_localhost: true
`, serverPort, metricsPort, jwksPort)
}
metricsPort := startServerWithMetrics(t, getConfig)
resp := makeMetricsCall(t, metricsPort, "")
require.Equal(t, http.StatusUnauthorized, resp.StatusCode)
}
func testValidMetricsAuth(t *testing.T) {
t.Parallel()
getConfig := func(serverPort, metricsPort int) string {
return fmt.Sprintf(`listener:
host: localhost
port: %d
metrics:
host: localhost
port: %d
jwks:
name: jwksname
location_url: http://127.0.0.1:%d/jwks.json
claims:
iss: dolthub.com
sub: test_sub
aud: test_aud
jwt_required_for_localhost: true
`, serverPort, metricsPort, jwksPort)
}
metricsPort := startServerWithMetrics(t, getConfig)
jwt := createJWT(t, "dolthub.com", "test_aud", "test_sub")
resp := makeMetricsCall(t, metricsPort, jwt)
require.Equal(t, http.StatusOK, resp.StatusCode)
}
func testBadIssuerClaim(t *testing.T) {
t.Parallel()
getConfig := func(serverPort, metricsPort int) string {
return fmt.Sprintf(`listener:
host: localhost
port: %d
metrics:
host: localhost
port: %d
jwks:
name: jwksname
location_url: http://127.0.0.1:%d/jwks.json
claims:
iss: dolthub.com
sub: test_sub
aud: test_aud
jwt_required_for_localhost: true
`, serverPort, metricsPort, jwksPort)
}
metricsPort := startServerWithMetrics(t, getConfig)
jwt := createJWT(t, "badissuer.com", "test_aud", "test_sub")
resp := makeMetricsCall(t, metricsPort, jwt)
require.Equal(t, http.StatusUnauthorized, resp.StatusCode)
}
func testBadAudienceClaim(t *testing.T) {
t.Parallel()
getConfig := func(serverPort, metricsPort int) string {
return fmt.Sprintf(`listener:
host: localhost
port: %d
metrics:
host: localhost
port: %d
jwks:
name: jwksname
location_url: http://127.0.0.1:%d/jwks.json
claims:
iss: dolthub.com
sub: test_sub
aud: test_aud
jwt_required_for_localhost: true
`, serverPort, metricsPort, jwksPort)
}
metricsPort := startServerWithMetrics(t, getConfig)
jwt := createJWT(t, "dolthub.com", "bad_aud", "test_sub")
resp := makeMetricsCall(t, metricsPort, jwt)
require.Equal(t, http.StatusUnauthorized, resp.StatusCode)
}
func testBadSubjectClaim(t *testing.T) {
t.Parallel()
getConfig := func(serverPort, metricsPort int) string {
return fmt.Sprintf(`listener:
host: localhost
port: %d
metrics:
host: localhost
port: %d
jwks:
name: jwksname
location_url: http://127.0.0.1:%d/jwks.json
claims:
iss: dolthub.com
sub: test_sub
aud: test_aud
jwt_required_for_localhost: true
`, serverPort, metricsPort, jwksPort)
}
metricsPort := startServerWithMetrics(t, getConfig)
jwt := createJWT(t, "dolthub.com", "test_aud", "bad_sub")
resp := makeMetricsCall(t, metricsPort, jwt)
require.Equal(t, http.StatusUnauthorized, resp.StatusCode)
}
integration-tests/go-sql-server-driver/sql_server_commit_concurrency_test.go
+385
View File
@@ -0,0 +1,385 @@
// Copyright 2024 Dolthub, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package main
import (
"context"
"fmt"
"math/rand"
"sync"
"sync/atomic"
"testing"
"time"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
driver "github.com/dolthub/doltgresql/integration-tests/go-sql-server-driver/driver"
)
func TestCommitConcurrency(t *testing.T) {
t.Parallel()
t.Run("SQL transaction with amend commit", testSQLTransactionWithAmendCommit)
t.Run("SQL racing amend", testSQLRacingAmend)
t.Run("SQL racing amend with serial column", testSQLRacingAmendWithSerial)
}
// testSQLTransactionWithAmendCommit verifies that two transactions started at the same state will not both be able
// to commit using --amend. The first transaction will be able to commit, but the second should get an error.
func testSQLTransactionWithAmendCommit(t *testing.T) {
t.Parallel()
ctx := context.Background()
ports := newPorts(t)
u, err := driver.NewDoltUser()
require.NoError(t, err)
t.Cleanup(func() {
u.Cleanup()
})
rs, err := u.MakeRepoStore()
require.NoError(t, err)
_, err = rs.MakeRepo("commit_concurrency_test")
require.NoError(t, err)
srvSettings := &driver.Server{
Args: []string{},
DynamicPort: "server_port",
}
server := StartServer(t, rs, "commit_concurrency_test", srvSettings, ports)
// Connect to the database
db, err := server.DB(driver.Connection{})
require.NoError(t, err)
t.Cleanup(func() {
db.Close()
})
_, err = db.ExecContext(ctx, `
CREATE TABLE test_table (
id serial PRIMARY KEY,
value VARCHAR(20)
);`)
require.NoError(t, err)
_, err = db.ExecContext(ctx, "INSERT INTO test_table (value) VALUES ('initial')")
require.NoError(t, err)
_, err = db.ExecContext(ctx, "SELECT dolt_commit('-A','-m', 'initial commit')")
require.NoError(t, err)
// Create a new context for the first (failing) transaction
ctx1, cancel1 := context.WithCancel(ctx)
defer cancel1()
tx1, err := db.BeginTx(ctx1, nil)
require.NoError(t, err)
_, err = tx1.ExecContext(ctx1, "UPDATE test_table SET value = 'amended by tx1' WHERE id = 1")
require.NoError(t, err)
// Create a new context for the second (succeeding) transaction
ctx2, cancel2 := context.WithCancel(ctx)
defer cancel2()
tx2, err := db.BeginTx(ctx2, nil)
require.NoError(t, err)
// Update data within the second transaction
_, err = tx2.ExecContext(ctx2, "UPDATE test_table SET value = 'amended by tx2' WHERE id = 1")
require.NoError(t, err)
_, err = tx2.ExecContext(ctx2, "SELECT dolt_commit('--amend', '-m', 'tx2 amended commit')")
require.NoError(t, err)
// Commit --amend will result in tx2 being committed. You can still make updates on tx1, but any commit should fail
_, err = tx1.ExecContext(ctx1, "INSERT INTO test_table (value) VALUES ('new row by tx1')")
require.NoError(t, err)
_, err = tx1.ExecContext(ctx1, "SELECT dolt_commit('--amend', '-m', 'should fail')")
require.Error(t, err)
require.Contains(t, err.Error(), "this transaction conflicts with a committed transaction from another client, try restarting transaction")
// Verify that the data in the head is what we would expect
row := db.QueryRowContext(ctx, "SELECT value FROM test_table WHERE id = 1")
var value string
err = row.Scan(&value)
require.NoError(t, err)
require.Equal(t, "amended by tx2", value)
// Verify the commit message
row = db.QueryRowContext(ctx, "SELECT message FROM dolt_log ORDER BY date DESC LIMIT 1")
var commitMessage string
err = row.Scan(&commitMessage)
require.NoError(t, err)
require.Equal(t, "tx2 amended commit", commitMessage)
}
func testSQLRacingAmend(t *testing.T) {
t.Parallel()
ctx := context.Background()
ports := newPorts(t)
u, err := driver.NewDoltUser()
require.NoError(t, err)
t.Cleanup(func() {
u.Cleanup()
})
rs, err := u.MakeRepoStore()
require.NoError(t, err)
_, err = rs.MakeRepo("racing_amend_test")
require.NoError(t, err)
srvSettings := &driver.Server{
Args: []string{},
DynamicPort: "server_port",
}
server := StartServer(t, rs, "racing_amend_test", srvSettings, ports)
db, err := server.DB(driver.Connection{})
require.NoError(t, err)
t.Cleanup(func() {
db.Close()
})
_, err = db.ExecContext(ctx, `
CREATE TABLE test_table (
id bigint PRIMARY KEY,
value VARCHAR(20)
);`)
require.NoError(t, err)
_, err = db.ExecContext(ctx, "INSERT INTO test_table VALUES (1, 'initial')")
require.NoError(t, err)
_, err = db.ExecContext(ctx, "SELECT dolt_commit('-A','-m', 'initial commit')")
require.NoError(t, err)
type txIdFunc struct {
txNum int
txFunc func() error
}
var transactions []txIdFunc
for txNum := 1; txNum <= 200; txNum++ {
txCtx, cancel := context.WithTimeout(ctx, 15*time.Second) // Should never hit, but just in case
tx, err := db.BeginTx(txCtx, nil)
require.NoError(t, err)
f := func() error {
defer cancel()
// update required to get a transaction conflict.
_, e2 := tx.ExecContext(txCtx, "UPDATE test_table SET value = $1 WHERE id = 1", fmt.Sprintf("tx%d value", txNum))
require.NoError(t, e2)
_, e2 = tx.ExecContext(txCtx, "INSERT INTO test_table (id, value) VALUES ($1, $2)", txNum+1, fmt.Sprintf("tx%d new row", txNum))
require.NoError(t, e2)
// We want other transactions to have the chance to mess with the db. We'll verify that what's committed is what we expect.
time.Sleep(time.Duration(rand.Intn(1000)+500) * time.Millisecond)
// This will commit the transaction, or error.
_, e2 = tx.ExecContext(txCtx, "SELECT dolt_commit('--amend','-a', '-m', $1)", fmt.Sprintf("tx%d amend", txNum))
if e2 != nil {
tx.Rollback()
}
return e2
}
transactions = append(transactions, txIdFunc{txNum: txNum, txFunc: f})
}
rand.Shuffle(len(transactions), func(i, j int) {
transactions[i], transactions[j] = transactions[j], transactions[i]
})
var atomicInt atomic.Int32
atomicInt.Store(-1)
var wg sync.WaitGroup
for _, txn := range transactions {
wg.Add(1)
go func() {
defer wg.Done()
// radomly sleep .5 - 1.5 seconds
time.Sleep(time.Duration(rand.Intn(1000)+500) * time.Millisecond)
err := txn.txFunc()
if err == nil {
// If there are multiple updates, something went wrong.
assert.True(t, atomicInt.CompareAndSwap(-1, int32(txn.txNum)))
}
// Errors are expected.
}()
}
wg.Wait()
winner := atomicInt.Load()
require.NotEqual(t, -1, winner)
// Verify there are only 2 rows in the table
rows, err := db.QueryContext(ctx, "SELECT COUNT(*) FROM test_table")
require.NoError(t, err)
defer rows.Close()
rows.Next()
var count int
err = rows.Scan(&count)
require.NoError(t, err)
require.Equal(t, 2, count)
// Verify the final state
row := db.QueryRowContext(ctx, "SELECT value FROM test_table WHERE id = 1")
var value string
err = row.Scan(&value)
require.NoError(t, err)
require.Equal(t, fmt.Sprintf("tx%d value", winner), value)
row = db.QueryRowContext(ctx, "SELECT value FROM test_table WHERE id != 1")
err = row.Scan(&value)
require.NoError(t, err)
require.Equal(t, fmt.Sprintf("tx%d new row", winner), value)
// Verify the commit message
row = db.QueryRowContext(ctx, "SELECT message FROM dolt_log ORDER BY date DESC LIMIT 1")
var commitMessage string
err = row.Scan(&commitMessage)
require.NoError(t, err)
require.Equal(t, fmt.Sprintf("tx%d amend", winner), commitMessage)
}
func testSQLRacingAmendWithSerial(t *testing.T) {
t.Skip("concurrent inserts on the same sequence fail with duplicate keys")
t.Parallel()
ctx := context.Background()
ports := newPorts(t)
u, err := driver.NewDoltUser()
require.NoError(t, err)
t.Cleanup(func() {
u.Cleanup()
})
rs, err := u.MakeRepoStore()
require.NoError(t, err)
_, err = rs.MakeRepo("racing_amend_test")
require.NoError(t, err)
srvSettings := &driver.Server{
Args: []string{},
DynamicPort: "server_port",
}
server := StartServer(t, rs, "racing_amend_test", srvSettings, ports)
db, err := server.DB(driver.Connection{})
require.NoError(t, err)
t.Cleanup(func() {
db.Close()
})
_, err = db.ExecContext(ctx, `
CREATE TABLE test_table (
id serial PRIMARY KEY,
value VARCHAR(20)
);`)
require.NoError(t, err)
_, err = db.ExecContext(ctx, "INSERT INTO test_table VALUES (1, 'initial')")
require.NoError(t, err)
_, err = db.ExecContext(ctx, "SELECT dolt_commit('-A','-m', 'initial commit')")
require.NoError(t, err)
type txIdFunc struct {
txNum int
txFunc func() error
}
var transactions []txIdFunc
for txNum := 1; txNum <= 200; txNum++ {
txCtx, cancel := context.WithTimeout(ctx, 15*time.Second) // Should never hit, but just in case
tx, err := db.BeginTx(txCtx, nil)
require.NoError(t, err)
f := func() error {
defer cancel()
// update required to get a transaction conflict.
_, e2 := tx.ExecContext(txCtx, "UPDATE test_table SET value = $1 WHERE id = 1", fmt.Sprintf("tx%d value", txNum))
require.NoError(t, e2)
_, e2 = tx.ExecContext(txCtx, "INSERT INTO test_table (value) VALUES ($1)", fmt.Sprintf("tx%d new row", txNum))
require.NoError(t, e2)
// We want other transactions to have the chance to mess with the db. We'll verify that what's committed is what we expect.
time.Sleep(time.Duration(rand.Intn(1000)+500) * time.Millisecond)
// This will commit the transaction, or error.
_, e2 = tx.ExecContext(txCtx, "SELECT dolt_commit('--amend','-a', '-m', $1)", fmt.Sprintf("tx%d amend", txNum))
if e2 != nil {
tx.Rollback()
}
return e2
}
transactions = append(transactions, txIdFunc{txNum: txNum, txFunc: f})
}
rand.Shuffle(len(transactions), func(i, j int) {
transactions[i], transactions[j] = transactions[j], transactions[i]
})
var atomicInt atomic.Int32
atomicInt.Store(-1)
var wg sync.WaitGroup
for _, txn := range transactions {
wg.Add(1)
go func() {
defer wg.Done()
// radomly sleep .5 - 1.5 seconds
time.Sleep(time.Duration(rand.Intn(1000)+500) * time.Millisecond)
err := txn.txFunc()
if err == nil {
// If there are multiple updates, something went wrong.
assert.True(t, atomicInt.CompareAndSwap(-1, int32(txn.txNum)))
}
// Errors are expected.
}()
}
wg.Wait()
winner := atomicInt.Load()
require.NotEqual(t, -1, winner)
// Verify there are only 2 rows in the table
rows, err := db.QueryContext(ctx, "SELECT COUNT(*) FROM test_table")
require.NoError(t, err)
defer rows.Close()
rows.Next()
var count int
err = rows.Scan(&count)
require.NoError(t, err)
require.Equal(t, 2, count)
// Verify the final state
row := db.QueryRowContext(ctx, "SELECT value FROM test_table WHERE id = 1")
var value string
err = row.Scan(&value)
require.NoError(t, err)
require.Equal(t, fmt.Sprintf("tx%d value", winner), value)
row = db.QueryRowContext(ctx, "SELECT value FROM test_table WHERE id != 1")
err = row.Scan(&value)
require.NoError(t, err)
require.Equal(t, fmt.Sprintf("tx%d new row", winner), value)
// Verify the commit message
row = db.QueryRowContext(ctx, "SELECT message FROM dolt_log ORDER BY date DESC LIMIT 1")
var commitMessage string
err = row.Scan(&commitMessage)
require.NoError(t, err)
require.Equal(t, fmt.Sprintf("tx%d amend", winner), commitMessage)
}
integration-tests/go-sql-server-driver/stats_gc_test.go
+314
View File
@@ -0,0 +1,314 @@
// Copyright 2025 Dolthub, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package main
import (
"context"
"errors"
"fmt"
"math/rand/v2"
"strings"
"testing"
"time"
"github.com/google/uuid"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"golang.org/x/sync/errgroup"
driver "github.com/dolthub/doltgresql/integration-tests/go-sql-server-driver/driver"
)
func TestStatsGCConcurrency(t *testing.T) {
// Doltgres' statistics provider does not implement the ExtendedStatsProvider
// interface that dolt_stats_gc() requires ("provider does not implement
// ExtendedStatsProvider"), so this test cannot run yet. The logic below is a
// faithful port of the Dolt test and should be enabled once Doltgres supports
// statistics GC.
t.Skip("dolt_stats_gc() is unsupported in Doltgres (statistics provider does not implement ExtendedStatsProvider)")
t.Parallel()
// At one time, it was possible for a GC of stats to block the
// analyzer from accessing stats for a long time. This test sets
// up the following condition:
//
// * Several branches, each with 10 tables, each with several indexes.
// All of the branches and tables and indexes are non-empty.
//
// * One thread calls `select dolt_stats_gc()`
//
// * Other threads continue to connect to the database and run
// queries.
//
// * We expect all queries to complete without much delay. If
// there is a long delay on the read queries, this fails the
// test.
ctx := context.Background()
u, err := driver.NewDoltUser()
require.NoError(t, err)
t.Cleanup(func() {
u.Cleanup()
})
rs, err := u.MakeRepoStore()
require.NoError(t, err)
_, err = rs.MakeRepo("stats_gc_concurrency_test")
require.NoError(t, err)
srvSettings := &driver.Server{
Args: []string{},
DynamicPort: "server",
}
ports := newPorts(t)
server := StartServer(t, rs, "stats_gc_concurrency_test", srvSettings, ports)
// Connect to the database
db, err := server.DB(driver.Connection{})
require.NoError(t, err)
t.Cleanup(func() {
db.Close()
})
db.SetMaxOpenConns(32)
db.SetMaxIdleConns(32)
numbranches := 3
var branches = []string{"main"}
for i := 0; i < numbranches; i++ {
branches = append(branches, uuid.New().String())
}
var tables = []string{
"t0", "t1", "t2", "t3", "t4",
"t5", "t6", "t7", "t8", "t9",
}
var columns = []string{
"c0", "c1", "c2", "c3", "c4",
"c5", "c6", "c7", "c8", "c9",
}
var columndefs = []string{}
for _, col := range columns {
columndefs = append(columndefs, fmt.Sprintf("%s bigint unique", col))
}
for _, table := range tables {
_, err = db.ExecContext(ctx, fmt.Sprintf(strings.Join([]string{
"CREATE TABLE %s (",
"id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY,",
strings.Join(columndefs, ",\n"),
");",
}, "\n"), table))
require.NoError(t, err)
}
_, err = db.ExecContext(ctx, "select dolt_commit('-Am', 'initial commit')")
require.NoError(t, err)
for _, branch := range branches[1:] {
_, err = db.ExecContext(ctx, fmt.Sprintf("select dolt_checkout('-b', '%s')", branch))
require.NoError(t, err)
for _, table := range tables {
_, err = db.ExecContext(ctx, fmt.Sprintf("insert into %s values %s", table, t0_values_str()))
require.NoError(t, err)
}
}
_, err = db.ExecContext(ctx, "select dolt_checkout('main')")
require.NoError(t, err)
for _, table := range tables {
_, err = db.ExecContext(ctx, fmt.Sprintf("insert into %s values %s", table, t0_values_str()))
require.NoError(t, err)
}
numThreads := 10
// The workers will run until this is closed. This will be closed after the stats_gc() is finished.
stop := make(chan struct{})
maxQuery := make([]time.Duration, numThreads)
eg, egCtx := errgroup.WithContext(ctx)
for i := 0; i < numThreads; i++ {
eg.Go(func() error {
for {
select {
case <-stop:
return nil
default:
}
err := func() (err error) {
start := time.Now()
defer func() {
elapsed := time.Since(start)
if elapsed > maxQuery[i] {
maxQuery[i] = elapsed
}
}()
conn, err := db.Conn(egCtx)
if err != nil {
return err
}
defer func() {
err = errors.Join(err, conn.Close())
}()
branch := branches[rand.IntN(len(branches))]
_, err = conn.ExecContext(egCtx, fmt.Sprintf("select dolt_checkout('%s')", branch))
if err != nil {
return err
}
t1 := tables[rand.IntN(len(tables))]
t2 := tables[rand.IntN(len(tables))]
for t2 == t1 {
t2 = tables[rand.IntN(len(tables))]
}
c1 := columns[rand.IntN(len(columns))]
c2 := columns[rand.IntN(len(columns))]
rows, err := conn.QueryContext(egCtx, fmt.Sprintf("select * from %s join %s on %s.%s = %s.%s",
t1, t2, t1, c1, t2, c2))
if err != nil {
return err
}
defer func() {
err = errors.Join(err, rows.Close())
}()
for rows.Next() {
}
return nil
}()
if err != nil {
return err
}
}
})
}
var gcDuration time.Duration
eg.Go(func() (err error) {
defer close(stop)
start := time.Now()
defer func() {
gcDuration = time.Since(start)
}()
conn, err := db.Conn(egCtx)
if err != nil {
return err
}
defer func() {
err = errors.Join(err, conn.Close())
}()
_, err = conn.ExecContext(egCtx, "select dolt_stats_gc()")
if err != nil {
return err
}
return nil
})
require.NoError(t, eg.Wait())
for _, d := range maxQuery {
assert.Greater(t, 3*time.Second, d)
assert.Greater(t, gcDuration, d)
}
}
func TestStatsAnalyzeTableSpeed(t *testing.T) {
// Doltgres does not provide the dolt_stats_timers() procedure that this
// test uses to slow background stats refresh, so the rate-limiting scenario
// it asserts cannot be set up. The logic below is a faithful port and
// should be enabled once Doltgres supports dolt_stats_timers().
t.Skip("dolt_stats_timers() is unsupported in Doltgres")
t.Parallel()
// At one time, calling ANALYZE would go through the rate-limiting system
// that background stats refresh goes through. This would make ANALYZE take
// a longer time than necessary, and it would be very dependent on the stats
// timers that were in effect.
//
// This tests that even with very large stats timers, ANALYZE behaves
// reasonably.
ctx := context.Background()
u, err := driver.NewDoltUser()
require.NoError(t, err)
t.Cleanup(func() {
u.Cleanup()
})
rs, err := u.MakeRepoStore()
require.NoError(t, err)
_, err = rs.MakeRepo("stats_analyze_table_test")
require.NoError(t, err)
srvSettings := &driver.Server{
Args: []string{},
DynamicPort: "server",
}
ports := newPorts(t)
server := StartServer(t, rs, "stats_analyze_table_test", srvSettings, ports)
// Connect to the database
db, err := server.DB(driver.Connection{})
require.NoError(t, err)
t.Cleanup(func() {
db.Close()
})
// Task rate-limiting of one per minute will make things more
// than slow enough that ANALYZE down below will fail on
// the context timeout if these rates are affecting the
// ANALYZE call.
_, err = db.ExecContext(ctx, "select dolt_stats_timers(60000000000, 60000000000)")
require.NoError(t, err)
var columns = []string{
"c0", "c1", "c2", "c3", "c4",
"c5", "c6", "c7", "c8", "c9",
}
var columndefs = []string{}
for _, col := range columns {
columndefs = append(columndefs, fmt.Sprintf("%s bigint unique", col))
}
_, err = db.ExecContext(ctx, fmt.Sprintf(strings.Join([]string{
"CREATE TABLE %s (",
"id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY,",
strings.Join(columndefs, ",\n"),
");",
}, "\n"), "t0"))
require.NoError(t, err)
_, err = db.ExecContext(ctx, "select dolt_commit('-Am', 'initial commit')")
require.NoError(t, err)
_, err = db.ExecContext(ctx, "select dolt_checkout('main')")
require.NoError(t, err)
_, err = db.ExecContext(ctx, fmt.Sprintf("insert into %s values %s", "t0", t0_values_strs(65536)))
require.NoError(t, err)
timeoutCtx, cancel := context.WithTimeout(ctx, time.Second*30)
defer cancel()
_, err = db.ExecContext(timeoutCtx, "ANALYZE t0")
require.NoError(t, err)
}
func t0_values_str() string {
return fmt.Sprintf("(%d, %d, %d, %d, %d, %d, %d, %d, %d, %d, %d)",
rand.IntN(1<<48), rand.IntN(1<<48), rand.IntN(1<<48), rand.IntN(1<<48),
rand.IntN(1<<48), rand.IntN(1<<48), rand.IntN(1<<48), rand.IntN(1<<48),
rand.IntN(1<<48), rand.IntN(1<<48), rand.IntN(1<<48))
}
func t0_values_strs(n int) string {
vals := make([]string, n)
for i := range vals {
vals[i] = t0_values_str()
}
return strings.Join(vals, ",")
}
integration-tests/go-sql-server-driver/testdata/1pk5col-ints.csv
Vendored
+3
View File
@@ -0,0 +1,3 @@
pk,c1,c2,c3,c4,c5
0,1,2,3,4,5
1,1,2,3,4,5
1 pk c1 c2 c3 c4 c5
2 0 1 2 3 4 5
3 1 1 2 3 4 5
integration-tests/go-sql-server-driver/testdata/test_jwks.json
Vendored
+12
View File
@@ -0,0 +1,12 @@
{
"keys": [
{
"kty": "RSA",
"e": "AQAB",
"use": "sig",
"kid": "749df841-6e38-48f1-a178-20ecdd0b09f7",
"alg": "RS256",
"n": "uPU3mTbLgki6xF05WLp1xFL3t9ZfrYygcgBq5-qJhz-n5dTicxiCt1X0PvDzomqGJFap7q_yB4z97w9GcoeRXzPrGyeVN324cOfCqgyrhR7pMXfrkAY885eqp3068Wo2V7OgS0yLlzD4XRjj4A9g8ssuZKhwgDcUqdiWk3ar33e3KMnz-GYqRrfNHnHkZ-WPqSwseM5ng9YgmKy8j3JeKLG336yGIRXQGAtZdekWBCBSlT_dW9bxiLdUMzS4ENzJzmYTrfcfQY6mm9C-DTprv3Lwq1h95Yhh7taIhRcBpin6Il5sBD7OLUUGHktDR_frEqkN8a2DeL5c81HOqT9-JQ"
}
]
}
integration-tests/go-sql-server-driver/testdata/test_jwks_private.json
Vendored
+18
View File
@@ -0,0 +1,18 @@
{
"keys": [
{
"p": "-dh54E1ei0vAT6CR-JjXnpQjBQLpILN1d9CY-hFl0S56oMlt1tU46nHGM5COVsQJv8OKCFWGQJY9BixlGAk9UGKRAxSKjhePsKDFr7vOdwmTg6mrF1JFa02Zuz3f7zRAsFd1rAtDcHTMmxISaIEZuBMeOougMJ1XdbLxqjkZLLM",
"kty": "RSA",
"q": "vYOPShvpX2wHMdLU7WTXPHre4P1sKvdZzEtFbOXnsfRNLeslAxFeTd7FpHc8wRA3cB5PTMzZU7jEyj3pHmZAq-2R_gu63gLjl8erI36cAmIgzB_seFQM8licuS6TbZvlYFM2M1NNjqTASLO5feGAtSoUSJmYvndn9OhvSa-Yxcc",
"d": "FQFsfg8WD8bYx0JbJ_ONOm29yngjR5-H_UqE2a_uTJjzJYwG59Fpzw6I_bj5woFcmLXq-LusviTKFiNi-dDhtrE7y0q0jKfPkasQlaV4uVaoX0DiVOoQdA3OiNUVI6PPZih1VPftho8-NbyE7MZyWUCwFSh4FmerBhseBsNcg7VwBr4CZimmSEKMJXHLMR7jZfrVJAuFGSHJWrdIS04sPcCyBX4V7FLCRBr8BRiM7LznwP4tDHG8-bferqtXH4-XwPY3HjaHRTK7gRQspCKPv-kPdnQl2rH0zJa5kY7MQq4RDtINXSefM_sek3RrhXUXvG2JF-1JiQ3EUi59eY7ZxQ",
"e": "AQAB",
"use": "sig",
"kid": "749df841-6e38-48f1-a178-20ecdd0b09f7",
"qi": "isaLZv3bKV0Osmfl5ay6LABeefMkun1tw2AJD6Ouqo9Z7mDLaTUDP5XxwypKry3ZVd-N1ZMkfgGi6Q-O-xLq2AU5mnI5bHbDAiN3P9Jp-Tex6c6rzT24fIbGQMV-Rul8z10_FsYbFv5CWr9V_jFBNrR8gX4XGtHn1qFVBo_K5Rk",
"dp": "tRlEvmFWdoGiFBW_uQKQyFF4UNmbQijSrNZ3DEwwEUAvgvx-sYo8hzORBy9w_VN7_ZQvKXtUpNxBv4fOf22zE-FeW204QWaysMTYhlkLfx1h373MVks8JltJY3-mIi0t9qRulxZS--Ctrnma_kUV72dsMeOjaZmjG51prolUxiE",
"alg": "RS256",
"dq": "skqLC9WmgLdJLX6EA7LTK3sNI-5HTUTXnnNSJVlF2Q1VbtXCRFiat_fVSR1Ecv2mqjxZro8qBrHVsc78-jSIszcWGkM-0o81Px4By6rZawSWhnOiLLImW_kxuKYw3PXFnhGq9C5y0Lf-jmdHIz57r_SekI6wPMBpdOcXi-M_fxE",
"n": "uPU3mTbLgki6xF05WLp1xFL3t9ZfrYygcgBq5-qJhz-n5dTicxiCt1X0PvDzomqGJFap7q_yB4z97w9GcoeRXzPrGyeVN324cOfCqgyrhR7pMXfrkAY885eqp3068Wo2V7OgS0yLlzD4XRjj4A9g8ssuZKhwgDcUqdiWk3ar33e3KMnz-GYqRrfNHnHkZ-WPqSwseM5ng9YgmKy8j3JeKLG336yGIRXQGAtZdekWBCBSlT_dW9bxiLdUMzS4ENzJzmYTrfcfQY6mm9C-DTprv3Lwq1h95Yhh7taIhRcBpin6Il5sBD7OLUUGHktDR_frEqkN8a2DeL5c81HOqT9-JQ"
}
]
}
integration-tests/go-sql-server-driver/testdef.go
+704
View File
@@ -0,0 +1,704 @@
// Copyright 2024 Dolthub, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package main
import (
"bytes"
"context"
"database/sql"
"fmt"
"io"
"os"
"path/filepath"
"sync"
"testing"
"text/template"
"time"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
yaml "gopkg.in/yaml.v3"
driver "github.com/dolthub/doltgresql/integration-tests/go-sql-server-driver/driver"
)
var GlobalPorts GlobalDynamicResources
type TestDef struct {
Tests []Test `yaml:"tests"`
// If true, RunTestfile will run each subtest in parallel.
Parallel bool `yaml:"parallel"`
}
// Test is a single test to run. The Repos and MultiRepos will be created, and
// any Servers defined within them will be started. The interactions and
// assertions defined in Conns will be run.
type Test struct {
Name string `yaml:"name"`
Repos []driver.TestRepo `yaml:"repos"`
MultiRepos []driver.MultiRepo `yaml:"multi_repos"`
Conns []driver.Connection `yaml:"connections"`
// Skip the entire test with this reason.
Skip string `yaml:"skip"`
}
// Set this environment variable to effectively disable timeouts for debugging.
const debugEnvKey = "DOLTGRES_SQL_SERVER_TEST_DEBUG"
var timeout = 20 * time.Second
func init() {
_, ok := os.LookupEnv(debugEnvKey)
if ok {
timeout = 1000 * time.Hour
}
}
func ParseTestsFile(path string) (TestDef, error) {
contents, err := os.ReadFile(path)
if err != nil {
return TestDef{}, err
}
dec := yaml.NewDecoder(bytes.NewReader(contents))
dec.KnownFields(true)
var res TestDef
err = dec.Decode(&res)
return res, err
}
// MakeRepo creates the database for |r| within the store |rs| and writes any
// with_files for the repo. Unlike Dolt, the doltgres server runs from the
// data-dir (the store) and each repo is a database subdirectory, so with_files
// are written relative to the store directory.
func MakeRepo(t *testing.T, rs driver.RepoStore, r driver.TestRepo, ports *DynamicResources) driver.Repo {
repo, err := rs.MakeRepo(r.Name)
require.NoError(t, err)
for _, f := range r.WithFiles {
f.Template = func(s string) string {
return ports.ApplyTemplate(s)
}
require.NoError(t, f.WriteAtDir(rs.Dir))
}
for _, remote := range r.WithRemotes {
url := remote.URL
url = ports.ApplyTemplate(url)
require.NoError(t, repo.CreateRemote(remote.Name, url))
}
return repo
}
// Simple interface for wrapping *testing.T. Used for retryingT.
type TestingT interface {
Fatal(...any)
FailNow()
Errorf(string, ...any)
Cleanup(func())
TempDir() string
}
// Globally available dynamic ports, backs every instance of
// DynamicPorts and hands them out in a thread-safe manner.
//
// XXX: This structure and its initialization does not currently look
// for "available" ports on the running host. It simply avoids handing
// out the same port to two separate tests that are running at the
// same time. It recycles ports as tests complete.
type GlobalDynamicResources struct {
mu sync.Mutex
available []int
}
func (g *GlobalDynamicResources) GetPort(t TestingT) int {
g.mu.Lock()
defer g.mu.Unlock()
if len(g.available) == 0 {
t.Fatal("cannot get a port; we are all out.")
}
next := g.available[len(g.available)-1]
g.available = g.available[:len(g.available)-1]
return next
}
func (g *GlobalDynamicResources) Return(n int) {
g.mu.Lock()
defer g.mu.Unlock()
g.available = append(g.available, n)
}
// Tracks dynamic resources available for expansion in test
// definitions, for example through `{{get_port ...}}` templates for
// server args and config files.
type DynamicResources struct {
// Where we go when we need a new one.
global *GlobalDynamicResources
t TestingT
// Where we put allocated ports. For a given test, the first
// use will get a new unused port from
// GlobalDynamicPorts. Then that same port will be returned
// from here for all uses. When the test finishes, its Cleanup
// returns the port to GlobalDynamicResources.
allocatedPorts map[string]int
// Where we put allocated temp directories. These get removed
// on cleanup.
allocatedTempDirs map[string]string
}
func (d *DynamicResources) GetPort(name string) (int, bool) {
if d.allocatedPorts != nil {
v, ok := d.allocatedPorts[name]
return v, ok
} else {
return 0, false
}
}
func (d *DynamicResources) GetTempDir(name string) (string, bool) {
if d.allocatedTempDirs != nil {
v, ok := d.allocatedTempDirs[name]
return v, ok
} else {
return "", false
}
}
func (d *DynamicResources) GetOrAllocatePort(name string) int {
v, ok := d.GetPort(name)
if ok {
return v
}
v = d.global.GetPort(d.t)
if d.allocatedPorts == nil {
d.allocatedPorts = make(map[string]int)
// We register one cleanup function for the entire
// DynamicPorts and we return them all at once.
//
// In cases where there are two dependent servers, we
// want to return all ports after both servers have
// been shut down. If we return them as we allocated
// them, it's possible that we allocated them to
// render the entire config for the first server, some
// referring to the second server, for example. If
// testing.T runs cleanups in FIFO order, and the
// Cleanup for running the second server is
// responsible for shutting it down, it is possible we
// would return the second server's ports before it is
// shut down if we didn't return them all at once.
d.t.Cleanup(func() {
for _, p := range d.allocatedPorts {
d.global.Return(p)
}
})
}
d.allocatedPorts[name] = v
return v
}
func (d *DynamicResources) GetOrAllocateTempDir(name string) string {
v, ok := d.GetTempDir(name)
if ok {
return v
}
v = d.t.TempDir()
if d.allocatedTempDirs == nil {
d.allocatedTempDirs = make(map[string]string)
d.t.Cleanup(func() {
d.allocatedTempDirs = nil
})
}
d.allocatedTempDirs[name] = v
return v
}
func (d *DynamicResources) ApplyTemplate(s string) string {
tmpl, err := template.New("sql").Funcs(map[string]any{
"get_port": d.GetOrAllocatePort,
"get_tempdir": d.GetOrAllocateTempDir,
}).Parse(s)
require.NoError(d.t, err)
var buf bytes.Buffer
err = tmpl.Execute(&buf, nil)
require.NoError(d.t, err)
return buf.String()
}
// testLogWriter is an io.Writer that sends each line to t.Log.
// It buffers partial lines until a newline is received.
type testLogWriter struct {
t *testing.T
buf []byte
}
func (w *testLogWriter) Write(p []byte) (int, error) {
w.buf = append(w.buf, p...)
for {
i := bytes.IndexByte(w.buf, '\n')
if i < 0 {
break
}
w.t.Helper()
w.t.Log(string(w.buf[:i]))
w.buf = w.buf[i+1:]
}
return len(p), nil
}
// Flush any remaining partial line.
func (w *testLogWriter) Flush() {
if len(w.buf) > 0 {
w.t.Helper()
w.t.Log(string(w.buf))
w.buf = w.buf[:0]
}
}
func newTestLogWriter(t *testing.T) io.Writer {
w := &testLogWriter{t: t}
t.Cleanup(w.Flush)
return w
}
// prepareDoltgresServerArgs translates the server |args| from a test
// definition into arguments valid for the doltgres binary, generating a config
// file in |cwd| that fixes the listener to |port| (and a unique unix socket so
// parallel servers do not collide). doltgres has a minimal CLI (no -P, -l,
// --max-connections, etc.), so the listener port can only be set via the
// config file.
//
// If the incoming args reference a config file via `--config <path>` (or
// `--config=<path>`), that file is loaded as the base config and the listener
// settings are overlaid onto it; otherwise a fresh config is generated. The
// `--data-dir` argument is added (as `.`) if not already present.
func prepareDoltgresServerArgs(t *testing.T, cwd, name string, port int, args []string) []string {
base := map[string]any{}
var passthrough []string
configPath := ""
for i := 0; i < len(args); i++ {
a := args[i]
switch {
case a == "--config" || a == "-config":
if i+1 < len(args) {
configPath = args[i+1]
i++
}
case len(a) > 9 && a[:9] == "--config=":
configPath = a[9:]
case len(a) > 8 && a[:8] == "-config=":
configPath = a[8:]
default:
passthrough = append(passthrough, a)
}
}
if configPath != "" {
p := configPath
if !filepath.IsAbs(p) {
p = filepath.Join(cwd, p)
}
if contents, err := os.ReadFile(p); err == nil {
_ = yaml.Unmarshal(contents, &base)
} else {
require.NoError(t, err)
}
}
listener, _ := base["listener"].(map[string]any)
if listener == nil {
listener = map[string]any{}
}
listener["host"] = "127.0.0.1"
listener["port"] = port
listener["socket"] = filepath.Join(os.TempDir(), fmt.Sprintf("dg-%d.sock", port))
base["listener"] = listener
out, err := yaml.Marshal(base)
require.NoError(t, err)
genPath := filepath.Join(cwd, fmt.Sprintf(".generated-%s-config.yaml", sanitizeName(name)))
require.NoError(t, os.WriteFile(genPath, out, 0640))
hasDataDir := false
for _, a := range passthrough {
if a == "--data-dir" || a == "-data-dir" || (len(a) > 11 && a[:11] == "--data-dir=") || (len(a) > 10 && a[:10] == "-data-dir=") {
hasDataDir = true
}
}
result := append([]string{}, passthrough...)
if !hasDataDir {
result = append(result, "--data-dir=.")
}
result = append(result, "--config="+genPath)
return result
}
func sanitizeName(s string) string {
out := make([]rune, 0, len(s))
for _, r := range s {
if r == '/' || r == '\\' || r == ' ' {
out = append(out, '_')
} else {
out = append(out, r)
}
}
return string(out)
}
func MakeServer(t *testing.T, dc driver.DoltCmdable, cwd string, s *driver.Server, resources *DynamicResources, manualOps ...driver.SqlServerOpt) *driver.SqlServer {
if s == nil {
return nil
}
if s.Port != 0 {
t.Fatal("cannot specify s.Port on these tests; please use {{get_port ...}} and dynamic_port: to specify a dynamic port.")
}
if s.DynamicPort == "" {
t.Fatal("you must specify s.DynamicPort on these tests; please use {{get_port ...}} and dynamic_port: to specify a dynamic port.")
}
port := resources.GetOrAllocatePort(s.DynamicPort)
rawArgs := make([]string, len(s.Args))
for i := range rawArgs {
rawArgs[i] = resources.ApplyTemplate(s.Args[i])
}
args := prepareDoltgresServerArgs(t, cwd, s.Name, port, rawArgs)
opts := []driver.SqlServerOpt{
driver.WithArgs(args...),
driver.WithName(s.Name),
driver.WithLogWriter(newTestLogWriter(t)),
driver.WithPort(port),
}
if len(s.Envs) > 0 {
opts = append(opts, driver.WithEnvs(s.Envs...))
}
opts = append(opts, manualOps...)
var server *driver.SqlServer
var err error
if s.DebugPort != 0 {
server, err = driver.DebugSqlServer(dc, s.DebugPort, opts...)
} else {
server, err = driver.StartSqlServer(dc, opts...)
}
require.NoError(t, err)
if len(s.ErrorMatches) > 0 {
err := server.ErrorStop()
require.Error(t, err)
output := server.Output.String()
for _, a := range s.ErrorMatches {
require.Regexp(t, a, output)
}
return nil
} else {
t.Cleanup(func() {
// We use assert, not require here, since FailNow() in
// a Cleanup does not make sense.
err := server.GracefulStop()
if assert.NoError(t, err) {
output := server.Output.String()
for _, a := range s.LogMatches {
assert.Regexp(t, a, output)
}
for _, a := range s.LogNotMatches {
assert.NotRegexp(t, a, output)
}
}
})
return server
}
}
// Runs the defined test, applying its asserts.
func (test Test) Run(t *testing.T) {
if test.Skip != "" {
t.Skip(test.Skip)
}
var ports DynamicResources
ports.global = &GlobalPorts
ports.t = t
u, err := driver.NewDoltUser()
require.NoError(t, err)
t.Cleanup(func() {
u.Cleanup()
})
servers := make(map[string]*driver.SqlServer)
for _, r := range test.Repos {
// Each repo with a server gets its own store (data-dir) so that
// independent doltgres server processes do not collide on the same
// on-disk databases.
rs, err := u.MakeRepoStore()
require.NoError(t, err)
MakeRepo(t, rs, r, &ports)
if r.Server != nil {
if r.Server.Name == "" {
r.Server.Name = r.Name
}
server := MakeServer(t, rs, rs.Dir, r.Server, &ports)
if server != nil {
server.DBName = r.Name
servers[r.Name] = server
}
}
}
for _, mr := range test.MultiRepos {
// Each MultiRepo gets its own data-dir.
rs, err := u.MakeRepoStore()
require.NoError(t, err)
for _, r := range mr.Repos {
MakeRepo(t, rs, r, &ports)
}
for _, f := range mr.WithFiles {
f.Template = func(s string) string {
return ports.ApplyTemplate(s)
}
require.NoError(t, f.WriteAtDir(rs.Dir))
}
if mr.Server != nil {
if mr.Server.Name == "" {
mr.Server.Name = mr.Name
}
server := MakeServer(t, rs, rs.Dir, mr.Server, &ports)
if server != nil {
servers[mr.Name] = server
}
}
}
for i, c := range test.Conns {
server := servers[c.On]
require.NotNilf(t, server, "error in test spec: could not find server %s for connection %d", c.On, i)
if c.RetryAttempts > 1 {
RetryTestRun(t, c.RetryAttempts, func(t TestingT) {
db, err := server.DB(c)
require.NoError(t, err)
defer db.Close()
conn, err := db.Conn(context.Background())
require.NoError(t, err)
defer conn.Close()
for _, q := range c.Queries {
RunQueryAttempt(t, conn, q, &ports)
}
})
} else {
func() {
db, err := server.DB(c)
require.NoError(t, err)
defer db.Close()
conn, err := db.Conn(context.Background())
require.NoError(t, err)
defer conn.Close()
for _, q := range c.Queries {
RunQuery(t, conn, q, &ports)
}
}()
}
if c.RestartServer != nil {
args := c.RestartServer.Args
if args != nil {
tmplArgs := make([]string, len(*args))
for i := range tmplArgs {
tmplArgs[i] = ports.ApplyTemplate((*args)[i])
}
prepared := prepareDoltgresServerArgs(t, server.Cmd.Dir, server.Name, server.Port, tmplArgs)
args = &prepared
}
err := server.Restart(args, c.RestartServer.Envs)
require.NoError(t, err)
}
}
}
func RunTestsFile(t *testing.T, path string) {
def, err := ParseTestsFile(path)
require.NoError(t, err)
parallel := def.Parallel
for _, test := range def.Tests {
t.Run(test.Name, func(t *testing.T) {
if parallel {
t.Parallel()
}
test.Run(t)
})
}
}
func RunSingleTest(t *testing.T, path string, testName string) {
def, err := ParseTestsFile(path)
require.NoError(t, err)
for _, test := range def.Tests {
if test.Name == testName {
t.Run(test.Name, test.Run)
}
}
}
type retryTestingT struct {
*testing.T
errorfStrings []string
errorfArgs [][]interface{}
failNow bool
}
func (r *retryTestingT) Errorf(format string, args ...interface{}) {
r.T.Helper()
r.errorfStrings = append(r.errorfStrings, format)
r.errorfArgs = append(r.errorfArgs, args)
}
func (r *retryTestingT) FailNow() {
r.T.Helper()
r.failNow = true
panic(r)
}
func (r *retryTestingT) try(attempts int, test func(TestingT)) {
for i := 0; i < attempts; i++ {
r.errorfStrings = nil
r.errorfArgs = nil
r.failNow = false
if i != 0 {
time.Sleep(driver.RetrySleepDuration)
}
func() {
defer func() {
if r := recover(); r != nil {
if _, ok := r.(*retryTestingT); ok {
} else {
panic(r)
}
}
}()
test(r)
}()
if !r.failNow && len(r.errorfStrings) == 0 {
return
}
}
for i := range r.errorfStrings {
r.T.Errorf(r.errorfStrings[i], r.errorfArgs[i]...)
}
if r.failNow {
r.T.FailNow()
}
}
func RetryTestRun(t *testing.T, attempts int, test func(TestingT)) {
if attempts == 0 {
attempts = 1
}
rtt := &retryTestingT{T: t}
rtt.try(attempts, test)
}
func RunQuery(t *testing.T, conn *sql.Conn, q driver.Query, ports *DynamicResources) {
RetryTestRun(t, q.RetryAttempts, func(t TestingT) {
RunQueryAttempt(t, conn, q, ports)
})
}
func RunQueryAttempt(t TestingT, conn *sql.Conn, q driver.Query, ports *DynamicResources) {
args := make([]any, len(q.Args))
for i := range q.Args {
args[i] = q.Args[i]
}
if q.Query != "" {
ctx, c := context.WithTimeout(context.Background(), timeout)
defer c()
rows, err := conn.QueryContext(ctx, q.Query, args...)
if err == nil {
defer rows.Close()
}
if q.ErrorMatch != "" {
require.Error(t, err, "expected error running query %s", q.Query)
require.Regexp(t, q.ErrorMatch, err.Error())
return
}
require.NoError(t, err)
cols, err := rows.Columns()
require.NoError(t, err)
require.Equal(t, q.Result.Columns, cols)
rowstrings, err := RowsToStrings(len(cols), rows)
require.NoError(t, err)
if q.Result.Rows.Or != nil {
match := *q.Result.Rows.Or
for i := range match {
for j := range match[i] {
for k := range match[i][j] {
match[i][j][k] = ports.ApplyTemplate(match[i][j][k])
}
}
}
require.Contains(t, match, rowstrings)
}
} else if q.Exec != "" {
ctx, c := context.WithTimeout(context.Background(), timeout)
defer c()
exec := q.Exec
exec = ports.ApplyTemplate(exec)
_, err := conn.ExecContext(ctx, exec, args...)
if q.ErrorMatch == "" {
require.NoError(t, err, "error running query %s: %v", q.Exec, err)
} else {
require.Error(t, err)
require.Regexp(t, q.ErrorMatch, err.Error())
}
}
}
func RowsToStrings(cols int, rows *sql.Rows) ([][]string, error) {
ret := make([][]string, 0)
for rows.Next() {
scanned := make([]any, cols)
for j := range scanned {
scanned[j] = new(sql.NullString)
}
err := rows.Scan(scanned...)
if err != nil {
return nil, err
}
printed := make([]string, cols)
for j := range scanned {
s := scanned[j].(*sql.NullString)
if !s.Valid {
printed[j] = "NULL"
} else {
printed[j] = s.String
}
}
ret = append(ret, printed)
}
return ret, rows.Err()
}
integration-tests/go-sql-server-driver/tests/README.md
+60
View File
@@ -0,0 +1,60 @@
# go-sql-server-driver tests (Doltgres port)
These tests are ported from Dolt's
`integration-tests/go-sql-server-driver`. They exercise a real `doltgres`
server process over the Postgres wire protocol, driven either by the YAML test
definition format (`tests/*.yaml`, run via `testdef.go`) or by standalone Go
tests that use the `driver` package directly.
## Running
The tests locate the `doltgres` binary via the `DOLTGRES_BIN_PATH` environment
variable, falling back to `doltgres` on the `PATH`.
```
DOLTGRES_BIN_PATH=/path/to/doltgres go test ./...
```
## Differences from the Dolt suite
The port preserves the structure and intent of the Dolt tests, translating only
what the Postgres dialect and wire protocol, and the doltgres binary, require:
* **Server CLI.** `doltgres` has a minimal CLI (`--config`, `--data-dir`) and
*no* `-P`/`-l`/`--max-connections` flags. The listener port can only be set
via the config file, so the test framework (`MakeServer` in `testdef.go`)
generates/merges a config file that injects the dynamic port, host, and a
unique unix socket. Ported YAML expresses server settings via a config file
(referenced with `--config`) rather than via Dolt CLI flags.
* **Data-dir model.** Doltgres serves databases out of a data-dir; each "repo"
is a database subdirectory. The server runs from the store (data-dir) with
`--data-dir=.`, and `with_files` are written relative to the store directory.
Databases are initialized by briefly running a server (there is no
`dolt init` equivalent).
* **Wire protocol.** Connections use the pgx stdlib driver. Default user is
`postgres`, password `password`.
* **SQL dialect.** MySQL syntax is translated to Postgres (e.g.
`AUTO_INCREMENT` -> `SERIAL`/`GENERATED`, backtick identifiers -> unquoted or
double-quoted, `int`/`varchar(n)` are fine, `select ... from dual` ->
`select ...`).
* **DOLT procedures.** Dolt's `CALL DOLT_*(...)` / `SELECT DOLT_*()` become
`SELECT dolt_*(...)` in Doltgres. Result columns default to the function name
(e.g. `dolt_add`), and the value is rendered as a Postgres array, e.g.
`{0}`.
* **Result columns.** Postgres lowercases unquoted identifiers and uses the
expression text / alias for computed columns; expected `columns:` and `rows:`
are adjusted accordingly. Prefer explicit `AS` aliases.
* **Cluster / remotesapi replication.** Doltgres does not yet implement Dolt's
cluster replication or the remotes API. Config structs for these are mirrored
into doltgres so the config files parse, but the tests that depend on the
feature are marked `skip:` with a reason until the feature lands.
When a ported test does not pass yet (missing feature or behavioral
difference), it is marked with `skip:` (YAML) or `t.Skip(...)` (Go) with a short
reason, per the porting instructions.
integration-tests/go-sql-server-driver/tests/sql-server-cluster-read-only.yaml
+185
View File
@@ -0,0 +1,185 @@
parallel: false
tests:
- name: users and grants cannot be run on standby
skip: "cluster replication / MySQL grants not yet supported in Doltgres"
multi_repos:
- name: server1
with_files:
- name: server.yaml
contents: |
log_level: trace
cluster:
standby_remotes:
- name: standby
remote_url_template: http://localhost:{{get_port "server2_cluster"}}/{database}
bootstrap_role: primary
bootstrap_epoch: 1
remotesapi:
port: {{get_port "server1_cluster"}}
server:
args: ["--config", "server.yaml"]
dynamic_port: server1
- name: server2
with_files:
- name: server.yaml
contents: |
log_level: trace
cluster:
standby_remotes:
- name: standby
remote_url_template: http://localhost:{{get_port "server1_cluster"}}/{database}
bootstrap_role: standby
bootstrap_epoch: 1
remotesapi:
port: {{get_port "server2_cluster"}}
server:
args: ["--config", "server.yaml"]
dynamic_port: server2
connections:
- on: server2
queries:
- exec: 'CREATE USER "brian" WITH PASSWORD ''brianspassword'''
error_match: 'database server is set to read only mode'
- exec: 'GRANT ALL PRIVILEGES ON ALL TABLES IN SCHEMA public TO "aaron"'
error_match: 'database server is set to read only mode'
- name: create database cannot be run on standby
skip: "cluster replication not yet implemented in Doltgres"
multi_repos:
- name: server1
with_files:
- name: server.yaml
contents: |
log_level: trace
cluster:
standby_remotes:
- name: standby
remote_url_template: http://localhost:{{get_port "server2_cluster"}}/{database}
bootstrap_role: primary
bootstrap_epoch: 1
remotesapi:
port: {{get_port "server1_cluster"}}
server:
args: ["--config", "server.yaml"]
dynamic_port: server1
- name: server2
with_files:
- name: server.yaml
contents: |
log_level: trace
cluster:
standby_remotes:
- name: standby
remote_url_template: http://localhost:{{get_port "server1_cluster"}}/{database}
bootstrap_role: standby
bootstrap_epoch: 1
remotesapi:
port: {{get_port "server2_cluster"}}
server:
args: ["--config", "server.yaml"]
dynamic_port: server2
connections:
- on: server2
queries:
- exec: 'CREATE DATABASE my_db'
error_match: 'database server is set to read only mode'
- name: drop database cannot be run on standby
skip: "cluster replication not yet implemented in Doltgres"
multi_repos:
- name: server1
with_files:
- name: server.yaml
contents: |
log_level: trace
cluster:
standby_remotes:
- name: standby
remote_url_template: http://localhost:{{get_port "server2_cluster"}}/{database}
bootstrap_role: primary
bootstrap_epoch: 1
remotesapi:
port: {{get_port "server1_cluster"}}
server:
args: ["--config", "server.yaml"]
dynamic_port: server1
- name: server2
with_files:
- name: server.yaml
contents: |
log_level: trace
cluster:
standby_remotes:
- name: standby
remote_url_template: http://localhost:{{get_port "server1_cluster"}}/{database}
bootstrap_role: standby
bootstrap_epoch: 1
remotesapi:
port: {{get_port "server2_cluster"}}
server:
args: ["--config", "server.yaml"]
dynamic_port: server2
connections:
- on: server1
queries:
- exec: "SET dolt_cluster_ack_writes_timeout_secs = 10"
- exec: 'CREATE DATABASE repo1'
- exec: 'USE repo1'
- exec: 'CREATE TABLE vals (i INT PRIMARY KEY)'
- exec: 'INSERT INTO vals VALUES (0),(1),(2),(3),(4)'
- on: server2
queries:
- exec: 'DROP DATABASE repo1'
error_match: 'database server is set to read only mode'
- name: when a server becomes primary it accepts writes
skip: "cluster replication not yet implemented in Doltgres"
multi_repos:
- name: server1
with_files:
- name: server.yaml
contents: |
log_level: trace
cluster:
standby_remotes:
- name: standby
remote_url_template: http://localhost:{{get_port "server2_cluster"}}/{database}
bootstrap_role: primary
bootstrap_epoch: 1
remotesapi:
port: {{get_port "server1_cluster"}}
server:
args: ["--config", "server.yaml"]
dynamic_port: server1
- name: server2
with_files:
- name: server.yaml
contents: |
log_level: trace
cluster:
standby_remotes:
- name: standby
remote_url_template: http://localhost:{{get_port "server1_cluster"}}/{database}
bootstrap_role: standby
bootstrap_epoch: 1
remotesapi:
port: {{get_port "server2_cluster"}}
server:
args: ["--config", "server.yaml"]
dynamic_port: server2
connections:
- on: server2
queries:
- exec: "select dolt_assume_cluster_role('primary', 2)"
- on: server2
queries:
- exec: "SET dolt_cluster_ack_writes_timeout_secs = 10"
- exec: 'CREATE DATABASE repo1'
- exec: 'USE repo1'
- exec: 'CREATE TABLE vals (i INT PRIMARY KEY)'
- exec: 'INSERT INTO vals VALUES (0),(1),(2),(3),(4)'
- on: server1
queries:
- exec: 'USE repo1'
- query: 'SELECT COUNT(*) as count FROM vals'
result:
columns: ["count"]
rows:
- ["5"]
integration-tests/go-sql-server-driver/tests/sql-server-cluster-tls.yaml
+460
View File
@@ -0,0 +1,460 @@
parallel: true
tests:
- name: tls, bad root, failover to standby fails
skip: "cluster replication not yet implemented in Doltgres"
multi_repos:
- name: server1
with_files:
- name: server.yaml
contents: |
log_level: trace
cluster:
standby_remotes:
- name: standby
remote_url_template: https://localhost:{{get_port "server2_cluster"}}/{database}
bootstrap_role: primary
bootstrap_epoch: 1
remotesapi:
port: {{get_port "server1_cluster"}}
tls_key: key.pem
tls_cert: cert.pem
tls_ca: root.pem
- name: key.pem
source_path: $TESTGENDIR/rsa_key.pem
- name: cert.pem
source_path: $TESTGENDIR/rsa_chain.pem
- name: root.pem
source_path: $TESTGENDIR/ed25519_root.pem
server:
args: ["--config", "server.yaml"]
dynamic_port: server1
- name: server2
with_files:
- name: server.yaml
contents: |
log_level: trace
cluster:
standby_remotes:
- name: standby
remote_url_template: https://localhost:{{get_port "server1_cluster"}}/{database}
bootstrap_role: standby
bootstrap_epoch: 1
remotesapi:
port: {{get_port "server2_cluster"}}
tls_key: key.pem
tls_cert: cert.pem
tls_ca: root.pem
- name: key.pem
source_path: $TESTGENDIR/rsa_key.pem
- name: cert.pem
source_path: $TESTGENDIR/rsa_chain.pem
- name: root.pem
source_path: $TESTGENDIR/rsa_root.pem
server:
args: ["--config", "server.yaml"]
dynamic_port: server2
connections:
- on: server1
queries:
- exec: 'create database repo1'
- exec: "use repo1"
- query: "select dolt_assume_cluster_role('standby', '11')"
error_match: failed to transition from primary to standby gracefully
- exec: "create table vals (i int primary key)"
- exec: "insert into vals values (0)"
- name: tls, expired leaf, failover to standby fails
skip: "cluster replication not yet implemented in Doltgres"
multi_repos:
- name: server1
with_files:
- name: server.yaml
contents: |
log_level: trace
cluster:
standby_remotes:
- name: standby
remote_url_template: https://localhost:{{get_port "server2_cluster"}}/{database}
bootstrap_role: primary
bootstrap_epoch: 1
remotesapi:
port: {{get_port "server1_cluster"}}
tls_key: key.pem
tls_cert: cert.pem
tls_ca: root.pem
- name: key.pem
source_path: $TESTGENDIR/rsa_key.pem
- name: cert.pem
source_path: $TESTGENDIR/rsa_chain.pem
- name: root.pem
source_path: $TESTGENDIR/rsa_root.pem
server:
args: ["--config", "server.yaml"]
dynamic_port: server1
- name: server2
with_files:
- name: server.yaml
contents: |
log_level: trace
cluster:
standby_remotes:
- name: standby
remote_url_template: https://localhost:{{get_port "server1_cluster"}}/{database}
bootstrap_role: standby
bootstrap_epoch: 1
remotesapi:
port: {{get_port "server2_cluster"}}
tls_key: key.pem
tls_cert: cert.pem
tls_ca: root.pem
- name: key.pem
source_path: $TESTGENDIR/rsa_exp_key.pem
- name: cert.pem
source_path: $TESTGENDIR/rsa_exp_chain.pem
- name: root.pem
source_path: $TESTGENDIR/rsa_root.pem
server:
args: ["--config", "server.yaml"]
dynamic_port: server2
connections:
- on: server1
queries:
- exec: 'create database repo1'
- exec: "use repo1"
- query: "select dolt_assume_cluster_role('standby', '11')"
error_match: failed to transition from primary to standby gracefully
- exec: "create table vals (i int primary key)"
- exec: "insert into vals values (0)"
- name: tls, mismatched dns, failover to standby fails
skip: "cluster replication not yet implemented in Doltgres"
multi_repos:
- name: server1
with_files:
- name: server.yaml
contents: |
log_level: trace
cluster:
standby_remotes:
- name: standby
remote_url_template: https://localhost:{{get_port "server2_cluster"}}/{database}
bootstrap_role: primary
bootstrap_epoch: 1
remotesapi:
port: {{get_port "server1_cluster"}}
tls_key: key.pem
tls_cert: cert.pem
tls_ca: root.pem
server_name_dns: ["does-not-match.dolt-instance.dolt-integration-test.example"]
- name: key.pem
source_path: $TESTGENDIR/rsa_key.pem
- name: cert.pem
source_path: $TESTGENDIR/rsa_chain.pem
- name: root.pem
source_path: $TESTGENDIR/rsa_root.pem
server:
args: ["--config", "server.yaml"]
dynamic_port: server1
- name: server2
with_files:
- name: server.yaml
contents: |
log_level: trace
cluster:
standby_remotes:
- name: standby
remote_url_template: https://localhost:{{get_port "server1_cluster"}}/{database}
bootstrap_role: standby
bootstrap_epoch: 1
remotesapi:
port: {{get_port "server2_cluster"}}
tls_key: key.pem
tls_cert: cert.pem
tls_ca: root.pem
- name: key.pem
source_path: $TESTGENDIR/rsa_key.pem
- name: cert.pem
source_path: $TESTGENDIR/rsa_chain.pem
- name: root.pem
source_path: $TESTGENDIR/rsa_root.pem
server:
args: ["--config", "server.yaml"]
dynamic_port: server2
connections:
- on: server1
queries:
- exec: 'create database repo1'
- exec: "use repo1"
- query: "select dolt_assume_cluster_role('standby', '11')"
error_match: failed to transition from primary to standby gracefully
- exec: "create table vals (i int primary key)"
- exec: "insert into vals values (0)"
- name: tls, mismatched url, failover to standby fails
skip: "cluster replication not yet implemented in Doltgres"
multi_repos:
- name: server1
with_files:
- name: server.yaml
contents: |
log_level: trace
cluster:
standby_remotes:
- name: standby
remote_url_template: https://localhost:{{get_port "server2_cluster"}}/{database}
bootstrap_role: primary
bootstrap_epoch: 1
remotesapi:
port: {{get_port "server1_cluster"}}
tls_key: key.pem
tls_cert: cert.pem
tls_ca: root.pem
server_name_urls: ["spiffe://dolt-integration-tests.dev.trust.dolthub.com.example/dolt-instance/does-not-match"]
- name: key.pem
source_path: $TESTGENDIR/rsa_key.pem
- name: cert.pem
source_path: $TESTGENDIR/rsa_chain.pem
- name: root.pem
source_path: $TESTGENDIR/rsa_root.pem
server:
args: ["--config", "server.yaml"]
dynamic_port: server1
- name: server2
with_files:
- name: server.yaml
contents: |
log_level: trace
cluster:
standby_remotes:
- name: standby
remote_url_template: https://localhost:{{get_port "server1_cluster"}}/{database}
bootstrap_role: standby
bootstrap_epoch: 1
remotesapi:
port: {{get_port "server2_cluster"}}
tls_key: key.pem
tls_cert: cert.pem
tls_ca: root.pem
- name: key.pem
source_path: $TESTGENDIR/rsa_key.pem
- name: cert.pem
source_path: $TESTGENDIR/rsa_chain.pem
- name: root.pem
source_path: $TESTGENDIR/rsa_root.pem
server:
args: ["--config", "server.yaml"]
dynamic_port: server2
connections:
- on: server1
queries:
- exec: 'create database repo1'
- exec: "use repo1"
- query: "select dolt_assume_cluster_role('standby', '11')"
error_match: failed to transition from primary to standby gracefully
- exec: "create table vals (i int primary key)"
- exec: "insert into vals values (0)"
- name: tls, good rsa certs, create new database, primary replicates to standby, fails over, new primary replicates to standby, fails over, new primary has all writes
skip: "cluster replication not yet implemented in Doltgres"
multi_repos:
- name: server1
with_files:
- name: server.yaml
contents: |
log_level: trace
cluster:
standby_remotes:
- name: standby
remote_url_template: https://localhost:{{get_port "server2_cluster"}}/{database}
bootstrap_role: primary
bootstrap_epoch: 1
remotesapi:
port: {{get_port "server1_cluster"}}
tls_key: key.pem
tls_cert: cert.pem
tls_ca: root.pem
- name: key.pem
source_path: $TESTGENDIR/rsa_key.pem
- name: cert.pem
source_path: $TESTGENDIR/rsa_chain.pem
- name: root.pem
source_path: $TESTGENDIR/rsa_root.pem
server:
args: ["--config", "server.yaml"]
dynamic_port: server1
- name: server2
with_files:
- name: server.yaml
contents: |
log_level: trace
cluster:
standby_remotes:
- name: standby
remote_url_template: https://localhost:{{get_port "server1_cluster"}}/{database}
bootstrap_role: standby
bootstrap_epoch: 1
remotesapi:
port: {{get_port "server2_cluster"}}
tls_key: key.pem
tls_cert: cert.pem
tls_ca: root.pem
- name: key.pem
source_path: $TESTGENDIR/rsa_key.pem
- name: cert.pem
source_path: $TESTGENDIR/rsa_chain.pem
- name: root.pem
source_path: $TESTGENDIR/rsa_root.pem
server:
args: ["--config", "server.yaml"]
dynamic_port: server2
connections:
- on: server1
queries:
- exec: 'create database repo1'
- exec: 'use repo1'
- exec: 'create table vals (i int primary key)'
- exec: 'insert into vals values (0),(1),(2),(3),(4)'
- query: "select dolt_assume_cluster_role('standby', 2)"
result:
columns: ["dolt_assume_cluster_role"]
rows: [["{0}"]]
- on: server2
queries:
- exec: 'use repo1'
- query: "select count(*) as count from vals"
result:
columns: ["count"]
rows: [["5"]]
- query: "select dolt_assume_cluster_role('primary', 2)"
result:
columns: ["dolt_assume_cluster_role"]
rows: [["{0}"]]
- on: server2
queries:
- exec: 'use repo1'
- exec: 'insert into vals values (5),(6),(7),(8),(9)'
- query: "select dolt_assume_cluster_role('standby', 3)"
result:
columns: ["dolt_assume_cluster_role"]
rows: [["{0}"]]
- on: server1
queries:
- exec: 'use repo1'
- query: "select count(*) as count from vals"
result:
columns: ["count"]
rows: [["10"]]
- query: "select dolt_assume_cluster_role('primary', 3)"
result:
columns: ["dolt_assume_cluster_role"]
rows: [["{0}"]]
- on: server1
queries:
- exec: 'use repo1'
- exec: 'insert into vals values (10),(11),(12),(13),(14)'
- query: "select count(*) as count from vals"
result:
columns: ["count"]
rows: [["15"]]
- name: tls, good ed25519 certs, create new database, primary replicates to standby, fails over, new primary replicates to standby, fails over, new primary has all writes
skip: "cluster replication not yet implemented in Doltgres"
multi_repos:
- name: server1
with_files:
- name: server.yaml
contents: |
log_level: trace
cluster:
standby_remotes:
- name: standby
remote_url_template: https://localhost:{{get_port "server2_cluster"}}/{database}
bootstrap_role: primary
bootstrap_epoch: 1
remotesapi:
port: {{get_port "server1_cluster"}}
tls_key: key.pem
tls_cert: cert.pem
tls_ca: root.pem
server_name_urls: ["spiffe://dolt-integration-tests.dev.trust.dolthub.com.example/dolt-instance"]
server_name_dns: ["dolt-instance.dolt-integration-test.example"]
- name: key.pem
source_path: $TESTGENDIR/ed25519_key.pem
- name: cert.pem
source_path: $TESTGENDIR/ed25519_chain.pem
- name: root.pem
source_path: $TESTGENDIR/ed25519_root.pem
server:
args: ["--config", "server.yaml"]
dynamic_port: server1
- name: server2
with_files:
- name: server.yaml
contents: |
log_level: trace
cluster:
standby_remotes:
- name: standby
remote_url_template: https://localhost:{{get_port "server1_cluster"}}/{database}
bootstrap_role: standby
bootstrap_epoch: 1
remotesapi:
port: {{get_port "server2_cluster"}}
tls_key: key.pem
tls_cert: cert.pem
tls_ca: root.pem
server_name_urls: ["spiffe://dolt-integration-tests.dev.trust.dolthub.com.example/dolt-instance"]
server_name_dns: ["dolt-instance.dolt-integration-test.example"]
- name: key.pem
source_path: $TESTGENDIR/ed25519_key.pem
- name: cert.pem
source_path: $TESTGENDIR/ed25519_chain.pem
- name: root.pem
source_path: $TESTGENDIR/ed25519_root.pem
server:
args: ["--config", "server.yaml"]
dynamic_port: server2
connections:
- on: server1
queries:
- exec: 'create database repo1'
- exec: 'use repo1'
- exec: 'create table vals (i int primary key)'
- exec: 'insert into vals values (0),(1),(2),(3),(4)'
- query: "select dolt_assume_cluster_role('standby', 2)"
result:
columns: ["dolt_assume_cluster_role"]
rows: [["{0}"]]
- on: server2
queries:
- exec: 'use repo1'
- query: "select count(*) as count from vals"
result:
columns: ["count"]
rows: [["5"]]
- query: "select dolt_assume_cluster_role('primary', 2)"
result:
columns: ["dolt_assume_cluster_role"]
rows: [["{0}"]]
- on: server2
queries:
- exec: 'use repo1'
- exec: 'insert into vals values (5),(6),(7),(8),(9)'
- query: "select dolt_assume_cluster_role('standby', 3)"
result:
columns: ["dolt_assume_cluster_role"]
rows: [["{0}"]]
- on: server1
queries:
- exec: 'use repo1'
- query: "select count(*) as count from vals"
result:
columns: ["count"]
rows: [["10"]]
- query: "select dolt_assume_cluster_role('primary', 3)"
result:
columns: ["dolt_assume_cluster_role"]
rows: [["{0}"]]
- on: server1
queries:
- exec: 'use repo1'
- exec: 'insert into vals values (10),(11),(12),(13),(14)'
- query: "select count(*) as count from vals"
result:
columns: ["count"]
rows: [["15"]]
integration-tests/go-sql-server-driver/tests/sql-server-cluster-users-and-grants.yaml
+418
View File
@@ -0,0 +1,418 @@
parallel: true
tests:
- name: users and grants replicate
skip: "cluster replication / MySQL grants not yet supported in Doltgres"
multi_repos:
- name: server1
with_files:
- name: server.yaml
contents: |
log_level: trace
cluster:
standby_remotes:
- name: standby
remote_url_template: http://localhost:{{get_port "server2_cluster"}}/{database}
bootstrap_role: primary
bootstrap_epoch: 1
remotesapi:
port: {{get_port "server1_cluster"}}
server:
args: ["--config", "server.yaml"]
dynamic_port: server1
- name: server2
with_files:
- name: server.yaml
contents: |
log_level: trace
cluster:
standby_remotes:
- name: standby
remote_url_template: http://localhost:{{get_port "server1_cluster"}}/{database}
bootstrap_role: standby
bootstrap_epoch: 1
remotesapi:
port: {{get_port "server2_cluster"}}
server:
args: ["--config", "server.yaml"]
dynamic_port: server2
connections:
- on: server1
queries:
- exec: "SET dolt_cluster_ack_writes_timeout_secs = 10"
- exec: 'create database repo1'
- exec: "use repo1"
- exec: 'create table vals (i int primary key)'
- exec: 'insert into vals values (0),(1),(2),(3),(4)'
- exec: 'create user aaron with password ''aaronspassword'''
- exec: 'GRANT ALL PRIVILEGES ON ALL TABLES IN SCHEMA public TO aaron'
- exec: 'insert into vals values (5),(6),(7),(8),(9)'
- on: server1
user: 'aaron'
password: 'aaronspassword'
queries:
- exec: "use repo1"
- exec: 'insert into vals values (10),(11),(12),(13),(14)'
- on: server2
user: 'aaron'
password: 'aaronspassword'
queries:
- exec: "use repo1"
- query: 'select count(*) as count from vals'
result:
columns: ["count"]
rows: [["15"]]
- name: branch control replicates
skip: "cluster replication / MySQL grants not yet supported in Doltgres"
multi_repos:
- name: server1
with_files:
- name: server.yaml
contents: |
log_level: trace
cluster:
standby_remotes:
- name: standby
remote_url_template: http://localhost:{{get_port "server2_cluster"}}/{database}
bootstrap_role: primary
bootstrap_epoch: 1
remotesapi:
port: {{get_port "server1_cluster"}}
server:
args: ["--config", "server.yaml"]
dynamic_port: server1
- name: server2
with_files:
- name: server.yaml
contents: |
log_level: trace
cluster:
standby_remotes:
- name: standby
remote_url_template: http://localhost:{{get_port "server1_cluster"}}/{database}
bootstrap_role: standby
bootstrap_epoch: 1
remotesapi:
port: {{get_port "server2_cluster"}}
server:
args: ["--config", "server.yaml"]
dynamic_port: server2
connections:
- on: server1
queries:
- exec: "SET dolt_cluster_ack_writes_timeout_secs = 10"
- exec: 'create database repo1'
- exec: "use repo1"
- exec: 'create table vals (i int primary key)'
- exec: 'insert into vals values (0),(1),(2),(3),(4)'
- exec: 'create user aaron with password ''aaronspassword'''
- exec: 'GRANT ALL PRIVILEGES ON ALL TABLES IN SCHEMA public TO aaron'
- exec: 'create user brian with password ''brianpassword'''
- exec: 'GRANT ALL PRIVILEGES ON ALL TABLES IN SCHEMA public TO brian'
- exec: 'delete from dolt_branch_control'
- exec: 'insert into dolt_branch_control values (''repo1'', ''main'', ''aaron'', ''%'', ''admin'')'
- on: server1
user: 'aaron'
password: 'aaronspassword'
queries:
- exec: "use repo1"
- exec: 'insert into vals values (5),(6),(7),(8),(9)'
- exec: 'insert into vals values (10),(11),(12),(13),(14)'
# Assert brian cannot write to `main` on server1
- on: server1
user: 'brian'
password: 'brianpassword'
queries:
- exec: "use repo1"
- exec: 'insert into vals values (15),(16),(17),(18),(19)'
error_match: 'does not have the correct permissions on branch `main`'
# Failover to `server2` so we can assert brian is restricted there as well.
- on: server1
queries:
- exec: "use repo1"
- exec: "select dolt_assume_cluster_role('standby', 2)"
- on: server2
queries:
- exec: "use repo1"
- exec: "select dolt_assume_cluster_role('primary', 2)"
# Assert aaron can write to main on server2 and brian cannot.
- on: server2
user: 'aaron'
password: 'aaronspassword'
queries:
- exec: "use repo1"
- query: 'select count(*) as count from vals'
result:
columns: ["count"]
rows: [["15"]]
- exec: 'insert into vals values (20),(21),(22),(23),(24)'
- on: server2
user: 'brian'
password: 'brianpassword'
queries:
- exec: "use repo1"
- query: 'select count(*) as count from vals'
result:
columns: ["count"]
rows: [["20"]]
- exec: 'insert into vals values (25),(26),(27),(28),(29)'
error_match: 'does not have the correct permissions on branch `main`'
# Now we are going to grant brian permission, and fail back over to server1,
# and ensure brian can write to the branch there.
- on: server2
user: 'aaron'
password: 'aaronspassword'
queries:
- exec: "use repo1"
- exec: 'insert into dolt_branch_control values (''repo1'', ''main'', ''brian'', ''%'', ''write'')'
- exec: 'insert into vals values (30)'
- on: server2
queries:
- exec: "select dolt_assume_cluster_role('standby', 3)"
- on: server1
queries:
- exec: "use repo1"
- exec: "select dolt_assume_cluster_role('primary', 3)"
# Here brian should be allowed to write now.
- on: server1
user: 'brian'
password: 'brianpassword'
queries:
- exec: "use repo1"
- exec: 'insert into vals values (31)'
- name: branch control block on write replication
skip: "cluster replication / MySQL grants not yet supported in Doltgres"
multi_repos:
- name: server1
with_files:
- name: server.yaml
contents: |
log_level: trace
cluster:
standby_remotes:
- name: standby
remote_url_template: http://localhost:{{get_port "server2_cluster"}}/{database}
bootstrap_role: primary
bootstrap_epoch: 1
remotesapi:
port: {{get_port "server1_cluster"}}
server:
args: ["--config", "server.yaml"]
dynamic_port: server1
- name: server2
with_files:
- name: server.yaml
contents: |
log_level: trace
cluster:
standby_remotes:
- name: standby
remote_url_template: http://localhost:{{get_port "server1_cluster"}}/{database}
bootstrap_role: standby
bootstrap_epoch: 1
remotesapi:
port: {{get_port "server2_cluster"}}
- name: nocluster.yaml
contents: |
log_level: trace
server:
args: ["--config", "nocluster.yaml"]
dynamic_port: server2
connections:
- on: server1
queries:
- exec: 'create user aaron with password ''aaronspassword'''
- exec: 'create database repo1'
- exec: 'use repo1'
- exec: "SET dolt_cluster_ack_writes_timeout_secs = 2"
- exec: 'delete from dolt_branch_control'
- query: 'show warnings'
result:
columns: ["Level", "Code", "Message"]
rows: [["Warning", "3024", "Timed out replication of commit to 1 out of 1 replicas."]]
- exec: 'insert into dolt_branch_control values (''repo1'', ''main'', ''aaron'', ''%'', ''admin'')'
- on: server2
restart_server:
args: ["--config", "server.yaml"]
- on: server1
restart_server: {}
- on: server1
queries:
- exec: "SET dolt_cluster_ack_writes_timeout_secs = 10"
- exec: 'use repo1'
- exec: 'delete from dolt_branch_control'
- query: 'show warnings'
result:
columns: ["Level", "Code", "Message"]
rows: []
- on: server2
queries:
- query: 'show warnings'
result:
columns: ["Level", "Code", "Message"]
rows: []
- name: users and grants block on write replication
skip: "cluster replication / MySQL grants not yet supported in Doltgres"
multi_repos:
- name: server1
with_files:
- name: server.yaml
contents: |
log_level: trace
cluster:
standby_remotes:
- name: standby
remote_url_template: http://localhost:{{get_port "server2_cluster"}}/{database}
bootstrap_role: primary
bootstrap_epoch: 1
remotesapi:
port: {{get_port "server1_cluster"}}
server:
args: ["--config", "server.yaml"]
dynamic_port: server1
- name: server2
with_files:
- name: server.yaml
contents: |
log_level: trace
cluster:
standby_remotes:
- name: standby
remote_url_template: http://localhost:{{get_port "server1_cluster"}}/{database}
bootstrap_role: standby
bootstrap_epoch: 1
remotesapi:
port: {{get_port "server2_cluster"}}
- name: nocluster.yaml
contents: |
log_level: trace
server:
args: ["--config", "nocluster.yaml"]
dynamic_port: server2
connections:
- on: server1
queries:
- exec: "SET dolt_cluster_ack_writes_timeout_secs = 2"
- exec: 'create user aaron with password ''aaronspassword'''
- query: 'show warnings'
result:
columns: ["Level", "Code", "Message"]
rows: [["Warning", "3024", "Timed out replication of commit to 1 out of 1 replicas."]]
- on: server2
restart_server:
args: ["--config", "server.yaml"]
- on: server1
restart_server: {}
- on: server1
queries:
# Because we run in parallel, do not be too agressive about the timeout...
- exec: "SET dolt_cluster_ack_writes_timeout_secs = 10"
- exec: 'create user brian with password ''brianspassword'''
- query: 'show warnings'
result:
columns: ["Level", "Code", "Message"]
rows: []
- on: server2
user: 'brian'
password: 'brianspassword'
queries:
- query: "show warnings"
result:
columns: ["Level", "Code", "Message"]
rows: []
- name: users and grants and branch control replicate to multiple standbys
skip: "cluster replication / MySQL grants not yet supported in Doltgres"
multi_repos:
- name: server1
with_files:
- name: server.yaml
contents: |
log_level: trace
cluster:
standby_remotes:
- name: standby1
remote_url_template: http://localhost:{{get_port "server2_cluster"}}/{database}
- name: standby2
remote_url_template: http://localhost:{{get_port "server3_cluster"}}/{database}
bootstrap_role: primary
bootstrap_epoch: 1
remotesapi:
port: {{get_port "server1_cluster"}}
server:
args: ["--config", "server.yaml"]
dynamic_port: server1
- name: server2
with_files:
- name: server.yaml
contents: |
log_level: trace
cluster:
standby_remotes:
- name: standby1
remote_url_template: http://localhost:{{get_port "server1_cluster"}}/{database}
- name: standby2
remote_url_template: http://localhost:{{get_port "server3_cluster"}}/{database}
bootstrap_role: standby
bootstrap_epoch: 1
remotesapi:
port: {{get_port "server2_cluster"}}
server:
args: ["--config", "server.yaml"]
dynamic_port: server2
- name: server3
with_files:
- name: server.yaml
contents: |
log_level: trace
cluster:
standby_remotes:
- name: standby1
remote_url_template: http://localhost:{{get_port "server1_cluster"}}/{database}
- name: standby2
remote_url_template: http://localhost:{{get_port "server2_cluster"}}/{database}
bootstrap_role: standby
bootstrap_epoch: 1
remotesapi:
port: {{get_port "server3_cluster"}}
server:
args: ["--config", "server.yaml"]
dynamic_port: server3
connections:
- on: server1
queries:
- exec: "SET dolt_cluster_ack_writes_timeout_secs = 10"
- exec: 'create database repo1'
- exec: "use repo1"
- exec: 'create table vals (i int primary key)'
- exec: 'insert into vals values (0),(1),(2),(3),(4)'
- exec: 'create user aaron with password ''aaronspassword'''
- exec: 'GRANT ALL PRIVILEGES ON ALL TABLES IN SCHEMA public TO aaron'
- exec: 'create user brian with password ''brianpassword'''
- exec: 'GRANT ALL PRIVILEGES ON ALL TABLES IN SCHEMA public TO brian'
- exec: 'delete from dolt_branch_control'
- exec: 'insert into dolt_branch_control values (''repo1'', ''main'', ''aaron'', ''%'', ''admin'')'
- on: server2
user: 'aaron'
password: 'aaronspassword'
queries:
- exec: "use repo1"
- query: 'select count(*) as count from vals'
result:
columns: ["count"]
rows: [["5"]]
- query: 'select count(*) as count from dolt_branch_control'
result:
columns: ["count"]
rows: [["1"]]
- on: server3
user: 'aaron'
password: 'aaronspassword'
queries:
- exec: "use repo1"
- query: 'select count(*) as count from vals'
result:
columns: ["count"]
rows: [["5"]]
- query: 'select count(*) as count from dolt_branch_control'
result:
columns: ["count"]
rows: [["1"]]
integration-tests/go-sql-server-driver/tests/sql-server-cluster.yaml
+2182
View File
File diff suppressed because it is too large Load Diff
integration-tests/go-sql-server-driver/tests/sql-server-config.yaml
+132
View File
@@ -0,0 +1,132 @@
parallel: true
tests:
- name: dolt_transaction_commit is global variable
skip: "dolt_transaction_commit not being respected after being set"
repos:
- name: repo1
server:
args: []
dynamic_port: repo1
connections:
- on: repo1
queries:
- exec: "SET dolt_transaction_commit = 1"
- query: "select current_setting('dolt_transaction_commit') as dolt_transaction_commit"
result:
columns: ["dolt_transaction_commit"]
rows: [["1"]]
- on: repo1
queries:
- query: "select count(*) from dolt_log"
result:
columns: ["count"]
rows: [["2"]]
- exec: "create table tmp (i int)"
- query: "select count(*) from dolt_log"
result:
columns: ["count"]
rows: [["3"]]
- name: set max_connections with yaml config
skip: "need to adapt to doltgres listener"
repos:
- name: repo1
with_files:
- name: "server.yaml"
contents: |
log_level: trace
listener:
max_connections: 999
server:
args: ["--config", "server.yaml"]
dynamic_port: repo1
connections:
- on: repo1
queries:
- query: "select current_setting('max_connections') as max_connections"
result:
columns: ["max_connections"]
rows: [["999"]]
- name: "dolt_auto_gc_enabled default"
skip: "default not set yet"
repos:
- name: repo1
server:
args: []
dynamic_port: repo1
connections:
- on: repo1
queries:
- query: "select current_setting('dolt_auto_gc_enabled') as dolt_auto_gc_enabled"
result:
columns: ["dolt_auto_gc_enabled"]
rows: [["1"]]
- exec: "SET dolt_auto_gc_enabled = 0"
error_match: "Variable 'dolt_auto_gc_enabled' is a read only variable"
- name: "@@global.dolt_auto_gc_enabled true"
skip: "config field not supported yet"
repos:
- name: repo1
with_files:
- name: "config.yaml"
contents: |
behavior:
auto_gc_behavior:
enable: true
server:
args: ["--config", "config.yaml"]
dynamic_port: repo1
connections:
- on: repo1
queries:
- query: "select current_setting('dolt_auto_gc_enabled') as dolt_auto_gc_enabled"
result:
columns: ["dolt_auto_gc_enabled"]
rows: [["1"]]
- name: "@@global.dolt_auto_gc_enabled false"
skip: "config field not supported yet"
repos:
- name: repo1
with_files:
- name: "config.yaml"
contents: |
behavior:
auto_gc_behavior:
enable: false
server:
args: ["--config", "config.yaml"]
dynamic_port: repo1
connections:
- on: repo1
queries:
- query: "select current_setting('dolt_auto_gc_enabled') as dolt_auto_gc_enabled"
result:
columns: ["dolt_auto_gc_enabled"]
rows: [["0"]]
- name: secure_file_priv set to /dev/null prevents loading files
skip: "field / feature needs to be supported"
repos:
- name: repo1
with_files:
- name: "config.yaml"
contents: |
system_variables:
secure_file_priv: "/dev/null"
server:
args: ["--config", "config.yaml"]
dynamic_port: repo1
connections:
- on: repo1
queries:
- query: "select LOAD_FILE('config.yaml') as load_file"
result:
columns: ["load_file"]
rows: [["NULL"]]
- query: "select LOAD_FILE('/etc/passwd') as load_file"
result:
columns: ["load_file"]
rows: [["NULL"]]
- exec: "create table loaded (contents text)"
- exec: "COPY loaded FROM 'config.yaml'"
error_match: "must be superuser or have privileges of the pg_read_server_files role|secure_file_priv"
- exec: "COPY loaded FROM '/config.yaml'"
error_match: "must be superuser or have privileges of the pg_read_server_files role|secure_file_priv"
integration-tests/go-sql-server-driver/tests/sql-server-large-blob-replication.yaml
+79
View File
@@ -0,0 +1,79 @@
parallel: true
tests:
- name: large blob values replicate to standby
# Cluster replication is not yet implemented in Doltgres, so this test is
# skipped. The cluster config and the large-blob schema/data (LONGBLOB ->
# bytea) are translated faithfully so this can be enabled once replication
# lands. The ASCII payloads are turned into bytea with convert_to(...,'UTF8'),
# which preserves the byte length asserted by LENGTH().
skip: "cluster/remotesapi replication not yet implemented in Doltgres"
multi_repos:
- name: server1
with_files:
- name: server.yaml
contents: |
log_level: trace
listener:
host: 0.0.0.0
cluster:
standby_remotes:
- name: standby
remote_url_template: http://localhost:{{get_port "server2_cluster"}}/{database}
bootstrap_role: primary
bootstrap_epoch: 1
remotesapi:
port: {{get_port "server1_cluster"}}
server:
args: ["--config", "server.yaml"]
dynamic_port: server1
- name: server2
with_files:
- name: server.yaml
contents: |
log_level: trace
listener:
host: 0.0.0.0
cluster:
standby_remotes:
- name: standby
remote_url_template: http://localhost:{{get_port "server1_cluster"}}/{database}
bootstrap_role: standby
bootstrap_epoch: 1
remotesapi:
port: {{get_port "server2_cluster"}}
server:
args: ["--config", "server.yaml"]
dynamic_port: server2
connections:
- on: server1
queries:
- exec: "SET dolt_cluster_ack_writes_timeout_secs = 10"
- exec: 'CREATE DATABASE repo1'
- exec: 'USE repo1'
- exec: |
CREATE TABLE large_blob (
id BIGINT PRIMARY KEY,
data BYTEA
)
- exec: "INSERT INTO large_blob VALUES (1, convert_to(REPEAT('abcdefghijklmnopqrst', 1250), 'UTF8'))"
- exec: "INSERT INTO large_blob VALUES (2, convert_to(REPEAT('ABCDEFGHIJKLMNOPQRST', 1250), 'UTF8'))"
- exec: "INSERT INTO large_blob VALUES (3, convert_to(REPEAT('01234567890123456789', 1250), 'UTF8'))"
- exec: "INSERT INTO large_blob VALUES (4, convert_to(REPEAT('zyxwvutsrqponmlkjihg', 1250), 'UTF8'))"
- exec: "INSERT INTO large_blob VALUES (5, convert_to(REPEAT('fedcba9876543210FEDC', 1250), 'UTF8'))"
- query: "SELECT COUNT(*) AS count FROM large_blob WHERE LENGTH(data) = 25000"
result:
columns: ["count"]
rows: [["5"]]
- on: server2
retry_attempts: 100
queries:
- exec: 'USE repo1'
- query: "SELECT COUNT(*) AS count FROM large_blob"
result:
columns: ["count"]
rows: [["5"]]
- query: "SELECT COUNT(*) AS count FROM large_blob WHERE LENGTH(data) = 25000"
result:
columns: ["count"]
rows: [["5"]]
integration-tests/go-sql-server-driver/tests/sql-server-large-json-replication.yaml
+87
View File
@@ -0,0 +1,87 @@
parallel: true
tests:
- name: large json values replicate to standby
# Cluster replication is not yet implemented in Doltgres, so this test is
# skipped. The cluster config and the large-json schema/data are translated
# faithfully (JSON_OBJECT/JSON_ARRAY -> json_build_object/json_build_array,
# LENGTH(doc) -> LENGTH(doc::text), doc->>'$.id' -> doc->>'id') so this can be
# enabled once replication lands.
skip: "cluster/remotesapi replication not yet implemented in Doltgres"
multi_repos:
- name: server1
with_files:
- name: server.yaml
contents: |
log_level: trace
listener:
host: 0.0.0.0
cluster:
standby_remotes:
- name: standby
remote_url_template: http://localhost:{{get_port "server2_cluster"}}/{database}
bootstrap_role: primary
bootstrap_epoch: 1
remotesapi:
port: {{get_port "server1_cluster"}}
server:
args: ["--config", "server.yaml"]
dynamic_port: server1
- name: server2
with_files:
- name: server.yaml
contents: |
log_level: trace
listener:
host: 0.0.0.0
cluster:
standby_remotes:
- name: standby
remote_url_template: http://localhost:{{get_port "server1_cluster"}}/{database}
bootstrap_role: standby
bootstrap_epoch: 1
remotesapi:
port: {{get_port "server2_cluster"}}
server:
args: ["--config", "server.yaml"]
dynamic_port: server2
connections:
- on: server1
queries:
- exec: "SET dolt_cluster_ack_writes_timeout_secs = 10"
- exec: 'CREATE DATABASE repo1'
- exec: 'USE repo1'
- exec: |
CREATE TABLE large_json (
id BIGINT PRIMARY KEY,
doc JSON
)
- exec: "INSERT INTO large_json VALUES (1, json_build_object('id', 1, 'payload', REPEAT('abcde', 2200)))"
- exec: "INSERT INTO large_json VALUES (2, json_build_object('id', 2, 'payload', REPEAT('fghij', 2200)))"
- exec: "INSERT INTO large_json VALUES (3, json_build_object('id', 3, 'payload', REPEAT('klmno', 2200)))"
- exec: "INSERT INTO large_json VALUES (4, json_build_object('id', 4, 'items', json_build_array(REPEAT('x',3700), REPEAT('y',3700), REPEAT('z',3700))))"
- exec: "INSERT INTO large_json VALUES (5, json_build_object('id', 5, 'nested', json_build_object('a', REPEAT('p',5500), 'b', REPEAT('q',5500))))"
- query: "SELECT COUNT(*) AS count FROM large_json WHERE LENGTH(doc::text) > 11000"
result:
columns: ["count"]
rows: [["5"]]
- query: "SELECT doc->>'id' AS id FROM large_json WHERE id = 3"
result:
columns: ["id"]
rows: [["3"]]
- on: server2
retry_attempts: 100
queries:
- exec: 'USE repo1'
- query: "SELECT COUNT(*) AS count FROM large_json"
result:
columns: ["count"]
rows: [["5"]]
- query: "SELECT COUNT(*) AS count FROM large_json WHERE LENGTH(doc::text) > 11000"
result:
columns: ["count"]
rows: [["5"]]
- query: "SELECT doc->>'id' AS id FROM large_json WHERE id = 3"
result:
columns: ["id"]
rows: [["3"]]
integration-tests/go-sql-server-driver/tests/sql-server-large-multi-column-replication.yaml
+106
View File
@@ -0,0 +1,106 @@
parallel: true
tests:
- name: row with multiple large columns replicates correctly
# Cluster replication is not yet implemented in Doltgres, so this test is
# skipped. The cluster config and the multi-large-column schema/data are
# translated faithfully (LONGTEXT/TEXT -> text, LONGBLOB -> bytea via
# convert_to(...,'UTF8'), JSON_OBJECT -> json_build_object, LENGTH(doc) ->
# LENGTH(doc::text), doc->>'$.row' -> doc->>'row') so this can be enabled once
# replication lands.
skip: "cluster/remotesapi replication not yet implemented in Doltgres"
multi_repos:
- name: server1
with_files:
- name: server.yaml
contents: |
log_level: trace
listener:
host: 0.0.0.0
cluster:
standby_remotes:
- name: standby
remote_url_template: http://localhost:{{get_port "server2_cluster"}}/{database}
bootstrap_role: primary
bootstrap_epoch: 1
remotesapi:
port: {{get_port "server1_cluster"}}
server:
args: ["--config", "server.yaml"]
dynamic_port: server1
- name: server2
with_files:
- name: server.yaml
contents: |
log_level: trace
listener:
host: 0.0.0.0
cluster:
standby_remotes:
- name: standby
remote_url_template: http://localhost:{{get_port "server1_cluster"}}/{database}
bootstrap_role: standby
bootstrap_epoch: 1
remotesapi:
port: {{get_port "server2_cluster"}}
server:
args: ["--config", "server.yaml"]
dynamic_port: server2
connections:
- on: server1
queries:
- exec: "SET dolt_cluster_ack_writes_timeout_secs = 10"
- exec: 'CREATE DATABASE repo1'
- exec: 'USE repo1'
- exec: |
CREATE TABLE multi_large (
id BIGINT PRIMARY KEY,
txt TEXT,
blob_data BYTEA,
doc JSON,
note TEXT
)
- exec: |
INSERT INTO multi_large VALUES (
1,
REPEAT('textcol-row1-', 1154),
convert_to(REPEAT('blobcol-row1-', 1924), 'UTF8'),
json_build_object('row', 1, 'data', REPEAT('jsoncol-row1-', 847)),
REPEAT('note-col-row1-', 357)
)
- exec: |
INSERT INTO multi_large VALUES (
2,
REPEAT('textcol-row2-', 1154),
convert_to(REPEAT('blobcol-row2-', 1924), 'UTF8'),
json_build_object('row', 2, 'data', REPEAT('jsoncol-row2-', 847)),
REPEAT('note-col-row2-', 357)
)
- exec: |
INSERT INTO multi_large VALUES (
3,
REPEAT('textcol-row3-', 1154),
convert_to(REPEAT('blobcol-row3-', 1924), 'UTF8'),
json_build_object('row', 3, 'data', REPEAT('jsoncol-row3-', 847)),
REPEAT('note-col-row3-', 357)
)
- query: "SELECT COUNT(*) AS count FROM multi_large WHERE LENGTH(txt) > 10000 AND LENGTH(blob_data) > 20000"
result:
columns: ["count"]
rows: [["3"]]
- on: server2
retry_attempts: 100
queries:
- exec: 'USE repo1'
- query: "SELECT COUNT(*) AS count FROM multi_large"
result:
columns: ["count"]
rows: [["3"]]
- query: "SELECT COUNT(*) AS count FROM multi_large WHERE LENGTH(txt) > 10000 AND LENGTH(blob_data) > 20000 AND LENGTH(doc::text) > 11000 AND LENGTH(note) > 4000"
result:
columns: ["count"]
rows: [["3"]]
- query: "SELECT doc->>'row' AS row FROM multi_large WHERE id = 2"
result:
columns: ["row"]
rows: [["2"]]
integration-tests/go-sql-server-driver/tests/sql-server-large-text-replication.yaml
+78
View File
@@ -0,0 +1,78 @@
parallel: true
tests:
- name: large text values replicate to standby
# Cluster replication is not yet implemented in Doltgres, so this test is
# skipped. The cluster config and the large-text schema/data (LONGTEXT ->
# text) are translated faithfully so this can be enabled once replication
# lands.
skip: "cluster/remotesapi replication not yet implemented in Doltgres"
multi_repos:
- name: server1
with_files:
- name: server.yaml
contents: |
log_level: trace
listener:
host: 0.0.0.0
cluster:
standby_remotes:
- name: standby
remote_url_template: http://localhost:{{get_port "server2_cluster"}}/{database}
bootstrap_role: primary
bootstrap_epoch: 1
remotesapi:
port: {{get_port "server1_cluster"}}
server:
args: ["--config", "server.yaml"]
dynamic_port: server1
- name: server2
with_files:
- name: server.yaml
contents: |
log_level: trace
listener:
host: 0.0.0.0
cluster:
standby_remotes:
- name: standby
remote_url_template: http://localhost:{{get_port "server1_cluster"}}/{database}
bootstrap_role: standby
bootstrap_epoch: 1
remotesapi:
port: {{get_port "server2_cluster"}}
server:
args: ["--config", "server.yaml"]
dynamic_port: server2
connections:
- on: server1
queries:
- exec: "SET dolt_cluster_ack_writes_timeout_secs = 10"
- exec: 'CREATE DATABASE repo1'
- exec: 'USE repo1'
- exec: |
CREATE TABLE large_text (
id BIGINT PRIMARY KEY,
txt TEXT
)
- exec: "INSERT INTO large_text VALUES (1, REPEAT('abcdefghij', 1500))"
- exec: "INSERT INTO large_text VALUES (2, REPEAT('klmnopqrst', 1500))"
- exec: "INSERT INTO large_text VALUES (3, REPEAT('uvwxyz0123', 1500))"
- exec: "INSERT INTO large_text VALUES (4, REPEAT('4567890abcd', 1363))"
- exec: "INSERT INTO large_text VALUES (5, REPEAT('efghijklmno', 1363))"
- query: "SELECT COUNT(*) AS count FROM large_text WHERE LENGTH(txt) > 10000"
result:
columns: ["count"]
rows: [["5"]]
- on: server2
retry_attempts: 100
queries:
- exec: 'USE repo1'
- query: "SELECT COUNT(*) AS count FROM large_text"
result:
columns: ["count"]
rows: [["5"]]
- query: "SELECT COUNT(*) AS count FROM large_text WHERE LENGTH(txt) > 10000"
result:
columns: ["count"]
rows: [["5"]]
integration-tests/go-sql-server-driver/tests/sql-server-large-values-failover.yaml
+94
View File
@@ -0,0 +1,94 @@
parallel: true
tests:
- name: large values survive failover to new primary
skip: "cluster replication/failover not yet implemented in Doltgres"
multi_repos:
- name: server1
with_files:
- name: server.yaml
contents: |
log_level: trace
behavior:
auto_gc_behavior:
enable: false
cluster:
standby_remotes:
- name: standby
remote_url_template: http://localhost:{{get_port "server2_cluster"}}/{database}
bootstrap_role: primary
bootstrap_epoch: 1
remotesapi:
port: {{get_port "server1_cluster"}}
server:
args: ["--config", "server.yaml"]
dynamic_port: server1
- name: server2
with_files:
- name: server.yaml
contents: |
log_level: trace
behavior:
auto_gc_behavior:
enable: false
cluster:
standby_remotes:
- name: standby
remote_url_template: http://localhost:{{get_port "server1_cluster"}}/{database}
bootstrap_role: standby
bootstrap_epoch: 1
remotesapi:
port: {{get_port "server2_cluster"}}
server:
args: ["--config", "server.yaml"]
dynamic_port: server2
connections:
- on: server1
queries:
- exec: 'SET dolt_cluster_ack_writes_timeout_secs = 10'
- exec: 'CREATE DATABASE repo1'
- exec: 'USE repo1'
- exec: |
CREATE TABLE large_vals (
id BIGINT PRIMARY KEY,
txt TEXT,
blob_col BYTEA
)
- exec: "INSERT INTO large_vals VALUES (1, REPEAT('textrow1-', 1667), REPEAT('blobrow1-', 2778)::bytea)"
- exec: "INSERT INTO large_vals VALUES (2, REPEAT('textrow2-', 1667), REPEAT('blobrow2-', 2778)::bytea)"
- exec: "INSERT INTO large_vals VALUES (3, REPEAT('textrow3-', 1667), REPEAT('blobrow3-', 2778)::bytea)"
- exec: "INSERT INTO large_vals VALUES (4, REPEAT('textrow4-', 1667), REPEAT('blobrow4-', 2778)::bytea)"
- exec: "INSERT INTO large_vals VALUES (5, REPEAT('textrow5-', 1667), REPEAT('blobrow5-', 2778)::bytea)"
# Transition to standby; server1's epoch advances to 2 on server2.
- query: "SELECT dolt_assume_cluster_role('standby', 2)"
result:
columns: ["dolt_assume_cluster_role"]
rows: [["{0}"]]
# Verify the standby received all rows, then promote it to primary.
- on: server2
retry_attempts: 100
queries:
- exec: 'USE repo1'
- query: "SELECT COUNT(*) AS count FROM large_vals WHERE LENGTH(txt) > 10000 AND LENGTH(blob_col) > 20000"
result:
columns: ["count"]
rows: [["5"]]
- query: "SELECT dolt_assume_cluster_role('primary', 2)"
result:
columns: ["dolt_assume_cluster_role"]
rows: [["{0}"]]
# Open a fresh connection to the new primary and confirm large values are intact.
# Also insert two additional large rows to prove the new primary is writable.
- on: server2
queries:
- exec: 'USE repo1'
- query: "SELECT COUNT(*) AS count FROM large_vals"
result:
columns: ["count"]
rows: [["5"]]
- exec: "INSERT INTO large_vals VALUES (6, REPEAT('textrow6-', 1667), REPEAT('blobrow6-', 2778)::bytea)"
- exec: "INSERT INTO large_vals VALUES (7, REPEAT('textrow7-', 1667), REPEAT('blobrow7-', 2778)::bytea)"
- query: "SELECT COUNT(*) AS count FROM large_vals WHERE LENGTH(txt) > 10000 AND LENGTH(blob_col) > 20000"
result:
columns: ["count"]
rows: [["7"]]
integration-tests/go-sql-server-driver/tests/sql-server-large-values-gc.yaml
+95
View File
@@ -0,0 +1,95 @@
parallel: true
tests:
- name: large values survive gc on primary then replicate to standby
skip: "cluster replication/failover not yet implemented in Doltgres"
multi_repos:
- name: server1
with_files:
- name: server.yaml
contents: |
log_level: trace
behavior:
auto_gc_behavior:
enable: false
cluster:
standby_remotes:
- name: standby
remote_url_template: http://localhost:{{get_port "server2_cluster"}}/{database}
bootstrap_role: primary
bootstrap_epoch: 1
remotesapi:
port: {{get_port "server1_cluster"}}
server:
args: ["--config", "server.yaml"]
dynamic_port: server1
- name: server2
with_files:
- name: server.yaml
contents: |
log_level: trace
behavior:
auto_gc_behavior:
enable: false
cluster:
standby_remotes:
- name: standby
remote_url_template: http://localhost:{{get_port "server1_cluster"}}/{database}
bootstrap_role: standby
bootstrap_epoch: 1
remotesapi:
port: {{get_port "server2_cluster"}}
server:
args: ["--config", "server.yaml"]
dynamic_port: server2
connections:
- on: server1
queries:
- exec: 'SET dolt_cluster_ack_writes_timeout_secs = 10'
- exec: 'CREATE DATABASE repo1'
- exec: 'USE repo1'
- exec: |
CREATE TABLE large_text (
id BIGINT PRIMARY KEY,
txt TEXT
)
- exec: "INSERT INTO large_text VALUES (1, REPEAT('abcdefghij', 1500))"
- exec: "INSERT INTO large_text VALUES (2, REPEAT('klmnopqrst', 1500))"
- exec: "INSERT INTO large_text VALUES (3, REPEAT('uvwxyz0123', 1500))"
- exec: "INSERT INTO large_text VALUES (4, REPEAT('4567890abc', 1500))"
- exec: "INSERT INTO large_text VALUES (5, REPEAT('defghijklm', 1500))"
- exec: 'SELECT dolt_gc()'
- query: 'SELECT COUNT(*) FROM large_text'
error_match: "this connection can no longer be used"
# After GC the connection is invalidated. Open a new connection to the primary
# and wait for the cluster to finish replicating the post-GC state.
- on: server1
queries:
- query: |
SELECT "database", standby_remote, role, epoch, replication_lag_millis, current_error
FROM dolt_cluster.dolt_cluster_status
ORDER BY "database" ASC
result:
columns: ["database","standby_remote","role","epoch","replication_lag_millis","current_error"]
rows:
- ["repo1","standby","primary","1","0","NULL"]
retry_attempts: 100
- exec: 'USE repo1'
- query: "SELECT COUNT(*) AS count FROM large_text WHERE LENGTH(txt) = 15000"
result:
columns: ["count"]
rows: [["5"]]
# Verify the standby received and preserved all large values post-GC, then
# confirm it can perform a shallow GC of its own.
- on: server2
queries:
- exec: 'USE repo1'
- query: "SELECT COUNT(*) AS count FROM large_text WHERE LENGTH(txt) = 15000"
result:
columns: ["count"]
rows: [["5"]]
- exec: "SELECT dolt_gc('--shallow')"
- query: "SELECT COUNT(*) AS count FROM large_text WHERE LENGTH(txt) = 15000"
result:
columns: ["count"]
rows: [["5"]]
integration-tests/go-sql-server-driver/tests/sql-server-orig.yaml
+445
View File
@@ -0,0 +1,445 @@
parallel: true
tests:
- name: read-only flag prevents modification
repos:
- name: repo1
server:
args: ["--config", "readonly.yaml"]
dynamic_port: server1
with_files:
- name: readonly.yaml
contents: |
log_level: trace
behavior:
read_only: true
connections:
- on: repo1
queries:
- query: "select count(*) as count from information_schema.tables where table_schema = 'public'"
result:
columns: ["count"]
rows: [["0"]]
- exec: "create table i_should_not_exist (c0 int)"
error_match: "read only mode"
- query: "select count(*) as count from information_schema.tables where table_schema = 'public'"
result:
columns: ["count"]
rows: [["0"]]
- name: read-only flag still allows select
repos:
- name: repo1
server:
args: []
dynamic_port: server1
with_files:
- name: readonly.yaml
contents: |
behavior:
read_only: true
connections:
- on: repo1
queries:
- exec: "create table t(c0 int)"
- exec: "insert into t values (1)"
restart_server:
args: ["--config", "readonly.yaml"]
- on: repo1
queries:
- query: "select * from t"
result:
columns: ["c0"]
rows: [["1"]]
- name: read-only flag prevents dolt_commit
skip: "Doltgres read-only mode does not block the dolt_commit() procedure; it is not classified as a write by the engine read-only guard"
repos:
- name: repo1
server:
args: ["--config", "readonly.yaml"]
dynamic_port: server1
with_files:
- name: readonly.yaml
contents: |
log_level: trace
behavior:
read_only: true
connections:
- on: repo1
queries:
- query: "select dolt_commit('--allow-empty', '-m', 'msg')"
error_match: "read only"
- name: read-only flag prevents dolt_reset
skip: "Doltgres read-only mode does not block the dolt_reset() procedure; it is not classified as a write by the engine read-only guard"
repos:
- name: repo1
server:
args: []
dynamic_port: server1
with_files:
- name: readonly.yaml
contents: |
log_level: trace
behavior:
read_only: true
connections:
- on: repo1
queries:
- query: "select dolt_commit('--allow-empty', '-m', 'msg')"
result:
columns: ["dolt_commit"]
rows: []
restart_server:
args: ["--config", "readonly.yaml"]
- on: repo1
queries:
- query: "select dolt_reset('--hard', 'HEAD~1')"
error_match: "read only"
- name: port in use
repos:
- name: repo1
server:
args: []
dynamic_port: server1
- name: repo2
server:
args: []
dynamic_port: server1
error_matches:
- "already in use"
- name: test basic querying
repos:
- name: repo1
server:
args: []
dynamic_port: server1
connections:
- on: repo1
queries:
- query: "select table_name from information_schema.tables where table_schema = 'public' order by table_name"
result:
columns: ["table_name"]
rows: []
- on: repo1
queries:
- exec: |
CREATE TABLE one_pk (
pk bigint NOT NULL,
c1 bigint,
c2 bigint,
PRIMARY KEY (pk)
)
- on: repo1
queries:
- query: "select table_name from information_schema.tables where table_schema = 'public' order by table_name"
result:
columns: ["table_name"]
rows: [["one_pk"]]
- on: repo1
queries:
- exec: "INSERT INTO one_pk (pk) VALUES (0)"
- on: repo1
queries:
- query: "SELECT * FROM one_pk ORDER BY pk"
result:
columns: ["pk","c1","c2"]
rows: [["0","NULL","NULL"]]
- on: repo1
queries:
- exec: "INSERT INTO one_pk (pk,c1) VALUES (1,1)"
- on: repo1
queries:
- exec: "INSERT INTO one_pk (pk,c1,c2) VALUES (2,2,2),(3,3,3)"
- on: repo1
queries:
- query: "SELECT * FROM one_pk ORDER BY pk"
result:
columns: ["pk","c1","c2"]
rows:
- ["0","NULL","NULL"]
- ["1","1","NULL"]
- ["2","2","2"]
- ["3","3","3"]
- on: repo1
queries:
- exec: "UPDATE one_pk SET c2=c1 WHERE c2 is NULL and c1 IS NOT NULL"
- name: test multiple queries on same connection
repos:
- name: repo1
server:
args: []
dynamic_port: server1
connections:
- on: repo1
queries:
- exec: |
CREATE TABLE one_pk (
pk bigint NOT NULL,
c1 bigint,
c2 bigint,
PRIMARY KEY (pk)
)
- exec: "INSERT INTO one_pk (pk) VALUES (0)"
- exec: "INSERT INTO one_pk (pk,c1) VALUES (1,1)"
- exec: "INSERT INTO one_pk (pk,c1,c2) VALUES (2,2,2),(3,3,3)"
- query: "SELECT * FROM one_pk ORDER BY pk"
result:
columns: ["pk","c1","c2"]
rows:
- ["0","NULL","NULL"]
- ["1","1","NULL"]
- ["2","2","2"]
- ["3","3","3"]
- on: repo1
queries:
- query: "SELECT * FROM one_pk ORDER BY pk"
result:
columns: ["pk","c1","c2"]
rows:
- ["0","NULL","NULL"]
- ["1","1","NULL"]
- ["2","2","2"]
- ["3","3","3"]
- name: test CREATE and DROP database via sql-server
# The original test creates a database, then `USE`s it to operate on its
# tables on the same connection. Postgres/Doltgres binds the database at
# connection time (no USE across databases), and this test framework can only
# open connections to servers registered by repo/multi-repo name (via `on:`),
# so a freshly-created database cannot be reached by name. We still exercise
# CREATE DATABASE / DROP DATABASE and verify pg_database visibility.
repos:
- name: repo1
server:
args: []
dynamic_port: server1
connections:
- on: repo1
queries:
- exec: "CREATE DATABASE test"
- query: "select datname from pg_database where datname in ('repo1','test') order by datname"
result:
columns: ["datname"]
rows:
- ["repo1"]
- ["test"]
- exec: "drop database test"
- query: "select datname from pg_database where datname in ('repo1','test') order by datname"
result:
columns: ["datname"]
rows:
- ["repo1"]
- name: COPY loads CSV data from a file
repos:
- name: repo1
with_files:
- name: "1pk5col-ints.csv"
source_path: "testdata/1pk5col-ints.csv"
server:
args: []
dynamic_port: server1
connections:
- on: repo1
queries:
- exec: "CREATE TABLE test(pk int primary key, c1 int, c2 int, c3 int, c4 int, c5 int)"
- exec: "COPY test FROM '1pk5col-ints.csv' WITH (FORMAT csv, HEADER)"
- query: "SELECT * FROM test"
result:
columns: ["pk","c1","c2","c3","c4","c5"]
rows:
- ["0","1","2","3","4","5"]
- ["1","1","2","3","4","5"]
- name: JSON queries
repos:
- name: repo1
server:
args: []
dynamic_port: server1
connections:
- on: repo1
queries:
- exec: |
CREATE TABLE js_test (
pk int NOT NULL,
js json,
PRIMARY KEY (pk)
)
- exec: |
INSERT INTO js_test VALUES (1, '{"a":1}')
- query: "SELECT * FROM js_test"
result:
columns: ["pk","js"]
rows:
or:
- [["1", '{"a": 1}']]
- [["1", '{"a":1}']]
- name: select a branch with dolt_checkout
# The original test used MySQL `USE repo1/feature-branch` to switch to a
# branch-revision database. Doltgres has no `USE` statement, so the equivalent
# session branch switch is done with dolt_checkout().
repos:
- name: repo1
server:
args: []
dynamic_port: server1
connections:
- on: repo1
queries:
- query: "select dolt_branch('feature-branch')"
result:
columns: ["dolt_branch"]
rows: [["{0}"]]
- query: "select dolt_checkout('feature-branch')"
result:
columns: ["dolt_checkout"]
rows:
or:
- [["{0,\"Switched to branch 'feature-branch'\"}"]]
- [["{0,Switched to branch 'feature-branch'}"]]
- exec: |
CREATE TABLE test (
pk int,
c1 int,
PRIMARY KEY (pk)
)
- on: repo1
queries:
- query: "select table_name from information_schema.tables where table_schema = 'public' order by table_name"
result:
columns: ["table_name"]
rows: []
- query: "select dolt_checkout('feature-branch')"
result:
columns: ["dolt_checkout"]
rows:
or:
- [["{0,\"Switched to branch 'feature-branch'\"}"]]
- [["{0,Switched to branch 'feature-branch'}"]]
- query: "select table_name from information_schema.tables where table_schema = 'public' order by table_name"
result:
columns: ["table_name"]
rows: [["test"]]
- name: auto increment for a table should reset between drops
repos:
- name: repo1
server:
args: []
dynamic_port: server1
connections:
- on: repo1
queries:
- exec: "CREATE TABLE t1(pk serial primary key, val int)"
- exec: "INSERT INTO t1 (val) VALUES (1),(2)"
- query: "SELECT * FROM t1 ORDER BY pk"
result:
columns: ["pk", "val"]
rows: [["1", "1"], ["2", "2"]]
- exec: "DROP TABLE t1"
- exec: "CREATE TABLE t1(pk serial primary key, val int)"
- exec: "INSERT INTO t1 (val) VALUES (1),(2)"
- query: "SELECT * FROM t1 ORDER BY pk"
result:
columns: ["pk", "val"]
rows: [["1", "1"], ["2", "2"]]
- name: Create a temporary table and validate that it doesn't persist after a session closes
repos:
- name: repo1
server:
args: []
dynamic_port: server1
connections:
- on: repo1
queries:
- query: "select table_name from information_schema.tables where table_schema = 'public' order by table_name"
result:
columns: ["table_name"]
rows: []
- exec: "CREATE TEMPORARY TABLE t1(pk int primary key, val int)"
- exec: "INSERT INTO t1 VALUES (1, 1),(2, 2)"
- on: repo1
queries:
- exec: "INSERT INTO t1 VALUES (1, 1),(2, 2)"
error_match: "(does not exist|not found)"
- name: create a collated database connected to a non-dolt database
skip: "MySQL-specific COLLATE database creation / non-Dolt database semantics not applicable to Doltgres"
repos:
- name: repo1
server:
args: []
dynamic_port: server1
connections:
- on: repo1
queries:
- exec: "USE mysql"
- exec: "CREATE DATABASE testdb COLLATE utf8mb4_unicode_ci"
- exec: "USE testdb"
- name: dolt_gc succeeds as first write on existing database without a journal after chunk journal is enabled
skip: "Requires the database to be initialized with the chunk journal disabled; the doltgres test harness initializes databases with the journal enabled, so this storage-internals scenario cannot be reproduced yet"
# The original test created the database via CREATE DATABASE on a multi-repo
# server, then USE'd it. Doltgres binds the database at connect time and this
# framework can only connect to databases registered by repo name (via `on:`),
# so the database is created up front as a repo and connected to directly. The
# server starts with the chunk journal disabled, data is written, then the
# server is restarted (re-enabling the chunk journal) and dolt_gc is run as the
# first write.
repos:
- name: mydb
server:
envs: ["DOLT_DISABLE_CHUNK_JOURNAL=true"]
args: []
dynamic_port: server1
connections:
- on: mydb
queries:
- exec: "CREATE TABLE vals (id int primary key, val int)"
- exec: "INSERT INTO vals VALUES (1, 1),(2, 2)"
restart_server:
envs: []
- on: mydb
queries:
- query: "select dolt_gc() as dolt_gc"
result:
columns: ["dolt_gc"]
rows: [["{0}"]]
- name: file remotes update appropriately
skip: "remotesapi/file-remote replication across servers not yet supported in Doltgres"
multi_repos:
- name: server1
server:
args: []
dynamic_port: server1
- name: server2
server:
args: []
dynamic_port: server2
- name: server3
server:
args: []
dynamic_port: server3
connections:
- on: server1
queries:
- exec: "CREATE DATABASE mydb"
- exec: "select dolt_remote('add', 'origin', 'file://{{get_tempdir \"remote_storage\"}}')"
- exec: "select dolt_push('origin', 'main:main')"
- on: server2
queries:
- exec: "select dolt_clone('file://{{get_tempdir \"remote_storage\"}}', 'mydb')"
- on: server3
queries:
- exec: "select dolt_clone('file://{{get_tempdir \"remote_storage\"}}', 'mydb')"
- on: server2
queries:
- exec: "CREATE TABLE test (id int PRIMARY KEY)"
- exec: "select dolt_commit('-Am', 'create test table')"
- exec: "select dolt_push('origin', 'main')"
- on: server3
queries:
- exec: "select dolt_pull('origin', 'main')"
- exec: "INSERT INTO test VALUES (1), (2), (47)"
- exec: "select dolt_commit('-am', 'Insert some initial values.')"
- exec: "select dolt_push('origin', 'main')"
- on: server1
queries:
- exec: "select dolt_pull('origin', 'main')"
- query: "SELECT * FROM test ORDER BY id ASC"
result:
columns: ["id"]
rows: [["1"], ["2"], ["47"]]
integration-tests/go-sql-server-driver/tests/sql-server-remotesapi.yaml
+100
View File
@@ -0,0 +1,100 @@
parallel: true
tests:
- name: can clone from server1 remotesapi endpoint
# The remotes API (used to serve/clone databases between servers) is not yet
# implemented in Doltgres, so this test is skipped. The config, remotesapi
# port wiring, and dolt_clone/dolt_commit flow are translated faithfully so it
# can be enabled once the feature lands.
skip: "cluster/remotesapi replication not yet implemented in Doltgres"
multi_repos:
- name: server1
repos:
- name: repo1
- name: repo2
with_files:
- name: server.yaml
contents: |
log_level: trace
remotesapi:
port: {{get_port "remotesapi_port"}}
server:
args: ["--config", "server.yaml"]
dynamic_port: server_port
- name: server2
repos:
- name: repo1
with_files:
- name: server.yaml
contents: |
log_level: trace
listener:
host: 0.0.0.0
server:
args: ["--config", "server.yaml"]
dynamic_port: server2_port
connections:
- on: server1
queries:
- exec: "use repo2"
- exec: "create table vals (id int primary key)"
- exec: "insert into vals values (0),(1),(2),(3),(4)"
- exec: "select dolt_commit('-Am', 'insert some data')"
- on: server2
queries:
- exec: "use repo1"
- exec: "select dolt_clone('http://localhost:{{get_port \"remotesapi_port\"}}/repo2')"
- exec: "use repo2"
- query: "select count(*) from vals"
result:
columns: ["count"]
rows: [["5"]]
- name: can clone from server1 remotesapi endpoint after a gc
skip: "cluster/remotesapi replication not yet implemented in Doltgres"
multi_repos:
- name: server1
repos:
- name: repo1
- name: repo2
with_files:
- name: server.yaml
contents: |
log_level: trace
remotesapi:
port: {{get_port "remotesapi_port"}}
server:
args: ["--config", "server.yaml"]
dynamic_port: server_port
- name: server2
repos:
- name: repo1
with_files:
- name: server.yaml
contents: |
log_level: trace
listener:
host: 0.0.0.0
server:
args: ["--config", "server.yaml"]
dynamic_port: server2_port
connections:
- on: server1
queries:
- exec: "use repo2"
- exec: "create table vals (id int primary key)"
- exec: "insert into vals values (0),(1),(2),(3),(4)"
- exec: "select dolt_commit('-Am', 'insert some data')"
- exec: "select dolt_gc()"
- on: server1
queries:
- exec: "use repo2"
- exec: "insert into vals values (5),(6),(7),(8),(9)"
- exec: "select dolt_commit('-Am', 'insert some more data')"
- on: server2
queries:
- exec: "use repo1"
- exec: "select dolt_clone('http://localhost:{{get_port \"remotesapi_port\"}}/repo2')"
- exec: "use repo2"
- query: "select count(*) from vals"
result:
columns: ["count"]
rows: [["10"]]
integration-tests/go-sql-server-driver/tests/sql-server-tls.yaml
+80
View File
@@ -0,0 +1,80 @@
parallel: true
tests:
- name: require_secure_transport no key or cert
repos:
- name: repo1
with_files:
- name: server.yaml
contents: |
listener:
require_secure_transport: true
server:
args: ["--config", "server.yaml"]
dynamic_port: server
error_matches:
- "require_secure_transport can only be `true` when a tls_key and tls_cert are provided."
- name: tls_key non-existent
repos:
- name: repo1
with_files:
- name: chain_key.pem
source_path: $TESTGENDIR/rsa_key.pem
- name: chain_cert.pem
source_path: $TESTGENDIR/rsa_chain.pem
- name: server.yaml
contents: |
listener:
tls_key: doesnotexist_key.pem
tls_cert: chain_cert.pem
server:
args: ["--config", "server.yaml"]
dynamic_port: server
error_matches:
- "no such file or directory"
- name: tls_cert non-existent
repos:
- name: repo1
with_files:
- name: chain_key.pem
source_path: $TESTGENDIR/rsa_key.pem
- name: chain_cert.pem
source_path: $TESTGENDIR/rsa_chain.pem
- name: server.yaml
contents: |
listener:
tls_key: chain_key.pem
tls_cert: doesnotexist_key.pem
server:
args: ["--config", "server.yaml"]
dynamic_port: server
error_matches:
- "no such file or directory"
# XXX: It would be nice to assert on the TLS use here using a status query or similar.
# It would be nice to assert on failing to connect using sslmode=disable.
- name: tls only server
repos:
- name: repo1
with_files:
- name: chain_key.pem
source_path: $TESTGENDIR/rsa_key.pem
- name: chain_cert.pem
source_path: $TESTGENDIR/rsa_chain.pem
- name: server.yaml
contents: |
listener:
tls_key: chain_key.pem
tls_cert: chain_cert.pem
require_secure_transport: true
server:
args: ["--config", "server.yaml"]
dynamic_port: server
connections:
- on: repo1
driver_params:
sslmode: require
queries:
- query: "select 1 as one"
result:
columns: ["one"]
rows: [["1"]]
integration-tests/go-sql-server-driver/tests/sql-server-type-diversity.yaml
+184
View File
@@ -0,0 +1,184 @@
parallel: true
tests:
- name: type diverse table replicates correctly
# Cluster replication is not yet implemented in Doltgres, so this test is
# skipped. The cluster config and the type-diverse schema/data are translated
# to Postgres faithfully so this can be enabled once replication lands.
#
# Type translation notes (MySQL -> Postgres):
# TINYINT -> smallint (Postgres has no 1-byte int)
# DOUBLE -> double precision
# DATETIME -> timestamp
# BLOB -> bytea
# BOOLEAN -> boolean
# JSON -> json
# ENUM(...) / SET(...) -> varchar (Postgres has no inline ENUM/SET); the
# spirit (constrained categorical / multi-value column) is preserved as
# text values.
skip: "cluster/remotesapi replication not yet implemented in Doltgres"
multi_repos:
- name: server1
with_files:
- name: server.yaml
contents: |
log_level: trace
listener:
host: 0.0.0.0
cluster:
standby_remotes:
- name: standby
remote_url_template: http://localhost:{{get_port "server2_cluster"}}/{database}
bootstrap_role: primary
bootstrap_epoch: 1
remotesapi:
port: {{get_port "server1_cluster"}}
server:
args: ["--config", "server.yaml"]
dynamic_port: server1
- name: server2
with_files:
- name: server.yaml
contents: |
log_level: trace
listener:
host: 0.0.0.0
cluster:
standby_remotes:
- name: standby
remote_url_template: http://localhost:{{get_port "server1_cluster"}}/{database}
bootstrap_role: standby
bootstrap_epoch: 1
remotesapi:
port: {{get_port "server2_cluster"}}
server:
args: ["--config", "server.yaml"]
dynamic_port: server2
connections:
- on: server1
queries:
- exec: "SET dolt_cluster_ack_writes_timeout_secs = 10"
- exec: 'CREATE DATABASE repo1'
- exec: 'USE repo1'
- exec: |
CREATE TABLE type_diverse (
id BIGINT PRIMARY KEY,
col_tinyint SMALLINT,
col_smallint SMALLINT,
col_int INT,
col_bigint BIGINT,
col_decimal DECIMAL(20,10),
col_float REAL,
col_double DOUBLE PRECISION,
col_char CHAR(50),
col_varchar VARCHAR(500),
col_text TEXT,
col_blob BYTEA,
col_date DATE,
col_time TIME,
col_datetime TIMESTAMP,
col_bool BOOLEAN,
col_json JSON,
col_enum VARCHAR(16),
col_set VARCHAR(64)
)
# Row 1: typical values.
- exec: |
INSERT INTO type_diverse VALUES (
1, 42, 1000, 1000000, 9000000000,
12345678.9012345678, 3.14, 2.71828182845905,
'hello', 'longer varchar value',
'some text content', '\x736f6d6520626c6f6220636f6e74656e74',
'2024-06-15', '12:30:00', '2024-06-15 12:30:00',
true, json_build_object('key','value','count',42),
'alpha', 'red,green'
)
# Row 2: minimum boundary values.
- exec: |
INSERT INTO type_diverse VALUES (
2, -128, -32768, -2147483648, -9223372036854775808,
-9999999999.9999999999, -1.5, -1.23456789,
'', '',
'', '\x',
'1000-01-01', '00:00:00', '1000-01-01 00:00:00',
false, json_build_object(),
'beta', 'green'
)
# Row 3: maximum/large values - large TEXT and BLOB cross the out-of-band threshold.
- exec: |
INSERT INTO type_diverse VALUES (
3, 127, 32767, 2147483647, 9223372036854775807,
9999999999.9999999999, 1.5, 9.87654321,
REPEAT('x',50), REPEAT('y',500),
REPEAT('z',15000), decode(REPEAT('62',20000), 'hex'),
'9999-12-31', '23:59:59', '9999-12-31 23:59:59',
true, json_build_object('data', REPEAT('d',11000)),
'gamma', 'red,green,blue'
)
# Row 4: all nullable columns are NULL.
- exec: 'INSERT INTO type_diverse (id) VALUES (4)'
- query: "SELECT COUNT(*) AS count FROM type_diverse"
result:
columns: ["count"]
rows: [["4"]]
- query: "SELECT COUNT(*) AS count FROM type_diverse WHERE col_int = -2147483648 AND col_bigint = -9223372036854775808"
result:
columns: ["count"]
rows: [["1"]]
- query: "SELECT COUNT(*) AS count FROM type_diverse WHERE LENGTH(col_text) = 15000 AND LENGTH(col_blob) = 20000"
result:
columns: ["count"]
rows: [["1"]]
- query: "SELECT COUNT(*) AS count FROM type_diverse WHERE col_tinyint IS NULL AND col_date IS NULL AND col_json IS NULL"
result:
columns: ["count"]
rows: [["1"]]
- on: server2
retry_attempts: 100
queries:
- exec: 'USE repo1'
- query: "SELECT COUNT(*) AS count FROM type_diverse"
result:
columns: ["count"]
rows: [["4"]]
# Integer boundaries must survive the replication round-trip.
- query: "SELECT COUNT(*) AS count FROM type_diverse WHERE col_tinyint = -128 AND col_int = -2147483648 AND col_bigint = -9223372036854775808"
result:
columns: ["count"]
rows: [["1"]]
- query: "SELECT COUNT(*) AS count FROM type_diverse WHERE col_tinyint = 127 AND col_int = 2147483647 AND col_bigint = 9223372036854775807"
result:
columns: ["count"]
rows: [["1"]]
# Large out-of-band values must be present with their correct sizes.
- query: "SELECT COUNT(*) AS count FROM type_diverse WHERE LENGTH(col_text) = 15000 AND LENGTH(col_blob) = 20000"
result:
columns: ["count"]
rows: [["1"]]
# Date and time boundary values must be preserved.
- query: "SELECT COUNT(*) AS count FROM type_diverse WHERE col_date = '1000-01-01' AND col_time = '00:00:00'"
result:
columns: ["count"]
rows: [["1"]]
- query: "SELECT COUNT(*) AS count FROM type_diverse WHERE col_date = '9999-12-31' AND col_time = '23:59:59'"
result:
columns: ["count"]
rows: [["1"]]
# ENUM/SET (translated to varchar) must replicate correctly.
- query: "SELECT col_enum, col_set FROM type_diverse WHERE id = 1"
result:
columns: ["col_enum","col_set"]
rows: [["alpha","red,green"]]
- query: "SELECT col_enum, col_set FROM type_diverse WHERE id = 3"
result:
columns: ["col_enum","col_set"]
rows: [["gamma","red,green,blue"]]
- query: "SELECT col_json->>'key' AS key FROM type_diverse WHERE id = 1"
result:
columns: ["key"]
rows: [["value"]]
# NULL values must survive replication.
- query: "SELECT COUNT(*) AS count FROM type_diverse WHERE col_tinyint IS NULL AND col_date IS NULL AND col_json IS NULL"
result:
columns: ["count"]
rows: [["1"]]
integration-tests/go-sql-server-driver/tests/sql-server-wide-int-table.yaml
+99
View File
@@ -0,0 +1,99 @@
parallel: true
tests:
- name: wide integer column table replicates correctly
skip: "cluster replication/failover not yet implemented in Doltgres"
multi_repos:
- name: server1
with_files:
- name: server.yaml
contents: |
log_level: trace
behavior:
auto_gc_behavior:
enable: false
cluster:
standby_remotes:
- name: standby
remote_url_template: http://localhost:{{get_port "server2_cluster"}}/{database}
bootstrap_role: primary
bootstrap_epoch: 1
remotesapi:
port: {{get_port "server1_cluster"}}
server:
args: ["--config", "server.yaml"]
dynamic_port: server1
- name: server2
with_files:
- name: server.yaml
contents: |
log_level: trace
behavior:
auto_gc_behavior:
enable: false
cluster:
standby_remotes:
- name: standby
remote_url_template: http://localhost:{{get_port "server1_cluster"}}/{database}
bootstrap_role: standby
bootstrap_epoch: 1
remotesapi:
port: {{get_port "server2_cluster"}}
server:
args: ["--config", "server.yaml"]
dynamic_port: server2
connections:
- on: server1
queries:
- exec: 'SET dolt_cluster_ack_writes_timeout_secs = 10'
- exec: 'CREATE DATABASE repo1'
- exec: 'USE repo1'
- exec: |
CREATE TABLE wide_int (
id BIGINT PRIMARY KEY,
c00 BIGINT, c01 BIGINT, c02 BIGINT, c03 BIGINT, c04 BIGINT,
c05 BIGINT, c06 BIGINT, c07 BIGINT, c08 BIGINT, c09 BIGINT,
c10 BIGINT, c11 BIGINT, c12 BIGINT, c13 BIGINT, c14 BIGINT,
c15 BIGINT, c16 BIGINT, c17 BIGINT, c18 BIGINT, c19 BIGINT,
c20 BIGINT, c21 BIGINT, c22 BIGINT, c23 BIGINT, c24 BIGINT,
c25 BIGINT, c26 BIGINT, c27 BIGINT, c28 BIGINT, c29 BIGINT
)
- exec: |
INSERT INTO wide_int VALUES
(1, 100,101,102,103,104, 105,106,107,108,109, 110,111,112,113,114, 115,116,117,118,119, 120,121,122,123,124, 125,126,127,128,129),
(2, 200,201,202,203,204, 205,206,207,208,209, 210,211,212,213,214, 215,216,217,218,219, 220,221,222,223,224, 225,226,227,228,229),
(3, 300,301,302,303,304, 305,306,307,308,309, 310,311,312,313,314, 315,316,317,318,319, 320,321,322,323,324, 325,326,327,328,329),
(4, -100,-101,-102,-103,-104,-105,-106,-107,-108,-109,-110,-111,-112,-113,-114,-115,-116,-117,-118,-119,-120,-121,-122,-123,-124,-125,-126,-127,-128,-129),
(5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0)
- query: "SELECT COUNT(*) AS count FROM wide_int"
result:
columns: ["count"]
rows: [["5"]]
- query: "SELECT c00, c15, c29 FROM wide_int WHERE id = 1"
result:
columns: ["c00","c15","c29"]
rows: [["100","115","129"]]
- on: server2
retry_attempts: 100
queries:
- exec: 'USE repo1'
- query: "SELECT COUNT(*) AS count FROM wide_int"
result:
columns: ["count"]
rows: [["5"]]
- query: "SELECT c00, c15, c29 FROM wide_int WHERE id = 1"
result:
columns: ["c00","c15","c29"]
rows: [["100","115","129"]]
- query: "SELECT c00, c15, c29 FROM wide_int WHERE id = 3"
result:
columns: ["c00","c15","c29"]
rows: [["300","315","329"]]
- query: "SELECT c00, c15, c29 FROM wide_int WHERE id = 4"
result:
columns: ["c00","c15","c29"]
rows: [["-100","-115","-129"]]
- query: "SELECT COUNT(*) AS count FROM wide_int WHERE c00 = 0 AND c29 = 0"
result:
columns: ["count"]
rows: [["1"]]
integration-tests/go-sql-server-driver/tests/sql-server-wide-table-failover.yaml
+125
View File
@@ -0,0 +1,125 @@
parallel: true
tests:
- name: wide table survives failover
skip: "cluster replication/failover not yet implemented in Doltgres"
multi_repos:
- name: server1
with_files:
- name: server.yaml
contents: |
log_level: trace
behavior:
auto_gc_behavior:
enable: false
cluster:
standby_remotes:
- name: standby
remote_url_template: http://localhost:{{get_port "server2_cluster"}}/{database}
bootstrap_role: primary
bootstrap_epoch: 1
remotesapi:
port: {{get_port "server1_cluster"}}
server:
args: ["--config", "server.yaml"]
dynamic_port: server1
- name: server2
with_files:
- name: server.yaml
contents: |
log_level: trace
behavior:
auto_gc_behavior:
enable: false
cluster:
standby_remotes:
- name: standby
remote_url_template: http://localhost:{{get_port "server1_cluster"}}/{database}
bootstrap_role: standby
bootstrap_epoch: 1
remotesapi:
port: {{get_port "server2_cluster"}}
server:
args: ["--config", "server.yaml"]
dynamic_port: server2
connections:
- on: server1
queries:
- exec: 'SET dolt_cluster_ack_writes_timeout_secs = 10'
- exec: 'CREATE DATABASE repo1'
- exec: 'USE repo1'
- exec: |
CREATE TABLE wide_mixed (
id BIGINT PRIMARY KEY,
i00 INT, i01 INT, i02 INT, i03 INT, i04 INT,
i05 INT, i06 INT, i07 INT, i08 INT, i09 INT,
t00 TEXT, t01 TEXT, t02 TEXT, t03 TEXT, t04 TEXT,
v00 VARCHAR(300), v01 VARCHAR(300), v02 VARCHAR(300), v03 VARCHAR(300), v04 VARCHAR(300)
)
- exec: |
INSERT INTO wide_mixed VALUES
(1,
1,2,3,4,5,6,7,8,9,10,
REPEAT('t00',667), REPEAT('t01',667), REPEAT('t02',667), REPEAT('t03',667), REPEAT('t04',667),
REPEAT('v',300), REPEAT('w',300), REPEAT('x',300), REPEAT('y',300), REPEAT('z',300))
- exec: |
INSERT INTO wide_mixed VALUES
(2,
11,12,13,14,15,16,17,18,19,20,
REPEAT('T00',667), REPEAT('T01',667), REPEAT('T02',667), REPEAT('T03',667), REPEAT('T04',667),
REPEAT('V',300), REPEAT('W',300), REPEAT('X',300), REPEAT('Y',300), REPEAT('Z',300))
- exec: |
INSERT INTO wide_mixed VALUES
(3,
-1,-2,-3,-4,-5,-6,-7,-8,-9,-10,
REPEAT('neg',667), REPEAT('zero',501), REPEAT('mix',668), REPEAT('data',501), REPEAT('test',501),
REPEAT('a',300), REPEAT('b',300), REPEAT('c',300), REPEAT('d',300), REPEAT('e',300))
- query: "SELECT COUNT(*) AS count FROM wide_mixed"
result:
columns: ["count"]
rows: [["3"]]
- query: "SELECT i00, i09, LEFT(t00,3) AS t00, LEFT(v04,1) AS v04 FROM wide_mixed WHERE id = 1"
result:
columns: ["i00","i09","t00","v04"]
rows: [["1","10","t00","z"]]
- query: "SELECT dolt_assume_cluster_role('standby', 2)"
result:
columns: ["dolt_assume_cluster_role"]
rows: [["{0}"]]
- on: server2
retry_attempts: 100
queries:
- exec: 'USE repo1'
- query: "SELECT COUNT(*) AS count FROM wide_mixed"
result:
columns: ["count"]
rows: [["3"]]
- query: "SELECT i00, i09, LEFT(t00,3) AS t00, LEFT(v04,1) AS v04 FROM wide_mixed WHERE id = 2"
result:
columns: ["i00","i09","t00","v04"]
rows: [["11","20","T00","Z"]]
- query: "SELECT dolt_assume_cluster_role('primary', 2)"
result:
columns: ["dolt_assume_cluster_role"]
rows: [["{0}"]]
- on: server2
queries:
- exec: 'USE repo1'
- exec: |
INSERT INTO wide_mixed VALUES
(4,
100,200,300,400,500,600,700,800,900,1000,
REPEAT('new',667), REPEAT('row',667), REPEAT('ins',667), REPEAT('ert',667), REPEAT('ed!',668),
REPEAT('N',300), REPEAT('E',300), REPEAT('W',300), REPEAT('R',300), REPEAT('O',300))
- query: "SELECT COUNT(*) AS count FROM wide_mixed"
result:
columns: ["count"]
rows: [["4"]]
- query: "SELECT i00, LEFT(t00,3) AS t00 FROM wide_mixed WHERE id = 4"
result:
columns: ["i00","t00"]
rows: [["100","new"]]
- query: "SELECT COUNT(*) AS count FROM wide_mixed WHERE LENGTH(t00) > 1000"
result:
columns: ["count"]
rows: [["4"]]
integration-tests/go-sql-server-driver/tests/sql-server-wide-text-table.yaml
+105
View File
@@ -0,0 +1,105 @@
parallel: true
tests:
- name: wide text column table replicates correctly
skip: "cluster replication/failover not yet implemented in Doltgres"
multi_repos:
- name: server1
with_files:
- name: server.yaml
contents: |
log_level: trace
behavior:
auto_gc_behavior:
enable: false
cluster:
standby_remotes:
- name: standby
remote_url_template: http://localhost:{{get_port "server2_cluster"}}/{database}
bootstrap_role: primary
bootstrap_epoch: 1
remotesapi:
port: {{get_port "server1_cluster"}}
server:
args: ["--config", "server.yaml"]
dynamic_port: server1
- name: server2
with_files:
- name: server.yaml
contents: |
log_level: trace
behavior:
auto_gc_behavior:
enable: false
cluster:
standby_remotes:
- name: standby
remote_url_template: http://localhost:{{get_port "server1_cluster"}}/{database}
bootstrap_role: standby
bootstrap_epoch: 1
remotesapi:
port: {{get_port "server2_cluster"}}
server:
args: ["--config", "server.yaml"]
dynamic_port: server2
connections:
- on: server1
queries:
- exec: 'SET dolt_cluster_ack_writes_timeout_secs = 10'
- exec: 'CREATE DATABASE repo1'
- exec: 'USE repo1'
# 20 TEXT columns × 2 KB per cell = ~40 KB of out-of-band data per row.
# Each row requires 20 separate external chunk references.
- exec: |
CREATE TABLE wide_text (
id BIGINT PRIMARY KEY,
t00 TEXT, t01 TEXT, t02 TEXT, t03 TEXT, t04 TEXT,
t05 TEXT, t06 TEXT, t07 TEXT, t08 TEXT, t09 TEXT,
t10 TEXT, t11 TEXT, t12 TEXT, t13 TEXT, t14 TEXT,
t15 TEXT, t16 TEXT, t17 TEXT, t18 TEXT, t19 TEXT
)
- exec: |
INSERT INTO wide_text VALUES
(1,
REPEAT('a',2000), REPEAT('b',2000), REPEAT('c',2000), REPEAT('d',2000), REPEAT('e',2000),
REPEAT('f',2000), REPEAT('g',2000), REPEAT('h',2000), REPEAT('i',2000), REPEAT('j',2000),
REPEAT('k',2000), REPEAT('l',2000), REPEAT('m',2000), REPEAT('n',2000), REPEAT('o',2000),
REPEAT('p',2000), REPEAT('q',2000), REPEAT('r',2000), REPEAT('s',2000), REPEAT('t',2000))
- exec: |
INSERT INTO wide_text VALUES
(2,
REPEAT('A',2000), REPEAT('B',2000), REPEAT('C',2000), REPEAT('D',2000), REPEAT('E',2000),
REPEAT('F',2000), REPEAT('G',2000), REPEAT('H',2000), REPEAT('I',2000), REPEAT('J',2000),
REPEAT('K',2000), REPEAT('L',2000), REPEAT('M',2000), REPEAT('N',2000), REPEAT('O',2000),
REPEAT('P',2000), REPEAT('Q',2000), REPEAT('R',2000), REPEAT('S',2000), REPEAT('T',2000))
- exec: |
INSERT INTO wide_text VALUES
(3,
REPEAT('1',2000), REPEAT('2',2000), REPEAT('3',2000), REPEAT('4',2000), REPEAT('5',2000),
REPEAT('6',2000), REPEAT('7',2000), REPEAT('8',2000), REPEAT('9',2000), REPEAT('0',2000),
REPEAT('1',2000), REPEAT('2',2000), REPEAT('3',2000), REPEAT('4',2000), REPEAT('5',2000),
REPEAT('6',2000), REPEAT('7',2000), REPEAT('8',2000), REPEAT('9',2000), REPEAT('0',2000))
- query: "SELECT COUNT(*) AS count FROM wide_text WHERE LENGTH(t00) = 2000 AND LENGTH(t19) = 2000"
result:
columns: ["count"]
rows: [["3"]]
- query: "SELECT LEFT(t00,1) AS t00, LEFT(t10,1) AS t10 FROM wide_text WHERE id = 1"
result:
columns: ["t00","t10"]
rows: [["a","k"]]
- on: server2
retry_attempts: 100
queries:
- exec: 'USE repo1'
- query: "SELECT COUNT(*) AS count FROM wide_text WHERE LENGTH(t00) = 2000 AND LENGTH(t19) = 2000"
result:
columns: ["count"]
rows: [["3"]]
- query: "SELECT LEFT(t00,1) AS t00, LEFT(t10,1) AS t10 FROM wide_text WHERE id = 1"
result:
columns: ["t00","t10"]
rows: [["a","k"]]
- query: "SELECT LEFT(t00,1) AS t00, LEFT(t10,1) AS t10 FROM wide_text WHERE id = 2"
result:
columns: ["t00","t10"]
rows: [["A","K"]]
integration-tests/go-sql-server-driver/tests/sql-server-wide-varchar-table.yaml
+116
View File
@@ -0,0 +1,116 @@
parallel: true
tests:
- name: wide varchar column table replicates and survives gc
skip: "cluster replication/failover not yet implemented in Doltgres"
multi_repos:
- name: server1
with_files:
- name: server.yaml
contents: |
log_level: trace
behavior:
auto_gc_behavior:
enable: false
cluster:
standby_remotes:
- name: standby
remote_url_template: http://localhost:{{get_port "server2_cluster"}}/{database}
bootstrap_role: primary
bootstrap_epoch: 1
remotesapi:
port: {{get_port "server1_cluster"}}
server:
args: ["--config", "server.yaml"]
dynamic_port: server1
- name: server2
with_files:
- name: server.yaml
contents: |
log_level: trace
behavior:
auto_gc_behavior:
enable: false
cluster:
standby_remotes:
- name: standby
remote_url_template: http://localhost:{{get_port "server1_cluster"}}/{database}
bootstrap_role: standby
bootstrap_epoch: 1
remotesapi:
port: {{get_port "server2_cluster"}}
server:
args: ["--config", "server.yaml"]
dynamic_port: server2
connections:
- on: server1
queries:
- exec: 'SET dolt_cluster_ack_writes_timeout_secs = 10'
- exec: 'CREATE DATABASE repo1'
- exec: 'USE repo1'
# 8 VARCHAR(300) columns: Dolt's per-column accounting is ~8000 bytes for
# VARCHAR(300), so 8 columns puts the row near (but under) Dolt's 64k inline limit.
- exec: |
CREATE TABLE wide_varchar (
id BIGINT PRIMARY KEY,
v00 VARCHAR(300), v01 VARCHAR(300), v02 VARCHAR(300), v03 VARCHAR(300),
v04 VARCHAR(300), v05 VARCHAR(300), v06 VARCHAR(300), v07 VARCHAR(300)
)
- exec: |
INSERT INTO wide_varchar VALUES
(1,
REPEAT('a',300), REPEAT('b',300), REPEAT('c',300), REPEAT('d',300),
REPEAT('e',300), REPEAT('f',300), REPEAT('g',300), REPEAT('h',300))
- exec: |
INSERT INTO wide_varchar VALUES
(2,
REPEAT('A',300), REPEAT('B',300), REPEAT('C',300), REPEAT('D',300),
REPEAT('E',300), REPEAT('F',300), REPEAT('G',300), REPEAT('H',300))
- exec: |
INSERT INTO wide_varchar VALUES
(3,
REPEAT('0',300), REPEAT('1',300), REPEAT('2',300), REPEAT('3',300),
REPEAT('4',300), REPEAT('5',300), REPEAT('6',300), REPEAT('7',300))
- query: "SELECT COUNT(*) AS count FROM wide_varchar WHERE LENGTH(v00) = 300 AND LENGTH(v07) = 300"
result:
columns: ["count"]
rows: [["3"]]
- exec: 'SELECT dolt_gc()'
- query: 'SELECT COUNT(*) FROM wide_varchar'
error_match: "this connection can no longer be used"
- on: server1
queries:
- query: |
SELECT "database", standby_remote, role, epoch, replication_lag_millis, current_error
FROM dolt_cluster.dolt_cluster_status
ORDER BY "database" ASC
result:
columns: ["database","standby_remote","role","epoch","replication_lag_millis","current_error"]
rows:
- ["repo1","standby","primary","1","0","NULL"]
retry_attempts: 100
- exec: 'USE repo1'
- query: "SELECT COUNT(*) AS count FROM wide_varchar WHERE LENGTH(v00) = 300 AND LENGTH(v07) = 300"
result:
columns: ["count"]
rows: [["3"]]
- query: "SELECT LEFT(v00, 1) AS v00, LEFT(v07, 1) AS v07 FROM wide_varchar WHERE id = 1"
result:
columns: ["v00","v07"]
rows: [["a","h"]]
- on: server2
queries:
- exec: 'USE repo1'
- query: "SELECT COUNT(*) AS count FROM wide_varchar WHERE LENGTH(v00) = 300 AND LENGTH(v07) = 300"
result:
columns: ["count"]
rows: [["3"]]
- query: "SELECT LEFT(v00, 1) AS v00, LEFT(v07, 1) AS v07 FROM wide_varchar WHERE id = 2"
result:
columns: ["v00","v07"]
rows: [["A","H"]]
- exec: "SELECT dolt_gc('--shallow')"
- query: "SELECT COUNT(*) AS count FROM wide_varchar WHERE LENGTH(v00) = 300 AND LENGTH(v07) = 300"
result:
columns: ["count"]
rows: [["3"]]