chore: import upstream snapshot with attribution

This commit is contained in:
wehub-resource-sync
2026-07-13 12:32:25 +08:00
commit e014feafe1
2285 changed files with 1131979 additions and 0 deletions
+52
View File
@@ -0,0 +1,52 @@
# Exclude VCS and CI metadata
.git
.git/**
.github/
.gitmodules
# OS / editor cruft
.DS_Store
.idea/
.vscode/
*.swp
# Node modules (only present in test helpers)
node_modules/
**/node_modules/
# Tests and large fixtures (not needed for image build)
# TODO: restore this ignore, required for CI for now
#testing/
# Build artifacts and caches (created locally or by scripts)
out/
bin/
dist/
coverage/
*.log
*.tmp
*.cache
*.pprof
# Go vendor cache (not used; modules are downloaded in image)
vendor/
# Local env/virtualenvs
venv/
# Sysbench copied files (from .gitignore)
SysbenchDockerfile
SysbenchDockerfile.dockerignore
sysbench-runner-tests-entrypoint.sh
config.json
integration-tests/
# Datadirs created locally (from .gitignore)
doltgres/
# Extension build outputs
core/extensions/pg_extension/output/
# Keep only what we might need for source builds
# (Go source, scripts, entrypoint, flatbuffers, core, server, utils, etc. remain included by default)
# If you need to further constrain context, switch to an allowlist pattern.
+3
View File
@@ -0,0 +1,3 @@
# These are supported funding model platforms
github: dolthub
@@ -0,0 +1 @@
node_modules
@@ -0,0 +1,50 @@
name: 'SES email action'
description: 'Send email with aws ses'
inputs:
region:
description: 'aws region'
required: true
default: ''
version:
description: 'doltgres version being benchmarked or ref of bats windows job'
required: true
default: ''
format:
description: 'doltgres noms bin format'
required: true
default: '__DOLT__'
template:
description: 'email template'
required: false
default: 'PerformanceBenchmarkingReleaseTemplate'
dataFile:
required: false
description: 'path to email data file'
default: ''
bodyPath:
required: false
description: 'path to email body file'
default: ''
subject:
required: false
description: 'path to email subject'
default: ''
toAddresses:
description: 'json string list of to addresses'
required: true
default: "[]"
ccAddresses:
description: 'json string list of cc addresses'
required: false
default: "[]"
replyToAddresses:
description: 'json string list of reply to addresses'
required: false
default: "[]"
workflowURL:
description: 'url of the workflow run'
default: ''
required: false
runs:
using: 'node20'
main: 'dist/index.js'
File diff suppressed because one or more lines are too long
+50
View File
@@ -0,0 +1,50 @@
const core = require('@actions/core');
const { SES } = require("@aws-sdk/client-ses");
const fs = require('fs');
const region = core.getInput('region');
const version = core.getInput('version');
const format = core.getInput('format');
const Template = core.getInput('template');
const dataFilePath = core.getInput('dataFile');
const CcAddresses = JSON.parse(core.getInput('ccAddresses'));
const ToAddresses = JSON.parse(core.getInput('toAddresses'));
const ReplyToAddresses = JSON.parse(core.getInput('replyToAddresses'));
const workflowURL = core.getInput('workflowURL');
const subject = core.getInput('subject');
const bodyPath = core.getInput('bodyPath');
const data = dataFilePath ? fs.readFileSync(dataFilePath, { encoding: 'utf-8' }) : "";
const body = bodyPath ? fs.readFileSync(bodyPath, { encoding: 'utf-8' }) : "";
const templated = {
version,
format,
results: data,
workflowURL,
subject,
body,
};
// Create sendEmail params
const params = {
Destination: { /* required */
CcAddresses,
ToAddresses,
},
Source: 'github-actions-bot@corp.ld-corp.com', /* required */
Template,
TemplateData: JSON.stringify(templated),
ReplyToAddresses,
};
console.log(params)
// Create the promise and SES service object
const sendPromise = new SES({region}).sendTemplatedEmail(params);
// Handle promise's fulfilled/rejected states
sendPromise
.then((data) => console.log("Successfully sent email:", data.MessageId))
.catch((err) => console.error(err, err.stack));
File diff suppressed because it is too large Load Diff
@@ -0,0 +1,16 @@
{
"name": "ses-email-action",
"version": "0.1.0",
"description": "",
"main": "index.js",
"keywords": [],
"author": "dolthub",
"license": "ISC",
"scripts": {
"build": "ncc build index.js"
},
"dependencies": {
"@actions/core": "^1.11.1",
"@aws-sdk/client-ses": "^3.1038.0"
}
}
+9
View File
@@ -0,0 +1,9 @@
:coffee: *An Automated Dependency Version Bump PR* :crown:
### Initial Changes
The changes contained in this PR were produced by `go get`ing the dependency.
```bash
go get github.com/dolthub/[dependency]/go@[commit]
```
@@ -0,0 +1,125 @@
#!/bin/sh
set -e
if [ "$#" -lt 9 ]; then
echo "Usage: ./get-job-json.sh <jobname> <fromServer> <fromVersion> <toServer> <toVersion> <timePrefix> <actorPrefix> <format> <issueNumber> <initBigRepo> <nomsBinFormat> <withTpcc>"
exit 1
fi
jobName="$1"
fromServer="$2"
fromVersion="$3"
toServer="$4"
toVersion="$5"
timePrefix="$6"
actorPrefix="$7"
format="$8"
issueNumber="$9"
initBigRepo="${10}"
nomsBinFormat="${11}"
withTpcc="${12}"
tpccRegex="tpcc%"
if [ -n "$initBigRepo" ]; then
initBigRepo="\"--init-big-repo=$initBigRepo\","
fi
if [ -n "$nomsBinFormat" ]; then
nomsBinFormat="\"--noms-bin-format=$nomsBinFormat\","
fi
if [ -n "$withTpcc" ]; then
withTpcc="\"--withTpcc=$withTpcc\","
fi
readTests="('oltp_read_only', 'oltp_point_select', 'select_random_points', 'select_random_ranges', 'covering_index_scan_postgres', 'index_scan_postgres', 'table_scan_postgres', 'groupby_scan_postgres', 'index_join_scan_postgres', 'types_table_scan_postgres', 'index_join_postgres')"
medianLatencyChangeReadsQuery="select f.test_name as read_tests, case when avg(f.latency_percentile) < 0.001 then 0.001 else avg(f.latency_percentile) end as from_latency_median, case when avg(t.latency_percentile) < 0.001 then 0.001 else avg(t.latency_percentile) end as to_latency_median, case when ((avg(t.latency_percentile) - avg(f.latency_percentile)) / (avg(f.latency_percentile) + .0000001)) < -0.1 then 1 when ((avg(t.latency_percentile) - avg(f.latency_percentile)) / (avg(f.latency_percentile) + .0000001)) > 0.1 then -1 else 0 end as is_faster from from_results as f join to_results as t on f.test_name = t.test_name where f.test_name in $readTests group by f.test_name;"
writeTests="('oltp_read_write', 'oltp_update_index', 'oltp_update_non_index', 'oltp_insert', 'oltp_write_only', 'bulk_insert', 'oltp_delete_insert_postgres', 'types_delete_insert_postgres')"
medianLatencyChangeWritesQuery="select f.test_name as write_tests, case when avg(f.latency_percentile) < 0.001 then 0.001 else avg(f.latency_percentile) end as from_latency_median, case when avg(t.latency_percentile) < 0.001 then 0.001 else avg(t.latency_percentile) end as to_latency_median, case when ((avg(t.latency_percentile) - avg(f.latency_percentile)) / (avg(f.latency_percentile) + .0000001)) < -0.1 then 1 when ((avg(t.latency_percentile) - avg(f.latency_percentile)) / (avg(f.latency_percentile) + .0000001)) > 0.1 then -1 else 0 end as is_faster from from_results as f join to_results as t on f.test_name = t.test_name where f.test_name in $writeTests group by f.test_name;"
tpccLatencyQuery="select f.test_name as test_name, case when avg(f.latency_percentile) < 0.001 then 0.001 else avg(f.latency_percentile) end as from_latency_median, case when avg(t.latency_percentile) < 0.001 then 0.001 else avg(t.latency_percentile) end as to_latency_median, case when ((avg(t.latency_percentile) - avg(f.latency_percentile)) / (avg(f.latency_percentile) + .0000001)) < -0.25 then 1 when ((avg(t.latency_percentile) - avg(f.latency_percentile)) / (avg(f.latency_percentile) + .0000001)) > 0.25 then -1 else 0 end as is_faster from from_results as f join to_results as t on f.test_name = t.test_name where f.test_name LIKE '$tpccRegex' group by f.test_name;"
tpccTpsQuery="select f.test_name as test_name, f.server_name, f.server_version, avg(f.sql_transactions_per_second) as tps, t.test_name as test_name, t.server_name, t.server_version, avg(t.sql_transactions_per_second) as tps, case when ((avg(t.sql_transactions_per_second) - avg(f.sql_transactions_per_second)) / (avg(f.sql_transactions_per_second) + .0000001)) < -0.5 then 1 when ((avg(t.sql_transactions_per_second) - avg(f.sql_transactions_per_second)) / (avg(f.sql_transactions_per_second) + .0000001)) > 0.5 then -1 else 0 end as is_faster from from_results as f join to_results as t on f.test_name = t.test_name where f.test_name LIKE 'tpcc%' group by f.test_name;"
echo '
{
"apiVersion": "batch/v1",
"kind": "Job",
"metadata": {
"name": "'$jobName'",
"namespace": "performance-benchmarking"
},
"spec": {
"backoffLimit": 1,
"template": {
"metadata": {
"annotations": {
"alert_recipients": "'$ACTOR_EMAIL'"
},
"labels": {
"k8s-liquidata-inc-monitored-job": "created-by-static-config"
}
},
"spec": {
"serviceAccountName": "performance-benchmarking",
"containers": [
{
"name": "performance-benchmarking",
"image": "407903926827.dkr.ecr.us-west-2.amazonaws.com/liquidata/performance-benchmarking:latest",
"resources": {
"limits": {
"cpu": "7000m"
}
},
"env": [
{ "name": "GOMAXPROCS", "value": "7" },
{ "name": "ACTOR", "value": "'$ACTOR'" },
{ "name": "ACTOR_EMAIL", "value": "'$ACTOR_EMAIL'" },
{ "name": "REPO_ACCESS_TOKEN", "value": "'$REPO_ACCESS_TOKEN'" }
],
"imagePullPolicy": "Always",
"args": [
"--schema=/app/schema.sql",
"--useDoltHubLuaScriptsRepo",
"--repo=doltgresql",
"--output='$format'",
"--postgres-exec=/usr/lib/postgresql/15/bin/postgres",
"--init-db-exec=/usr/lib/postgresql/15/bin/initdb",
"--email-template=DoltgresPerformanceBenchmarkingReleaseTemplate",
"--from-server='$fromServer'",
"--from-version='$fromVersion'",
"--to-server='$toServer'",
"--to-version='$toVersion'",
"--bucket=performance-benchmarking-github-actions-results",
"--region=us-west-2",
"--issue-number='$issueNumber'",
"--results-dir='$timePrefix'",
"--results-prefix='$actorPrefix'",
'"$withTpcc"'
'"$initBigRepo"'
'"$nomsBinFormat"'
"--sysbenchQueries='"$medianLatencyChangeReadsQuery"'",
"--sysbenchQueries='"$medianLatencyChangeWritesQuery"'",
"--tpccQueries='"$tpccLatencyQuery"'",
"--tpccQueries='"$tpccTpsQuery"'"
]
}
],
"restartPolicy": "Never",
"nodeSelector": {
"performance-benchmarking-worker": "true"
},
"tolerations": [
{
"effect": "NoSchedule",
"key": "dedicated",
"operator": "Equal",
"value": "performance-benchmarking-worker"
}
]
}
}
}
}
'
@@ -0,0 +1,133 @@
#!/bin/sh
set -e
if [ "$#" -lt 9 ]; then
echo "Usage: ./get-job-json.sh <jobname> <fromServer> <fromVersion> <toServer> <toVersion> <timeprefix> <actorprefix> <format> <issueNumber> <initBigRepo> <nomsBinFormat> <withTpcc>"
exit 1
fi
jobname="$1"
fromServer="$2"
fromVersion="$3"
toServer="$4"
toVersion="$5"
timeprefix="$6"
actorprefix="$7"
format="$8"
issueNumber="$9"
initBigRepo="${10}"
nomsBinFormat="${11}"
withTpcc="${12}"
precision="1"
tpccRegex="tpcc%"
if [ -n "$initBigRepo" ]; then
initBigRepo="\"--init-big-repo=$initBigRepo\","
fi
if [ -n "$nomsBinFormat" ]; then
nomsBinFormat="\"--noms-bin-format=$nomsBinFormat\","
fi
if [ -n "$withTpcc" ]; then
withTpcc="\"--withTpcc=$withTpcc\","
fi
readTests="('oltp_read_only', 'oltp_point_select', 'select_random_points', 'select_random_ranges', 'covering_index_scan_postgres', 'index_scan_postgres', 'table_scan_postgres', 'groupby_scan_postgres', 'index_join_scan_postgres', 'types_table_scan_postgres', 'index_join_postgres')"
medianLatencyMultiplierReadsQuery="select f.test_name as read_tests, f.server_name, f.server_version, avg(f.latency_percentile) as from_latency_median, t.server_name, t.server_version, avg(t.latency_percentile) as to_latency_median, ROUND(avg(t.latency_percentile) / (avg(f.latency_percentile) + .000001), $precision) as multiplier from from_results as f join to_results as t on f.test_name = t.test_name where f.test_name in $readTests group by f.test_name;"
meanMultiplierReadsQuery="select round(avg(multipliers), $precision) as reads_mean_multiplier from (select (round(avg(t.latency_percentile) / (avg(f.latency_percentile) + .000001), $precision)) as multipliers from from_results as f join to_results as t on f.test_name = t.test_name where f.test_name in $readTests group by f.test_name)"
writeTests="('oltp_read_write', 'oltp_update_index', 'oltp_update_non_index', 'oltp_insert', 'oltp_write_only', 'oltp_delete_insert_postgres', 'types_delete_insert_postgres')"
medianLatencyMultiplierWritesQuery="select f.test_name as write_tests, f.server_name, f.server_version, avg(f.latency_percentile) as from_latency_median, t.server_name, t.server_version, avg(t.latency_percentile) as to_latency_median, ROUND(avg(t.latency_percentile) / (avg(f.latency_percentile) + .000001), $precision) as multiplier from from_results as f join to_results as t on f.test_name = t.test_name where f.test_name in $writeTests group by f.test_name;"
meanMultiplierWritesQuery="select round(avg(multipliers), $precision) as writes_mean_multiplier from (select (round(avg(t.latency_percentile) / (avg(f.latency_percentile) + .000001), $precision)) as multipliers from from_results as f join to_results as t on f.test_name = t.test_name where f.test_name in $writeTests group by f.test_name)"
meanMultiplierOverallQuery="select round(avg(multipliers), $precision) as overall_mean_multiplier from (select (round(avg(t.latency_percentile) / (avg(f.latency_percentile) + .000001), $precision)) as multipliers from from_results as f join to_results as t on f.test_name = t.test_name where f.test_name != 'bulk_insert' group by f.test_name)"
tpccLatencyQuery="select f.test_name as test_name, f.server_name, f.server_version, avg(f.latency_percentile) as from_latency_median, t.server_name, t.server_version, avg(t.latency_percentile) as to_latency_median, ROUND(avg(t.latency_percentile) / (avg(f.latency_percentile) + .000001), $precision) as multiplier from from_results as f join to_results as t on f.test_name = t.test_name where f.test_name LIKE '$tpccRegex' group by f.test_name;"
tpccTpsQuery="select f.test_name as test_name, f.server_name, f.server_version, avg(f.sql_transactions_per_second) as tps, t.test_name as test_name, t.server_name, t.server_version, avg(t.sql_transactions_per_second) as tps from from_results as f join to_results as t on f.test_name = t.test_name where f.test_name LIKE 'tpcc%' group by f.test_name;"
echo '
{
"apiVersion": "batch/v1",
"kind": "Job",
"metadata": {
"name": "'$jobname'",
"namespace": "performance-benchmarking"
},
"spec": {
"backoffLimit": 3,
"template": {
"metadata": {
"annotations": {
"alert_recipients": "'$ACTOR_EMAIL'"
},
"labels": {
"app": "performance-benchmarking",
"k8s-liquidata-inc-monitored-job": "created-by-static-config"
}
},
"spec": {
"serviceAccountName": "performance-benchmarking",
"containers": [
{
"name": "performance-benchmarking",
"image": "407903926827.dkr.ecr.us-west-2.amazonaws.com/liquidata/performance-benchmarking:latest",
"resources": {
"limits": {
"cpu": "7000m"
}
},
"env": [
{ "name": "GOMAXPROCS", "value": "7" },
{ "name": "ACTOR", "value": "'$ACTOR'" },
{ "name": "ACTOR_EMAIL", "value": "'$ACTOR_EMAIL'" },
{ "name": "REPO_ACCESS_TOKEN", "value": "'$REPO_ACCESS_TOKEN'" }
],
"imagePullPolicy": "Always",
"args": [
"--schema=/app/schema.sql",
"--useDoltHubLuaScriptsRepo",
"--repo=doltgresql",
"--output='$format'",
"--email-template=DoltgresPerformanceBenchmarkingReleaseTemplate",
"--postgres-exec=/usr/lib/postgresql/15/bin/postgres",
"--init-db-exec=/usr/lib/postgresql/15/bin/initdb",
"--from-server='$fromServer'",
"--from-version='$fromVersion'",
"--to-server='$toServer'",
"--to-version='$toVersion'",
"--bucket=performance-benchmarking-github-actions-results",
"--region=us-west-2",
"--results-dir='$timeprefix'",
"--results-prefix='$actorprefix'",
'"$withTpcc"'
'"$initBigRepo"'
'"$nomsBinFormat"'
"--sysbenchQueries='"$medianLatencyMultiplierReadsQuery"'",
"--sysbenchQueries='"$meanMultiplierReadsQuery"'",
"--sysbenchQueries='"$medianLatencyMultiplierWritesQuery"'",
"--sysbenchQueries='"$meanMultiplierWritesQuery"'",
"--sysbenchQueries='"$meanMultiplierOverallQuery"'",
"--tpccQueries='"$tpccLatencyQuery"'",
"--tpccQueries='"$tpccTpsQuery"'"
]
}
],
"restartPolicy": "Never",
"nodeSelector": {
"performance-benchmarking-worker": "true"
},
"tolerations": [
{
"effect": "NoSchedule",
"key": "dedicated",
"operator": "Equal",
"value": "performance-benchmarking-worker"
}
]
}
}
}
}
'
+97
View File
@@ -0,0 +1,97 @@
#!/bin/bash
set -e
if [ -z "$KUBECONFIG" ]; then
echo "Must set KUBECONFIG"
exit 1
fi
if [ -z "$TEMPLATE_SCRIPT" ]; then
echo "Must set TEMPLATE_SCRIPT"
exit 1
fi
if [ -z "$NOMS_BIN_FORMAT" ]; then
echo "Must set NOMS_BIN_FORMAT"
exit 1
fi
if [ -z "$FROM_SERVER" ] || [ -z "$FROM_VERSION" ] || [ -z "$TO_SERVER" ] || [ -z "$TO_VERSION" ]; then
echo "Must set FROM_SERVER FROM_VERSION TO_SERVER and TO_VERSION"
exit 1
fi
if [ -z "$ACTOR" ]; then
echo "Must set ACTOR"
exit 1
fi
if [ -z "$MODE" ]; then
echo "Must set MODE"
exit 1
fi
if [ "$NOMS_BIN_FORMAT" = "__DOLT__" ]; then
INIT_BIG_REPO="false"
fi
echo "Setting from $FROM_SERVER: $FROM_VERSION"
echo "Setting to $TO_SERVER: $TO_VERSION"
# use first 8 characters of TO_VERSION to differentiate
# jobs
short=${TO_VERSION:0:8}
lowered=$(echo "$ACTOR" | tr '[:upper:]' '[:lower:]')
actorShort="$lowered-$short"
# random sleep
sleep 0.$[ ( $RANDOM % 10 ) + 1 ]s
timesuffix=`date +%s%N`
jobname="$actorShort"
if [ -n "$WITH_TPCC" ]; then
jobname="$jobname-tpcc"
fi
jobname="$jobname-$timesuffix"
timeprefix=$(date +%Y/%m/%d)
actorprefix="$MODE/$lowered/$jobname/$NOMS_BIN_FORMAT"
format="markdown"
if [[ "$MODE" = "release" || "$MODE" = "nightly" ]]; then
format="html"
fi
# set value to ISSUE_NUMBER environment variable
# or default to -1
issuenumber=${ISSUE_NUMBER:-"-1"}
source \
"$TEMPLATE_SCRIPT" \
"$jobname" \
"$FROM_SERVER" \
"$FROM_VERSION" \
"$TO_SERVER" \
"$TO_VERSION" \
"$timeprefix" \
"$actorprefix" \
"$format" \
"$issuenumber" \
"$INIT_BIG_REPO" \
"$NOMS_BIN_FORMAT" \
"$WITH_TPCC" > job.json
out=$(KUBECONFIG="$KUBECONFIG" kubectl apply -f job.json || true)
if [ "$out" != "job.batch/$jobname created" ]; then
echo "something went wrong creating job... this job likely already exists in the cluster"
echo "$out"
exit 1
else
echo "$out"
fi
exit 0
@@ -0,0 +1,16 @@
#!/bin/bash
set -e
if [[ $# -ne 1 ]]; then
echo "Usage: ./validate-commentor.sh COMMENTOR"
exit 1
fi
validcommentors="coffeegoddd bheni jennifersp Hydrocharged reltuk tbantle22 timsehn zachmu max-hoffman nicktobey fulghum"
contains() {
[[ $1 =~ (^|[[:space:]])$2($|[[:space:]]) ]] && echo "valid=true" >> $GITHUB_OUTPUT || exit 0
}
contains "$validcommentors" "$1"
@@ -0,0 +1,114 @@
#!/bin/sh
set -e
if [ "$#" -lt 5 ]; then
echo "Usage: ./get-job-json.sh <jobname> <version> <timeprefix> <actorprefix> <format> <nomsBinFormat> <issueNumber> <regressComp> <branchRef>"
exit 1
fi
jobname="$1"
version="$2"
timeprefix="$3"
actorprefix="$4"
format="$5"
nomsBinFormat="$6"
issueNumber="$7"
regressComp="$8"
branchRef="$9"
precision="6"
if [ -n "$nomsBinFormat" ]; then
nomsBinFormat="\"--noms-bin-format=$nomsBinFormat\","
fi
if [ -n "$issueNumber" ]; then
issueNumber="\"--issue-number=$issueNumber\","
fi
regressPrec=""
if [ -n "$regressComp" ]; then
regressComp="\"--regress-compare=$regressComp\","
regressPrec="\"--regress-precision=$precision\","
branchRef="\"--branch=$branchRef\","
fi
resultCountQuery="select version, result, count(*) as total from results where result != 'skipped' group by result;"
testCountQuery="select version, count(*) as total_tests from results where result != 'skipped';"
correctnessQuery="select ROUND(100.0 * (cast(ok_results.total as decimal) / (cast(all_results.total as decimal) + .000001)), $precision) as correctness_percentage from (select count(*) as total from results where result = 'ok') as ok_results join (select count(*) as total from results where result != 'skipped') as all_results"
echo '
{
"apiVersion": "batch/v1",
"kind": "Job",
"metadata": {
"name": "'$jobname'",
"namespace": "sql-correctness"
},
"spec": {
"backoffLimit": 3,
"template": {
"metadata": {
"annotations": {
"alert_recipients": "'$ACTOR_EMAIL'"
},
"labels": {
"k8s-liquidata-inc-monitored-job": "created-by-static-config"
}
},
"spec": {
"serviceAccountName": "sql-correctness",
"containers": [
{
"name": "sql-correctness",
"image": "407903926827.dkr.ecr.us-west-2.amazonaws.com/liquidata/sql-correctness:latest",
"resources": {
"limits": {
"cpu": "7000m"
}
},
"env": [
{ "name": "REPO_ACCESS_TOKEN", "value": "'$REPO_ACCESS_TOKEN'"},
{ "name": "ACTOR", "value": "'$ACTOR'"},
{ "name": "ACTOR_EMAIL", "value": "'$ACTOR_EMAIL'"},
{ "name": "DOLT_DEFAULT_BIN_FORMAT", "value": "'$NOMS_BIN_FORMAT'"}
],
"args": [
"--schema=/app/correctness.sql",
"--output='$format'",
"--version='$version'",
"--doltgres",
"--email-template=DoltgresSqlCorrectnessReleaseTemplate",
'"$nomsBinFormat"'
'"$issueNumber"'
'"$regressComp"'
'"$regressPrec"'
'"$branchRef"'
"--bucket=sql-correctness-github-actions-results",
"--region=us-west-2",
"--results-dir='$timeprefix'",
"--results-prefix='$actorprefix'",
"'"$resultCountQuery"'",
"'"$testCountQuery"'",
"'"$correctnessQuery"'"
]
}
],
"restartPolicy": "Never",
"nodeSelector": {
"sql-correctness-worker": "true"
},
"tolerations": [
{
"effect": "NoSchedule",
"key": "dedicated",
"operator": "Equal",
"value": "sql-correctness-worker"
}
]
}
}
}
}
'
+106
View File
@@ -0,0 +1,106 @@
#!/bin/bash
set -e
if [ -z "$KUBECONFIG" ]; then
echo "Must set KUBECONFIG"
exit 1
fi
if [ -z "$TEMPLATE_SCRIPT" ]; then
echo "Must set TEMPLATE_SCRIPT"
exit 1
fi
if [ -z "$NOMS_BIN_FORMAT" ]; then
echo "Must set NOMS_BIN_FORMAT"
exit 1
fi
if [ -z "$VERSION" ]; then
echo "Must set VERSION for correctness run"
exit 1
fi
if [ -z "$ACTOR" ]; then
echo "Must set ACTOR"
exit 1
fi
if [ -z "$MODE" ]; then
echo "Must set MODE"
exit 1
fi
if [ -n "$PR_NUMBER" ]; then
if [ -z "$REGRESS_COMP" ]; then
echo "Must set REGRESS_COMP for PR correctness comparisons"
exit 1
fi
if [ -z "$PR_BRANCH_REF" ]; then
echo "Must set PR_BRANCH_REF for PR correctness comparisons"
exit 1
fi
fi
# use first 8 characters of VERSION to differentiate
# jobs
short=${VERSION:0:8}
lowered=$(echo "$ACTOR" | tr '[:upper:]' '[:lower:]')
actorShort="$lowered-$short"
# random sleep
sleep 0.$[ ( $RANDOM % 10 ) + 1 ]s
timesuffix=`date +%s%N`
jobname=""
if [ -n "$PR_NUMBER" ]; then
jobname="$lowered-$PR_NUMBER"
else
jobname="$actorShort-$timesuffix"
fi
timeprefix=$(date +%Y/%m/%d)
actorprefix="$MODE/$lowered/$jobname/$NOMS_BIN_FORMAT"
format="markdown"
if [[ "$MODE" = "release" || "$MODE" = "nightly" ]]; then
format="html"
fi
# set value to PR_NUMBER environment variable
# or default to -1
issuenumber=${PR_NUMBER:-"-1"}
source \
"$TEMPLATE_SCRIPT" \
"$jobname" \
"$VERSION" \
"$timeprefix" \
"$actorprefix" \
"$format" \
"$NOMS_BIN_FORMAT" \
"$issuenumber" \
"$REGRESS_COMP" \
"$PR_BRANCH_REF" > job.json
# delete existing job with same name if this is a pr job
if [ -n "$PR_NUMBER" ]; then
out=$(KUBECONFIG="$KUBECONFIG" kubectl delete job/"$jobname" -n sql-correctness || true)
echo "Delete pr job if exists: $out"
sleep 45
fi
out=$(KUBECONFIG="$KUBECONFIG" kubectl apply -f job.json || true)
if [ "$out" != "job.batch/$jobname created" ]; then
echo "something went wrong creating job... this job likely already exists in the cluster"
echo "$out"
exit 1
else
echo "$out"
fi
exit 0
+65
View File
@@ -0,0 +1,65 @@
#!/bin/bash
set -ex
set -o pipefail
root="`dirname \"$0\"`"
dir="`realpath \"$root\"`"
doltgres_root="`realpath \"$dir/../../\"`"
version=""
sed_cmd_i=""
start_marker=""
end_marker=""
dest_file="$doltgres_root/README.md"
os_type="darwin"
start_template='<!-- START_%s_RESULTS_TABLE -->'
end_template='<!-- END_%s_RESULTS_TABLE -->'
if [ "$#" -ne 3 ]; then
echo "usage: update-perf.sh <version> <perf-path> <correctness-path>"
exit 1;
fi
version="$1"
new_perf="$2"
new_correctness="$3"
if [[ "$OSTYPE" == "linux-gnu"* ]]; then
os_type="linux"
fi
# update the version
if [ "$os_type" == "linux" ]
then
sed -i 's/Here are the benchmarks for DoltgreSQL version `'".*"'`/Here are the benchmarks for DoltgreSQL version `'"$version"'`/' "$dest_file"
sed -i 's/Here are DoltgreSQL'"'"'s sqllogictest results for version `'".*"'`./Here are DoltgreSQL'"'"'s sqllogictest results for version `'"$version"'`./' "$dest_file"
else
sed -i '' "s/Here are the benchmarks for DoltgreSQL version \\\`.*\\\`/Here are the benchmarks for DoltgreSQL version \\\`$version\\\`/" "$dest_file"
sed -i '' 's/Here are DoltgreSQL'"'"'s sqllogictest results for version `'".*"'`./Here are DoltgreSQL'"'"'s sqllogictest results for version `'"$version"'`./' "$dest_file"
fi
perf_start_marker=$(printf "$start_template" "LATENCY")
perf_end_marker=$(printf "$end_template" "LATENCY")
correctness_start_marker=$(printf "$start_template" "CORRECTNESS")
correctness_end_marker=$(printf "$end_template" "CORRECTNESS")
# store in variable
updated_perf_with_markers=$(printf "$perf_start_marker\n$(cat $new_perf)\n$perf_end_marker\n")
updated_correctness_with_markers=$(printf "$correctness_start_marker\n$(cat $new_correctness)\n$correctness_end_marker\n")
echo "$updated_perf_with_markers" > "$new_perf"
echo "$updated_correctness_with_markers" > "$new_correctness"
if [ "$os_type" == "linux" ]
then
sed -i '/<!-- END_LATENCY/r '"$new_perf"'' -e '/<!-- START_LATENCY/,/<!-- END_LATENCY/d' "$dest_file"
sed -i '/<!-- END_CORRECTNESS/r '"$new_correctness"'' -e '/<!-- START_CORRECTNESS/,/<!-- END_CORRECTNESS/d' "$dest_file"
else
sed -i '/\<!-- END_LATENCY/r '"$new_perf"'' -e '/\<!-- START_LATENCY/,/\<!-- END_LATENCY/d' "$dest_file"
sed -i '/\<!-- END_CORRECTNESS/r '"$new_correctness"'' -e '/\<!-- START_CORRECTNESS/,/\<!-- END_CORRECTNESS/d' "$dest_file"
fi
+210
View File
@@ -0,0 +1,210 @@
name: Bump Deps
on:
repository_dispatch:
types: [ bump-dependency ]
jobs:
get-label:
name: Get Label
outputs:
label: ${{ steps.get-label.outputs.label }}
runs-on: ubuntu-22.04
steps:
- name: Get Label
id: get-label
env:
REPO: ${{ github.event.client_payload.dependency }}
run: |
if [ "$REPO" == "dolt" ]
then
echo "label=dolt-bump" >> $GITHUB_OUTPUT
else
echo "$REPO is unsupported"
exit 1
fi
stale-bump-prs:
name: Retrieving Stale Bump PRs
needs: get-label
outputs:
stale-pulls: ${{ steps.get-stale-prs.outputs.open-pulls }}
runs-on: ubuntu-22.04
steps:
- name: Get Open Bump PRs
id: get-stale-prs
uses: actions/github-script@v7
env:
LABEL: ${{ needs.get-label.outputs.label }}
with:
debug: true
github-token: ${{ secrets.GITHUB_TOKEN }}
script: |
try {
const { LABEL } = process.env;
const { owner, repo } = context.repo;
const res = await github.rest.pulls.list({
owner,
repo,
state: 'open',
sort: 'created',
direction: 'desc',
});
const { data } = res;
const reduced = data.reduce((acc, p) => {
if (p.labels.length < 1) return acc;
let keepAlive = false;
let shouldPush = false;
for (const label of p.labels) {
if (label.name === LABEL) {
shouldPush = true;
}
if (label.name === "keep-alive") {
keepAlive = true;
}
}
if (shouldPush) {
acc.push({
number: p.number,
keepAlive,
headRef: p.head.ref,
});
}
return acc;
}, []);
console.log(reduced);
if (reduced.length > 0) core.setOutput("open-pulls", JSON.stringify(reduced));
process.exit(0);
} catch(err) {
console.log("Error:", err);
process.exit(1);
}
open-bump-pr:
needs: [get-label, stale-bump-prs]
name: Open Bump PR
runs-on: ubuntu-22.04
outputs:
latest-pr: ${{ steps.latest-pr.outputs.pr_url }}
steps:
- uses: actions/checkout@v6
with:
token: ${{ secrets.GITHUB_TOKEN }}
- name: Set up Go 1.x
uses: actions/setup-go@v5
with:
go-version-file: go.mod
- name: Bump dependency
run: |
go get github.com/dolthub/${{ github.event.client_payload.dependency }}/go@${{ github.event.client_payload.head_commit_sha }}
go mod tidy
- name: Get short hash
id: short-sha
run: |
commit=${{ github.event.client_payload.head_commit_sha }}
short=${commit:0:8}
echo "short=$short" >> $GITHUB_OUTPUT
- name: Get Assignee and Reviewer
id: get_reviewer
run: |
if [ "${{ github.event.client_payload.assignee }}" == "github-actions[bot]" ]
then
echo "assignee=zachmu" >> $GITHUB_OUTPUT
else
assignee="${{ github.event.client_payload.assignee }}"
echo "assignee=$assignee" >> $GITHUB_OUTPUT
fi
if [ "${{ github.event.client_payload.assignee }}" == "zachmu" ]
then
echo "reviewer=Hydrocharged" >> $GITHUB_OUTPUT
else
echo "reviewer=zachmu" >> $GITHUB_OUTPUT
fi
- name: Create and Push new branch
run: |
git config --global --add user.name "${{ steps.get_reviewer.outputs.assignee }}"
git config --global --add user.email "${{ github.event.client_payload.assignee_email }}"
branchname=${{ format('{0}-{1}', steps.get_reviewer.outputs.assignee, steps.short-sha.outputs.short) }}
git checkout -b "$branchname"
git add .
git commit -m "${{ format('[ga-bump-dep] Bump dependency in Doltgres by {0}', steps.get_reviewer.outputs.assignee) }}"
git push origin "$branchname"
- name: pull-request
uses: repo-sync/pull-request@v2
id: latest-pr
with:
source_branch: ${{ format('{0}-{1}', steps.get_reviewer.outputs.assignee, steps.short-sha.outputs.short ) }}
destination_branch: "main"
github_token: ${{ secrets.PUBLIC_REPO_ACCESS_TOKEN || secrets.REPO_ACCESS_TOKEN || secrets.GITHUB_TOKEN }}
pr_title: "[auto-bump] [no-release-notes] dependency by ${{ steps.get_reviewer.outputs.assignee }}"
pr_template: ".github/markdown-templates/dep-bump.md"
pr_reviewer: ${{ steps.get_reviewer.outputs.reviewer }}
pr_assignee: ${{ github.event.client_payload.assignee }}
pr_label: ${{ needs.get-label.outputs.label }}
comment-on-stale-prs:
needs: [open-bump-pr, stale-bump-prs]
if: ${{ needs.stale-bump-prs.outputs.stale-pulls != '' }}
runs-on: ubuntu-22.04
strategy:
matrix:
pull: ${{ fromJson(needs.stale-bump-prs.outputs.stale-pulls) }}
steps:
- name: Comment/Close Stale PRs
id: get-stale-prs
uses: actions/github-script@v7
env:
PULL: ${{ toJson(matrix.pull) }}
SUPERSEDED_BY: ${{ needs.open-bump-pr.outputs.latest-pr }}
with:
debug: true
github-token: ${{ secrets.GITHUB_TOKEN }}
script: |
try {
const { owner, repo } = context.repo;
const { PULL, SUPERSEDED_BY } = process.env;
const pull = JSON.parse(PULL);
if (pull.keepAlive) process.exit(0);
const checkSuiteRes = await github.rest.checks.listSuitesForRef({
owner,
repo,
ref: pull.headRef,
});
if (checkSuiteRes.data) {
for (const suite of checkSuiteRes.data.check_suites) {
console.log("suite id:", suite.id);
console.log("suite app slug:", suite.app.slug);
console.log("suite status:", suite.status);
console.log("suite conclusion:", suite.conclusion);
}
}
console.log(`Closing open pr ${pull.number}`);
await github.rest.issues.createComment({
issue_number: pull.number,
owner,
repo,
body: `This PR has been superseded by ${SUPERSEDED_BY}`
});
await github.rest.pulls.update({
owner,
repo,
pull_number: pull.number,
state: 'closed',
});
process.exit(0);
} catch(err) {
console.log("Error:", err);
process.exit(1);
}
@@ -0,0 +1,69 @@
name: Create Release Notes
on:
workflow_dispatch:
inputs:
version:
description: 'SemVer format release tag, i.e. 0.2.4'
required: true
repository_dispatch:
types: [ release-notes ]
jobs:
create-release-notes:
runs-on: ubuntu-22.04
steps:
- name: Get Vars
id: get_vars
run: |
if [ "$EVENT_NAME" == "workflow_dispatch" ]
then
release_id=$(curl -H "Accept: application/vnd.github.v3+json" https://api.github.com/repos/dolthub/doltgresql/releases/tags/v${{ github.event.inputs.version }} | jq '.id')
echo "version=${{ github.event.inputs.version }}" >> $GITHUB_OUTPUT
echo "release_id=$release_id" >> $GITHUB_OUTPUT
else
echo "version=${{ github.event.client_payload.version }}" >> $GITHUB_OUTPUT
echo "release_id=${{ github.event.client_payload.release_id }}" >> $GITHUB_OUTPUT
fi
env:
EVENT_NAME: ${{ github.event_name }}
- name: Checkout Release Notes Generator
uses: actions/checkout@v6
with:
repository: dolthub/release-notes-generator
token: ${{ secrets.GITHUB_TOKEN }}
- name: Install Dependencies
run: sudo ./install-deps.sh
env:
PERL_MM_USE_DEFAULT: 1
- name: Create Notes
run: |
git clone https://github.com/dolthub/doltgresql.git
./gen_release_notes.pl \
--token "$TOKEN" dolthub/doltgresql v${{ steps.get_vars.outputs.version }} > changelog.txt
env:
TOKEN: ${{ secrets.GITHUB_TOKEN }}
- name: Post Changelog to Release
uses: actions/github-script@v7
with:
debug: true
github-token: ${{ secrets.GITHUB_TOKEN }}
script: |
const fs = require('fs');
const path = require('path')
try {
const body = fs.readFileSync(path.join(process.env.WORKSPACE, "changelog.txt"), { encoding: "utf8" })
const res = await github.rest.repos.updateRelease({
owner: "dolthub",
repo: "doltgresql",
release_id: parseInt(process.env.RELEASE_ID, 10),
body,
});
console.log("Successfully updated release notes", res)
} catch (err) {
console.log("Error", err);
process.exit(1);
}
env:
WORKSPACE: ${{ github.workspace }}
RELEASE_ID: ${{ steps.get_vars.outputs.release_id }}
@@ -0,0 +1,77 @@
name: Push Docker Image to DockerHub
on:
workflow_dispatch:
inputs:
version:
description: 'SemVer format release tag, i.e. 0.24.5'
required: true
repository_dispatch:
types: [ push-docker-image ]
jobs:
get-release-id:
name: Get Doltgres Release Id
runs-on: ubuntu-22.04
outputs:
release_id: ${{ steps.get_release.outputs.release_id }}
steps:
- name: Get Release
id: get_release
run: |
release_id="$RELEASE_ID"
if [ "$EVENT_TYPE" == "workflow_dispatch" ]; then
release_id=$(curl -H "Accept: application/vnd.github.v3+json" https://api.github.com/repos/dolthub/doltgresql/releases/tags/v${{ github.event.inputs.version }} | jq '.id')
fi
echo "release_id=$release_id" >> $GITHUB_OUTPUT
env:
EVENT_TYPE: ${{ github.event_name }}
RELEASE_ID: ${{ github.event.client_payload.release_id }}
docker-image-push:
name: Push Docker Image
needs: get-release-id
runs-on: ubuntu-22.04
steps:
- name: Checkout
uses: actions/checkout@v6
- name: Login to Docker Hub
uses: docker/login-action@v3
with:
username: ${{ secrets.DOCKER_HUB_USERNAME }}
password: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }}
- name: Set up QEMU
uses: docker/setup-qemu-action@v3
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v3
with:
platforms: linux/amd64,linux/arm64
- name: Build and push doltgres image
uses: docker/build-push-action@v5
with:
platforms: linux/amd64,linux/arm64
context: .
file: ./Dockerfile
push: true
tags: dolthub/doltgresql:${{ github.event.inputs.version || github.event.client_payload.version }} , dolthub/doltgresql:latest
build-args: |
DOLTGRES_VERSION=${{ github.event.inputs.version || github.event.client_payload.version }}
- name: Update Docker Hub Readme for doltgres image
uses: peter-evans/dockerhub-description@v4
with:
username: ${{ secrets.DOCKER_HUB_USERNAME }}
password: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }}
repository: dolthub/doltgresql
readme-filepath: ./dockerREADME.md
- run: |
gh api \
--method PATCH \
-H "Accept: application/vnd.github+json" \
-H "X-GitHub-Api-Version: 2022-11-28" \
/repos/$REPO_OWNER/$REPO_NAME/releases/$RELEASE_ID \
-F "draft=false" -F "prerelease=false" -F "make_latest=true"
env:
GH_TOKEN: ${{ secrets.REPO_ACCESS_TOKEN || secrets.GITHUB_TOKEN }}
REPO_OWNER: dolthub
REPO_NAME: doltgresql
RELEASE_ID: ${{ needs.get-release-id.outputs.release_id }}
+238
View File
@@ -0,0 +1,238 @@
name: Release DoltgreSQL
on:
workflow_dispatch:
inputs:
version:
description: 'SemVer format release tag, i.e. 0.2.4'
required: true
jobs:
format-version:
runs-on: ubuntu-22.04
outputs:
version: ${{ steps.format_version.outputs.version }}
steps:
- name: Format Input
id: format_version
run: |
version="${{ github.event.inputs.version }}"
if [[ $version == v* ]];
then
version="${version:1}"
fi
echo "version=$version" >> $GITHUB_OUTPUT
windows-extension-support:
runs-on: windows-latest
steps:
- name: Checkout code
uses: actions/checkout@v6
with:
token: ${{ secrets.PUBLIC_REPO_ACCESS_TOKEN || secrets.REPO_ACCESS_TOKEN || secrets.GITHUB_TOKEN }}
- name: Build SQL Syntax
run: ./build.sh
working-directory: ./postgres/parser
shell: bash
- name: Upload Extension Artifacts
uses: actions/upload-artifact@v4
with:
name: windows-extension-artifacts
path: |
./core/extensions/pg_extension/output/pg_extension.dll
./core/extensions/pg_extension/output/postgres.exe
if-no-files-found: error
retention-days: 1
create-release:
needs: [format-version, windows-extension-support]
name: Create release
runs-on: ubuntu-22.04
outputs:
release_id: ${{ steps.create_release.outputs.id }}
steps:
- name: Checkout code
uses: actions/checkout@v6
with:
token: ${{ secrets.PUBLIC_REPO_ACCESS_TOKEN || secrets.REPO_ACCESS_TOKEN || secrets.GITHUB_TOKEN }}
- name: Build SQL Syntax
run: ./build.sh
working-directory: ./postgres/parser
shell: bash
- name: Update Doltgres version command
run: sed -i -e 's/Version\s*=\s*".*"/Version = "'"$NEW_VERSION"'"/' "$FILE"
env:
FILE: ${{ format('{0}/server/server.go', github.workspace) }}
NEW_VERSION: ${{ needs.format-version.outputs.version }}
- name: Set minver TBD to version
run: sed -i -e 's/minver:"TBD"/minver:"'"$NEW_VERSION"'"/' "$FILE"
env:
FILE: ${{ format('{0}/servercfg/config.go', github.workspace) }}
NEW_VERSION: ${{ needs.format-version.outputs.version }}
- name: update minver_validation.txt
working-directory: ./
run: go run -mod=readonly ./utils/genminver_validation/ $FILE
env:
FILE: ${{ format('{0}/servercfg/cfgdetails/testdata/minver_validation.txt', github.workspace) }}
- uses: EndBug/add-and-commit@v9.1.1
with:
message: ${{ format('Update DoltgreSQL version to {0}', needs.format-version.outputs.version) }}
add: ${{ format('["{0}/server/server.go", "{0}/servercfg/config.go", "{0}/servercfg/cfgdetails/testdata/minver_validation.txt"]', github.workspace) }}
cwd: "."
new_branch: cd-release
push: 'origin cd-release --set-upstream --force'
- name: Create Pull Request
run: gh pr create --base main --head "cd-release" --title "[no-release-notes] Release v${{ needs.format-version.outputs.version }}" --body "Created by the Release workflow to update DoltgreSQL's version"
env:
GH_TOKEN: ${{ secrets.PUBLIC_REPO_ACCESS_TOKEN || secrets.REPO_ACCESS_TOKEN || secrets.GITHUB_TOKEN }}
- name: Enable Pull Request Auto-Merge
run: gh pr merge --merge --auto "cd-release"
env:
GH_TOKEN: ${{ secrets.PUBLIC_REPO_ACCESS_TOKEN || secrets.REPO_ACCESS_TOKEN || secrets.GITHUB_TOKEN }}
- name: Download Extension Artifacts
uses: actions/download-artifact@v4
with:
name: windows-extension-artifacts
path: ./core/extensions/pg_extension/output
- name: Install Go
uses: actions/setup-go@v5
with:
go-version-file: go.mod
- name: Free up disk space
shell: bash
run: |
sudo find /opt -maxdepth 1 -mindepth 1 \
! -path /opt/containerd \
! -path /opt/actionarchivecache \
! -path /opt/runner \
! -path /opt/runner-cache \
-exec rm -rf {} \;
- name: Build Binaries
id: build_binaries
shell: bash -ex {0}
run: |
latest=$(git rev-parse HEAD)
echo "commitish=$latest" >> $GITHUB_OUTPUT
GO_BUILD_VERSION=1.26 scripts/build_all_binaries.sh
- name: Fix ownership of build artifacts
run: sudo chown -R "$USER:$USER" out
- name: Create Release
id: create_release
uses: dolthub/create-release@v2
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
with:
tag_name: v${{ needs.format-version.outputs.version }}
release_name: ${{ needs.format-version.outputs.version }}
draft: false
prerelease: true
commitish: ${{ steps.build_binaries.outputs.commitish }}
- name: Create install script
id: create-install-script
shell: bash
run: scripts/build_install_script.sh
env:
DOLTGRES_VERSION: ${{ needs.format-version.outputs.version }}
- name: Upload Linux AMD64 Distro
id: upload-linux-amd64-distro
uses: dolthub/upload-release-asset@v2
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
with:
upload_url: ${{ steps.create_release.outputs.upload_url }}
asset_path: out/doltgresql-linux-amd64.tar.gz
asset_name: doltgresql-linux-amd64.tar.gz
asset_content_type: application/zip
- name: Upload Linux ARM64 Distro
id: upload-linux-arm64-distro
uses: dolthub/upload-release-asset@v2
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
with:
upload_url: ${{ steps.create_release.outputs.upload_url }}
asset_path: out/doltgresql-linux-arm64.tar.gz
asset_name: doltgresql-linux-arm64.tar.gz
asset_content_type: application/zip
- name: Upload OSX AMD64 Distro
id: upload-osx-amd64-distro
uses: dolthub/upload-release-asset@v2
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
with:
upload_url: ${{ steps.create_release.outputs.upload_url }}
asset_path: out/doltgresql-darwin-amd64.tar.gz
asset_name: doltgresql-darwin-amd64.tar.gz
asset_content_type: application/zip
- name: Upload OSX ARM64 Distro
id: upload-osx-arm64-distro
uses: dolthub/upload-release-asset@v2
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
with:
upload_url: ${{ steps.create_release.outputs.upload_url }}
asset_path: out/doltgresql-darwin-arm64.tar.gz
asset_name: doltgresql-darwin-arm64.tar.gz
asset_content_type: application/zip
- name: Upload Windows Distro
id: upload-windows-distro
uses: dolthub/upload-release-asset@v2
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
with:
upload_url: ${{ steps.create_release.outputs.upload_url }}
asset_path: out/doltgresql-windows-amd64.zip
asset_name: doltgresql-windows-amd64.zip
asset_content_type: application/zip
- name: Upload Windows Distro 7z
id: upload-windows-distro-7z
uses: dolthub/upload-release-asset@v2
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
with:
upload_url: ${{ steps.create_release.outputs.upload_url }}
asset_path: out/doltgresql-windows-amd64.7z
asset_name: doltgresql-windows-amd64.7z
asset_content_type: application/x-7z-compressed
- name: Upload install script
id: upload-install-script
uses: dolthub/upload-release-asset@v2
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
with:
upload_url: ${{ steps.create_release.outputs.upload_url }}
asset_path: out/install.sh
asset_name: install.sh
asset_content_type: text/plain
create-release-notes:
needs: [format-version, create-release]
runs-on: ubuntu-22.04
steps:
- name: Trigger Release Notes
uses: peter-evans/repository-dispatch@v2.0.0
with:
token: ${{ secrets.GITHUB_TOKEN }}
event-type: release-notes
client-payload: '{"version": "${{ needs.format-version.outputs.version }}", "release_id": "${{ needs.create-release.outputs.release_id }}"}'
trigger-performance-benchmark-email:
needs: [format-version, create-release]
runs-on: ubuntu-22.04
steps:
- name: Trigger Performance Benchmarks
uses: peter-evans/repository-dispatch@v2.0.0
with:
token: ${{ secrets.REPO_ACCESS_TOKEN }}
event-type: release-doltgres
client-payload: '{"version": "${{ needs.format-version.outputs.version }}", "actor": "${{ github.actor }}"}'
docker-image-push:
needs: [format-version, create-release]
runs-on: ubuntu-22.04
steps:
- name: Trigger Push Docker Image
uses: peter-evans/repository-dispatch@v3
with:
token: ${{ secrets.REPO_ACCESS_TOKEN }}
event-type: push-docker-image
client-payload: '{"version": "${{ needs.format-version.outputs.version }}", "release_id": "${{ needs.create-release.outputs.release_id }}" }'
+76
View File
@@ -0,0 +1,76 @@
name: Test Bats Unix
on: [pull_request]
concurrency:
group: ci-bats-unix-${{ github.event.pull_request.number || github.ref }}
cancel-in-progress: true
jobs:
test:
name: Bats tests
defaults:
run:
shell: bash
runs-on: ${{ matrix.os }}
strategy:
fail-fast: false
matrix:
os: [ ubuntu-22.04, macos-latest ]
env:
use_credentials: ${{ secrets.AWS_SECRET_ACCESS_KEY != '' && secrets.AWS_ACCESS_KEY_ID != '' }}
steps:
- uses: actions/checkout@v6
- name: Set up Go
uses: actions/setup-go@v5
with:
go-version-file: go.mod
id: go
- uses: actions/setup-node@v4
with:
node-version: ^16
- name: Create CI Bin
run: |
mkdir -p ./.ci_bin
echo "$(pwd)/.ci_bin" >> $GITHUB_PATH
- name: Install ICU4C (MacOS)
if: ${{ matrix.os == 'macos-latest' }}
run: |
dir=$(brew --cellar icu4c)
dir="$dir"/$(ls "$dir")
echo CGO_CPPFLAGS=-I$dir/include >> $GITHUB_ENV
echo CGO_LDFLAGS=-L$dir/lib >> $GITHUB_ENV
- name: Verify Bats Naming Scheme
run: ./check_bats_fmt.sh
working-directory: ./testing/bats/setup
- name: Build SQL Syntax
run: ./build.sh
working-directory: ./postgres/parser
shell: bash
- name: Install Bats
run: |
npm i bats
echo "$(pwd)/node_modules/.bin" >> $GITHUB_PATH
working-directory: ./.ci_bin
- name: Install DoltgreSQL
working-directory: ./
run: |
go build -mod=readonly -o .ci_bin/doltgres ./cmd/doltgres
- name: Install PSQL Ubuntu
if: matrix.os == 'ubuntu-22.04'
run: |
sudo sh -c 'echo "deb https://apt.postgresql.org/pub/repos/apt $(lsb_release -cs)-pgdg main" > /etc/apt/sources.list.d/pgdg.list'
wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | sudo apt-key add -
sudo apt-get update
sudo apt-get install --yes --no-install-recommends postgresql-client-15
- name: Install PSQL MacOS
if: matrix.os == 'macos-latest'
run: |
brew install postgresql@15
brew link --overwrite postgresql@15
- name: Test all Unix
env:
SQL_ENGINE: "local-engine"
BATS_TEST_RETRIES: "3"
run: |
bats --tap .
working-directory: ./testing/bats
+84
View File
@@ -0,0 +1,84 @@
name: Check Formatting, Committers and Generated Code
on:
pull_request:
branches: [ main ]
concurrency:
group: ci-check-repo-${{ github.event.pull_request.number || github.ref }}
cancel-in-progress: true
jobs:
verify:
name: Verify format
runs-on: ubuntu-22.04
outputs:
format: ${{ steps.should_format.outputs.format }}
steps:
- uses: actions/checkout@v6
with:
submodules: true
- name: Set up Go
uses: actions/setup-go@v5
with:
go-version-file: go.mod
- name: Check all
id: should_format
run: |
./scripts/check_bats_fmt.sh
if ./scripts/check_fmt.sh ; then
echo "code is formatted"
else
echo "code is not formatted"
if [ "${{ github.repository }}" != "dolthub/doltgresql" ]; then
echo "Pull requests from forks must be manually formatted."
echo "Please run scripts/format_repo.sh to format this pull request."
exit 1;
fi
echo "format=true" >> $GITHUB_OUTPUT
fi
./postgres/parser/build.sh
GOFLAGS="-mod=readonly" go build ./...
go vet -unsafeptr=false -mod=readonly ./...
env:
BRANCH_NAME: ${{ github.head_ref }}
CHANGE_TARGET: ${{ github.base_ref }}
format:
needs: verify
if: ${{ needs.verify.outputs.format == 'true' }}
name: Format PR
runs-on: ubuntu-22.04
steps:
- uses: actions/checkout@v6
with:
ref: ${{ github.event.pull_request.head.ref || github.ref }}
repository: ${{ github.event.pull_request.head.repo.full_name || github.repository }}
submodules: true
token: ${{ secrets.REPO_ACCESS_TOKEN || secrets.GITHUB_TOKEN }}
- name: Set up Go
uses: actions/setup-go@v5
with:
go-version-file: go.mod
- name: Install goimports
run: go install golang.org/x/tools/cmd/goimports@latest
- name: Format repo
run: ./scripts/format_repo.sh
env:
BRANCH_NAME: ${{ github.head_ref }}
CHANGE_TARGET: ${{ github.base_ref }}
- name: Changes detected
id: detect-changes
run: |
changes=$(git status --porcelain)
if [ ! -z "$changes" ]; then
echo "has-changes=true" >> $GITHUB_OUTPUT
fi
- uses: EndBug/add-and-commit@v9.1.4
if: ${{ steps.detect-changes.outputs.has-changes == 'true' }}
with:
message: "[ga-format-pr] Run scripts/format_repo.sh"
add: "."
cwd: "."
pull: "--ff"
@@ -0,0 +1,66 @@
name: Check Compatibility
on:
pull_request:
branches: [ main ]
paths:
- 'go/**'
- 'core/**'
- 'server/**'
- 'integration-tests/**'
concurrency:
group: ci-compatibility-tests-${{ github.event.pull_request.number || github.ref }}
cancel-in-progress: true
jobs:
test:
name: Compatibility Test
runs-on: ubuntu-22.04
steps:
- uses: actions/checkout@v4
- name: Set up Go
uses: actions/setup-go@v5
with:
go-version-file: go.mod
id: go
- uses: actions/setup-node@v4
with:
node-version: ^16
- name: Build SQL Syntax
run: ./build.sh
working-directory: ./postgres/parser
shell: bash
- name: Create CI Bin
run: |
mkdir -p ./.ci_bin
echo "$(pwd)/.ci_bin" >> $GITHUB_PATH
- name: Install Bats
run: |
npm i bats
echo "$(pwd)/node_modules/.bin" >> $GITHUB_PATH
working-directory: ./.ci_bin
- name: Install DoltgreSQL
run: |
go build -mod=readonly -o .ci_bin/doltgres ./cmd/doltgres
- name: Install PSQL
run: |
sudo sh -c 'echo "deb https://apt.postgresql.org/pub/repos/apt $(lsb_release -cs)-pgdg main" > /etc/apt/sources.list.d/pgdg.list'
wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | sudo apt-key add -
sudo apt-get update
sudo apt-get install --yes --no-install-recommends postgresql-client-15
- name: Make scripts executable
run: chmod +x runner.sh test_files/setup_repo.sh
working-directory: ./integration-tests/compatibility
- name: Run compatibility tests
run: ./runner.sh
working-directory: ./integration-tests/compatibility
@@ -0,0 +1,50 @@
name: Test Go SQL Server Driver
on: [pull_request]
concurrency:
group: ci-go-sql-server-driver-${{ github.event.pull_request.number || github.ref }}
cancel-in-progress: true
jobs:
test:
name: Go SQL Server Driver tests
defaults:
run:
shell: bash
runs-on: ${{ matrix.os }}
strategy:
fail-fast: false
matrix:
os: [ ubuntu-22.04, macos-latest ]
steps:
- uses: actions/checkout@v6
- name: Set up Go
uses: actions/setup-go@v5
with:
go-version-file: go.mod
id: go
- name: Create CI Bin
run: |
mkdir -p ./.ci_bin
echo "$(pwd)/.ci_bin" >> $GITHUB_PATH
- name: Install ICU4C (MacOS)
if: ${{ matrix.os == 'macos-latest' }}
run: |
dir=$(brew --cellar icu4c)
dir="$dir"/$(ls "$dir")
echo CGO_CPPFLAGS=-I$dir/include >> $GITHUB_ENV
echo CGO_LDFLAGS=-L$dir/lib >> $GITHUB_ENV
- name: Build SQL Syntax
run: ./build.sh
working-directory: ./postgres/parser
shell: bash
- name: Install DoltgreSQL
working-directory: ./
run: |
go build -mod=readonly -o .ci_bin/doltgres ./cmd/doltgres
echo "DOLTGRES_BIN_PATH=$(pwd)/.ci_bin/doltgres" >> $GITHUB_ENV
- name: Test Go SQL Server Driver
working-directory: ./integration-tests/go-sql-server-driver
run: |
go test --timeout=20m ./...
@@ -0,0 +1,30 @@
name: Test PostgreSQL Client integrations
on: [pull_request]
concurrency:
group: ci-postgres-client-tests-${{ github.event.pull_request.number || github.ref }}
cancel-in-progress: true
jobs:
postgres_client_integrations_job:
runs-on: ubuntu-22.04
timeout-minutes: 45
name: Run tests
steps:
- name: Free disk space
run: |
NAME="DISK-CLEANUP"
echo "[${NAME}] Starting background cleanup..."
[ -d "$AGENT_TOOLSDIRECTORY" ] && sudo rm -rf "$AGENT_TOOLSDIRECTORY" &
[ -d /usr/share/dotnet ] && sudo rm -rf /usr/share/dotnet &
[ -d /usr/local/lib/android ] && sudo rm -rf /usr/local/lib/android &
[ -d /opt/ghc ] && sudo rm -rf /opt/ghc &
[ -d /usr/local/share/boost ] && sudo rm -rf /usr/local/share/boost &
- name: Checkout
uses: actions/checkout@v6
- name: Set up Docker
uses: docker/setup-docker-action@v4
- name: Build Docker image
run: docker build -t postgres-client-tests --file testing/postgres-client-tests/Dockerfile .
- name: Run tests
run: docker run --detach=false postgres-client-tests
+26
View File
@@ -0,0 +1,26 @@
name: Static Analysis & Linter
on: [pull_request]
concurrency:
group: ci-staticcheck-${{ github.event.pull_request.number || github.ref }}
cancel-in-progress: true
jobs:
ci:
name: Run Staticcheck
runs-on: ubuntu-22.04
steps:
- uses: actions/checkout@v6
with:
submodules: true
- name: Set up Go
uses: actions/setup-go@v5
with:
go-version-file: go.mod
- name: Build SQL Syntax
run: ./build.sh
working-directory: ./postgres/parser
shell: bash
- name: Run check
run: ./run_staticcheck.sh
working-directory: ./scripts
+48
View File
@@ -0,0 +1,48 @@
name: Email Team Members
on:
repository_dispatch:
types: [ email-report ]
permissions:
id-token: write # This is required for requesting the JWT
contents: read # This is required for actions/checkout
jobs:
email-team:
runs-on: ubuntu-22.04
name: Email Team Members
steps:
- uses: actions/checkout@v6
- name: Configure AWS Credentials
uses: aws-actions/configure-aws-credentials@v4
with:
role-session-name: GitHub_to_AWS_via_FederatedOIDC
role-to-assume: ${{ secrets.AWS_ROLE_TO_ASSUME_CORP }}
aws-region: us-west-2
- name: Get Results
id: get-results
run: aws s3api get-object --bucket="$BUCKET" --key="$KEY" results.log
env:
KEY: ${{ github.event.client_payload.key }}
BUCKET: ${{ github.event.client_payload.bucket }}
- name: Get Addresses
id: get-addresses
run: |
addresses="$TEAM"
if [ ! -z "$RECIPIENT" ]; then
addresses="[\"$RECIPIENT\"]"
fi
echo "addresses=$addresses" >> $GITHUB_OUTPUT
env:
RECIPIENT: ${{ github.event.client_payload.email_recipient }}
TEAM: '["${{ secrets.PERF_REPORTS_EMAIL_ADDRESS }}"]'
- name: Send Email
uses: ./.github/actions/ses-email-action
with:
template: ${{ github.event.client_payload.template }}
region: us-west-2
version: ${{ github.event.client_payload.version }}
format: ${{ github.event.client_payload.noms_bin_format }}
toAddresses: ${{ steps.get-addresses.outputs.addresses }}
dataFile: ${{ format('{0}/results.log', github.workspace) }}
@@ -0,0 +1,75 @@
name: Benchmark Latency
on:
repository_dispatch:
types: [ benchmark-latency ]
permissions:
id-token: write # This is required for requesting the JWT
contents: read # This is required for actions/checkout
jobs:
performance:
runs-on: ubuntu-22.04
name: Benchmark Performance
steps:
- name: Checkout
uses: actions/checkout@v6
- uses: azure/setup-kubectl@v4
with:
version: 'v1.23.6'
- name: Install aws-iam-authenticator
run: |
curl -o aws-iam-authenticator https://amazon-eks.s3.us-west-2.amazonaws.com/1.18.8/2020-09-18/bin/linux/amd64/aws-iam-authenticator && \
chmod +x ./aws-iam-authenticator && \
sudo cp ./aws-iam-authenticator /usr/local/bin/aws-iam-authenticator
aws-iam-authenticator version
- name: Configure AWS Credentials
uses: aws-actions/configure-aws-credentials@v4
with:
role-session-name: GitHub_to_AWS_via_FederatedOIDC
role-to-assume: ${{ secrets.AWS_ROLE_TO_ASSUME_CORP }}
aws-region: us-west-2
- name: Create and Auth kubeconfig
run: |
echo "$CONFIG" > kubeconfig
KUBECONFIG=kubeconfig kubectl config set-credentials github-actions-doltgresql --exec-api-version=client.authentication.k8s.io/v1alpha1 --exec-command=aws-iam-authenticator --exec-arg=token --exec-arg=-i --exec-arg=eks-cluster-1
KUBECONFIG=kubeconfig kubectl config set-context github-actions-doltgresql-context --cluster=eks-cluster-1 --user=github-actions-doltgresql --namespace=performance-benchmarking
KUBECONFIG=kubeconfig kubectl config use-context github-actions-doltgresql-context
env:
CONFIG: ${{ secrets.CORP_KUBECONFIG }}
- name: Create Sysbench Performance Benchmarking K8s Job
run: ./.github/scripts/performance-benchmarking/run-benchmarks.sh
env:
FROM_SERVER: ${{ github.event.client_payload.from_server }}
FROM_VERSION: ${{ github.event.client_payload.from_version }}
TO_SERVER: ${{ github.event.client_payload.to_server }}
TO_VERSION: ${{ github.event.client_payload.to_version }}
MODE: ${{ github.event.client_payload.mode }}
ISSUE_NUMBER: ${{ github.event.client_payload.issue_number }}
ACTOR: ${{ github.event.client_payload.actor }}
ACTOR_EMAIL: ${{ github.event.client_payload.actor_email }}
REPO_ACCESS_TOKEN: ${{ secrets.REPO_ACCESS_TOKEN }}
KUBECONFIG: "./kubeconfig"
INIT_BIG_REPO: ${{ github.event.client_payload.init_big_repo }}
NOMS_BIN_FORMAT: "__DOLT__"
TEMPLATE_SCRIPT: ${{ github.event.client_payload.template_script }}
# TODO: skipping tpcc with Doltgres and Postgres, not currently working
# - name: Create TPCC Performance Benchmarking K8s Job
# run: ./.github/scripts/performance-benchmarking/run-benchmarks.sh
# env:
# FROM_SERVER: ${{ github.event.client_payload.from_server }}
# FROM_VERSION: ${{ github.event.client_payload.from_version }}
# TO_SERVER: ${{ github.event.client_payload.to_server }}
# TO_VERSION: ${{ github.event.client_payload.to_version }}
# MODE: ${{ github.event.client_payload.mode }}
# ISSUE_NUMBER: ${{ github.event.client_payload.issue_number }}
# ACTOR: ${{ github.event.client_payload.actor }}
# ACTOR_EMAIL: ${{ github.event.client_payload.actor_email }}
# REPO_ACCESS_TOKEN: ${{ secrets.REPO_ACCESS_TOKEN }}
# KUBECONFIG: "./kubeconfig"
# INIT_BIG_REPO: ${{ github.event.client_payload.init_big_repo }}
# NOMS_BIN_FORMAT: "__DOLT__"
# WITH_TPCC: "true"
# TEMPLATE_SCRIPT: ${{ github.event.client_payload.template_script }}
@@ -0,0 +1,53 @@
name: SQL Correctness
on:
repository_dispatch:
types: [ sql-correctness ]
permissions:
id-token: write # This is required for requesting the JWT
contents: read # This is required for actions/checkout
jobs:
correctness:
runs-on: ubuntu-22.04
name: Dolt SQL Correctness
steps:
- uses: actions/checkout@v6
- uses: azure/setup-kubectl@v4
with:
version: 'v1.23.6'
- name: Configure AWS Credentials
uses: aws-actions/configure-aws-credentials@v4
with:
role-session-name: GitHub_to_AWS_via_FederatedOIDC
role-to-assume: ${{ secrets.AWS_ROLE_TO_ASSUME_CORP }}
aws-region: us-west-2
- name: Install aws-iam-authenticator
run: |
curl -o aws-iam-authenticator https://amazon-eks.s3.us-west-2.amazonaws.com/1.18.8/2020-09-18/bin/linux/amd64/aws-iam-authenticator && \
chmod +x ./aws-iam-authenticator && \
sudo cp ./aws-iam-authenticator /usr/local/bin/aws-iam-authenticator
aws-iam-authenticator version
- name: Create and Auth kubeconfig
run: |
echo "$CONFIG" > kubeconfig
KUBECONFIG=kubeconfig kubectl config set-credentials github-actions-doltgresql --exec-api-version=client.authentication.k8s.io/v1alpha1 --exec-command=aws-iam-authenticator --exec-arg=token --exec-arg=-i --exec-arg=eks-cluster-1
KUBECONFIG=kubeconfig kubectl config set-context github-actions-doltgresql-context --cluster=eks-cluster-1 --user=github-actions-doltgresql --namespace=performance-benchmarking
KUBECONFIG=kubeconfig kubectl config use-context github-actions-doltgresql-context
env:
CONFIG: ${{ secrets.CORP_KUBECONFIG }}
- name: Create SQL Correctness K8s Job
run: ./.github/scripts/sql-correctness/run-correctness.sh
env:
PR_BRANCH_REF: ${{ github.event.client_payload.branch_ref }}
REGRESS_COMP: ${{ github.event.client_payload.regress_comp }}
PR_NUMBER: ${{ github.event.client_payload.issue_number }}
VERSION: ${{ github.event.client_payload.version }}
MODE: ${{ github.event.client_payload.mode }}
ACTOR: ${{ github.event.client_payload.actor }}
ACTOR_EMAIL: ${{ github.event.client_payload.actor_email }}
REPO_ACCESS_TOKEN: ${{ secrets.REPO_ACCESS_TOKEN }}
KUBECONFIG: "./kubeconfig"
NOMS_BIN_FORMAT: "__DOLT__"
TEMPLATE_SCRIPT: ${{ github.event.client_payload.template_script }}
@@ -0,0 +1,19 @@
name: Label Customer Issues
on:
issues:
types: [opened]
pull_request_target:
branches: [main]
types: [opened]
jobs:
label_customer_issues:
runs-on: ubuntu-latest
steps:
- uses: dolthub/label-customer-issues@main
with:
repo-token: ${{ secrets.REPO_ACCESS_TOKEN }}
issue-label: customer issue
pr-label: contribution
exclude: dependabot
+129
View File
@@ -0,0 +1,129 @@
name: Mini Sysbench
on:
pull_request:
types: [opened, synchronize, reopened]
permissions:
contents: read
pull-requests: write
jobs:
mini-sysbench:
runs-on: ubuntu-latest
steps:
- name: Checkout DoltgreSQL
uses: actions/checkout@v6
with:
ref: ${{ github.event.pull_request.head.sha }}
- name: Setup Git User
uses: fregante/setup-git-user@v2
- name: Install Go
uses: actions/setup-go@v5
with:
go-version-file: go.mod
- name: Install Sysbench
run: |
curl -s https://packagecloud.io/install/repositories/akopytov/sysbench/script.deb.sh | sudo bash
sudo apt -y install sysbench
- name: Test PR branch
id: test_doltgresql_pr
continue-on-error: true
run: |
./postgres/parser/build.sh
./scripts/quick_sysbench.sh
mv ./scripts/mini_sysbench/results.log ./scripts/mini_sysbench/results1.log
cat ./scripts/mini_sysbench/results1.log
- name: Test main branch
id: test_doltgresql_main
continue-on-error: true
run: |
git reset --hard
git fetch --all --unshallow
git checkout origin/main
./postgres/parser/build.sh
./scripts/quick_sysbench.sh
mv ./scripts/mini_sysbench/results.log ./scripts/mini_sysbench/results2.log
cat ./scripts/mini_sysbench/results2.log
- name: Check Sysbench Logs
id: check_logs
run: |
cd scripts/mini_sysbench
if [[ -f "results1.log" && -f "results2.log" ]]; then
echo "logs_exist=true" >> $GITHUB_OUTPUT
echo "logs exist"
else
echo "logs_exist=false" >> $GITHUB_OUTPUT
echo "One of the branches could not successfully run the benchmarks."
echo "Please review them for errors, which should be fixed."
exit 1
fi
- name: Build Sysbench Results Comment
id: build_results
if: steps.check_logs.outputs.logs_exist == 'true'
run: |
cd testing/go/benchmark
output=$(go run .)
echo "program_output<<EOF" >> $GITHUB_OUTPUT
echo "$output" >> $GITHUB_OUTPUT
echo "EOF" >> $GITHUB_OUTPUT
echo "$output"
- name: Is PR From Fork
id: from_fork
run: |
if [ "${{ github.event.pull_request.head.repo.full_name }}" != "${{ github.repository }}" ]; then
echo "This is running from a fork, skipping commenting"
echo "fork=true" >> $GITHUB_OUTPUT
else
echo "This is not running from a fork"
echo "fork=false" >> $GITHUB_OUTPUT
fi
- name: Post Comment
if: steps.from_fork.outputs.fork == 'false' && steps.build_results.outputs.program_output
uses: actions/github-script@v6
env:
PROGRAM_OUTPUT: ${{ steps.build_results.outputs.program_output }}
with:
github-token: ${{ secrets.GITHUB_TOKEN }}
script: |
const commentMarker = '<!-- go-run-output-sysbench -->'
const output = process.env.PROGRAM_OUTPUT
const body = `${commentMarker}\n${output}`
// List comments on the PR
const { data: comments } = await github.rest.issues.listComments({
issue_number: context.issue.number,
owner: context.repo.owner,
repo: context.repo.repo,
})
// Check if a comment already exists
const comment = comments.find(comment => comment.body.includes(commentMarker))
if (comment) {
// Update the existing comment
await github.rest.issues.updateComment({
comment_id: comment.id,
owner: context.repo.owner,
repo: context.repo.repo,
body: body
})
} else {
// Create a new comment
await github.rest.issues.createComment({
issue_number: context.issue.number,
owner: context.repo.owner,
repo: context.repo.repo,
body: body
})
}
@@ -0,0 +1,22 @@
name: Nightly Benchmarks
on:
workflow_dispatch:
schedule:
- cron: '0 2 * * *'
jobs:
perf:
runs-on: ubuntu-22.04
name: Trigger Benchmark Latency and SQL Correctness K8s Workflows
steps:
- uses: peter-evans/repository-dispatch@v2.0.0
with:
token: ${{ secrets.REPO_ACCESS_TOKEN }}
event-type: benchmark-latency
client-payload: '{"from_server": "postgres", "from_version": "15.5", "to_server": "doltgres", "to_version": "${{ github.sha }}", "mode": "nightly", "actor": "${{ github.actor }}", "template_script": "./.github/scripts/performance-benchmarking/get-postgres-doltgres-job-json.sh"}'
- uses: peter-evans/repository-dispatch@v2.0.0
with:
token: ${{ secrets.REPO_ACCESS_TOKEN }}
event-type: sql-correctness
client-payload: '{"version": "${{ github.sha }}", "mode": "nightly", "actor": "${{ github.actor }}", "template_script": "./.github/scripts/sql-correctness/get-doltgres-correctness-job-json.sh"}'
+22
View File
@@ -0,0 +1,22 @@
name: Open update README PR
on:
repository_dispatch:
types: [ open-update-pr ]
jobs:
open-pr:
runs-on: ubuntu-22.04
steps:
- name: Checkout code
uses: actions/checkout@v6
with:
ref: main
token: ${{ secrets.REPO_ACCESS_TOKEN || secrets.GITHUB_TOKEN }}
- name: pull-request
uses: repo-sync/pull-request@v2
with:
source_branch: ${{ github.event.client_payload.source_branch }}
destination_branch: ${{ github.event.client_payload.destination_branch }}
github_token: ${{ secrets.REPO_ACCESS_TOKEN || secrets.GITHUB_TOKEN }}
pr_title: ${{ github.event.client_payload.title }}
@@ -0,0 +1,46 @@
name: Benchmark DoltgreSQL vs PostgreSQL
on:
repository_dispatch:
types: [ release-doltgres ]
workflow_dispatch:
inputs:
version:
description: 'SemVer format release tag, i.e. 0.24.5'
required: true
default: ''
email:
description: 'Email address to receive results'
required: true
default: ''
jobs:
set-version-actor:
name: Set Version and Actor
runs-on: ubuntu-22.04
outputs:
version: ${{ steps.set-vars.outputs.version }}
actor: ${{ steps.set-vars.outputs.actor }}
actor_email: ${{ steps.set-vars.outputs.actor_email }}
steps:
- name: Set variables
id: set-vars
run: |
echo "version=$VERSION" >> $GITHUB_OUTPUT
echo "actor=$ACTOR" >> $GITHUB_OUTPUT
echo "actor_email=$ACTOR_EMAIL" >> $GITHUB_OUTPUT
env:
VERSION: ${{ github.event.inputs.version || github.event.client_payload.version }}
ACTOR: ${{ github.event.client_payload.actor || github.actor }}
ACTOR_EMAIL: ${{ github.event.inputs.email }}
benchmark-dolt-mysql:
runs-on: ubuntu-22.04
needs: set-version-actor
name: Trigger Benchmark Latency and Benchmark Import K8s Workflows
steps:
- uses: peter-evans/repository-dispatch@v2.0.0
with:
token: ${{ secrets.REPO_ACCESS_TOKEN }}
event-type: benchmark-latency
client-payload: '{"from_server": "postgres", "from_version": "15.5", "to_server": "doltgres", "to_version": "${{ needs.set-version-actor.outputs.version }}", "mode": "release", "actor": "${{ needs.set-version-actor.outputs.actor }}", "actor_email": "${{ needs.set-version-actor.outputs.actor_email }}", "template_script": "./.github/scripts/performance-benchmarking/get-postgres-doltgres-job-json.sh"}'
@@ -0,0 +1,64 @@
name: Benchmark Pull Requests
on:
pull_request:
types: [ opened ]
issue_comment:
types: [ created ]
jobs:
validate-commentor:
runs-on: ubuntu-22.04
outputs:
valid: ${{ steps.set_valid.outputs.valid }}
steps:
- uses: actions/checkout@v6
- name: Validate Commentor
id: set_valid
run: ./.github/scripts/performance-benchmarking/validate-commentor.sh "$ACTOR"
env:
ACTOR: ${{ github.actor }}
check-comments:
runs-on: ubuntu-22.04
needs: validate-commentor
if: ${{ needs.validate-commentor.outputs.valid == 'true' }}
outputs:
benchmark: ${{ steps.set_benchmark.outputs.benchmark }}
comment-body: ${{ steps.set_body.outputs.body }}
steps:
- name: Check for Deploy Trigger
uses: dolthub/pull-request-comment-trigger@master
id: check
with:
trigger: '#benchmark'
reaction: rocket
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
- name: Set Benchmark
if: ${{ steps.check.outputs.triggered == 'true' }}
id: set_benchmark
run: |
echo "benchmark=true" >> $GITHUB_OUTPUT
performance:
runs-on: ubuntu-22.04
needs: [validate-commentor, check-comments]
if: ${{ needs.check-comments.outputs.benchmark == 'true' }}
name: Trigger Benchmark Latency K8s Workflow
steps:
- uses: dolthub/pull-request-comment-branch@v4
id: comment-branch
with:
repo_token: ${{ secrets.GITHUB_TOKEN }}
- name: Get pull number
uses: actions/github-script@v7
id: get_pull_number
with:
github-token: ${{ secrets.GITHUB_TOKEN }}
script: core.setOutput("pull_number", JSON.stringify(context.issue.number));
- uses: peter-evans/repository-dispatch@v2.0.0
with:
token: ${{ secrets.REPO_ACCESS_TOKEN }}
event-type: benchmark-latency
client-payload: '{"from_server": "doltgres", "from_version": "${{ github.sha }}", "to_server": "doltgres", "to_version": "${{ steps.comment-branch.outputs.head_sha }}", "mode": "pullRequest", "issue_number": "${{ steps.get_pull_number.outputs.pull_number }}", "init_big_repo": "true", "actor": "${{ github.actor }}", "template_script": "./.github/scripts/performance-benchmarking/get-doltgres-doltgres-job-json.sh"}'
+54
View File
@@ -0,0 +1,54 @@
name: Post to Pull Request
on:
repository_dispatch:
types: [ pull-report ]
permissions:
id-token: write # This is required for requesting the JWT
contents: read # This is required for actions/checkout
jobs:
report-pull-request:
name: Report Performance Benchmarks on Pull Request
runs-on: ubuntu-22.04
if: ${{ github.event.client_payload.issue_number != -1 }}
steps:
- name: Checkout
uses: actions/checkout@v6
- name: Configure AWS Credentials
uses: aws-actions/configure-aws-credentials@v4
with:
role-session-name: GitHub_to_AWS_via_FederatedOIDC
role-to-assume: ${{ secrets.AWS_ROLE_TO_ASSUME_CORP }}
aws-region: us-west-2
- name: Get benchmark results
id: get-results
run: aws s3api get-object --bucket="$BUCKET" --key="$KEY" results.log
env:
KEY: ${{ github.event.client_payload.key }}
BUCKET: ${{ github.event.client_payload.bucket }}
- name: Post results to PR
uses: actions/github-script@v7
with:
github-token: ${{ secrets.REPO_ACCESS_TOKEN || secrets.GITHUB_TOKEN }}
script: |
const { ACTOR, FORMAT, ISSUE_NUMBER, GITHUB_WORKSPACE } = process.env;
const issue_number = parseInt(ISSUE_NUMBER, 10);
const { owner, repo } = context.repo;
fs = require('fs');
fs.readFile(`${GITHUB_WORKSPACE}/results.log`, 'utf8', function (err,data) {
if (err) {
return console.log(err);
}
return github.rest.issues.createComment({
issue_number,
owner,
repo,
body: `@${ACTOR} ${FORMAT}\n ${data}`
});
});
env:
ACTOR: ${{ github.event.client_payload.actor }}
ISSUE_NUMBER: ${{ github.event.client_payload.issue_number }}
FORMAT: ${{ github.event.client_payload.noms_bin_format }}
+283
View File
@@ -0,0 +1,283 @@
name: Regression Tests
on:
pull_request:
types: [opened, synchronize, reopened]
env:
REGRESSION_TESTING: true
permissions:
contents: read
pull-requests: write
concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number }}
cancel-in-progress: true
jobs:
resolve-main:
name: Resolve main
runs-on: ubuntu-latest
outputs:
main_sha: ${{ steps.resolve.outputs.main_sha }}
steps:
- name: Checkout DoltgreSQL
uses: actions/checkout@v6
with:
ref: main
- name: Resolve main SHA
id: resolve
run: echo "main_sha=$(git rev-parse HEAD)" >> "$GITHUB_OUTPUT"
pr-regression:
name: PR regression
runs-on: ubuntu-latest
needs: resolve-main
steps:
- name: Checkout DoltgreSQL
uses: actions/checkout@v6
with:
ref: ${{ github.event.pull_request.head.sha }}
fetch-depth: 0
- name: Setup Git User
uses: fregante/setup-git-user@v2
- name: Merge base into PR
run: |
git fetch origin ${{ needs.resolve-main.outputs.main_sha }}
git merge ${{ needs.resolve-main.outputs.main_sha }} --no-commit --no-ff
- name: Install Go
uses: actions/setup-go@v5
with:
go-version-file: go.mod
cache: true
- name: Test PR branch
continue-on-error: true
run: |
./postgres/parser/build.sh
cd testing/go/regression
mkdir -p out
{
echo "pr_head=${{ github.event.pull_request.head.sha }}"
echo "main_head=${{ needs.resolve-main.outputs.main_sha }}"
echo "merge_base=$(git merge-base HEAD ${{ needs.resolve-main.outputs.main_sha }})"
git status --short --branch
git diff --cached --stat
go version
go env
go list -m github.com/dolthub/dolt/go github.com/dolthub/go-mysql-server
} > out/pr-metadata.txt
cd tool
set +e
go test --timeout=90m ./... --count=1 2>&1 | tee ../out/pr-go-test.log
test_status=${PIPESTATUS[0]}
set -e
if [ -f ../out/results.trackers ]; then
cp ../out/results.trackers ../out/results2.trackers
fi
exit "$test_status"
- name: Upload PR regression artifacts
if: always()
uses: actions/upload-artifact@v4
with:
name: regression-pr-${{ github.run_id }}-${{ github.run_attempt }}
path: |
testing/go/regression/out/results2.trackers
testing/go/regression/out/pr-go-test.log
testing/go/regression/out/pr-metadata.txt
if-no-files-found: warn
retention-days: 14
compression-level: 0
main-regression:
name: Main regression
runs-on: ubuntu-latest
needs: resolve-main
steps:
- name: Checkout DoltgreSQL
uses: actions/checkout@v6
with:
ref: ${{ needs.resolve-main.outputs.main_sha }}
- name: Install Go
uses: actions/setup-go@v5
with:
go-version-file: go.mod
cache: true
- name: Test main branch
continue-on-error: true
run: |
./postgres/parser/build.sh
cd testing/go/regression
mkdir -p out
{
echo "main_head=${{ needs.resolve-main.outputs.main_sha }}"
git status --short --branch
go version
go env
go list -m github.com/dolthub/dolt/go github.com/dolthub/go-mysql-server
} > out/main-metadata.txt
cd tool
set +e
go test --timeout=90m ./... --count=1 2>&1 | tee ../out/main-go-test.log
test_status=${PIPESTATUS[0]}
set -e
if [ -f ../out/results.trackers ]; then
cp ../out/results.trackers ../out/results1.trackers
fi
exit "$test_status"
- name: Upload main regression artifacts
if: always()
uses: actions/upload-artifact@v4
with:
name: regression-main-${{ github.run_id }}-${{ github.run_attempt }}
path: |
testing/go/regression/out/results1.trackers
testing/go/regression/out/main-go-test.log
testing/go/regression/out/main-metadata.txt
if-no-files-found: warn
retention-days: 14
compression-level: 0
regression-tests:
runs-on: ubuntu-latest
needs:
- resolve-main
- pr-regression
- main-regression
if: always() && needs.pr-regression.result != 'cancelled' && needs.main-regression.result != 'cancelled'
steps:
- name: Checkout DoltgreSQL
uses: actions/checkout@v6
with:
ref: ${{ needs.resolve-main.outputs.main_sha }}
- name: Install Go
uses: actions/setup-go@v5
with:
go-version-file: go.mod
cache: true
- name: Download PR regression artifacts
uses: actions/download-artifact@v4
continue-on-error: true
with:
name: regression-pr-${{ github.run_id }}-${{ github.run_attempt }}
path: testing/go/regression/out
- name: Download main regression artifacts
uses: actions/download-artifact@v4
continue-on-error: true
with:
name: regression-main-${{ github.run_id }}-${{ github.run_attempt }}
path: testing/go/regression/out
- name: Check result trackers
id: check_trackers
run: |
cd testing/go/regression/out
if [[ -f "results1.trackers" && -f "results2.trackers" ]]; then
echo "trackers_exist=true" >> $GITHUB_OUTPUT
echo "trackers exist"
else
echo "trackers_exist=false" >> $GITHUB_OUTPUT
echo "One of the branches could not successfully complete their tests."
echo "Please review uploaded regression artifacts for errors, which must be fixed."
exit 1
fi
- name: Build Regression Test Results Comment
id: build_results
if: steps.check_trackers.outputs.trackers_exist == 'true'
run: |
./postgres/parser/build.sh
cd testing/go/regression/tool
# Pass via file, not $GITHUB_OUTPUT: large output through env trips ARG_MAX in github-script.
go run . results1.trackers results2.trackers > "$RUNNER_TEMP/regression-comment.md"
cp "$RUNNER_TEMP/regression-comment.md" ../out/regression-comment.md
cat "$RUNNER_TEMP/regression-comment.md"
if [ -s "$RUNNER_TEMP/regression-comment.md" ]; then
echo "has_output=true" >> "$GITHUB_OUTPUT"
else
echo "has_output=false" >> "$GITHUB_OUTPUT"
fi
- name: Upload comparison artifacts
if: always()
uses: actions/upload-artifact@v4
with:
name: regression-comparison-${{ github.run_id }}-${{ github.run_attempt }}
path: testing/go/regression/out/regression-comment.md
if-no-files-found: warn
retention-days: 14
compression-level: 0
- name: Is PR From Fork
id: from_fork
run: |
if [ "${{ github.event.pull_request.head.repo.full_name }}" != "${{ github.repository }}" ]; then
echo "This is running from a fork, skipping commenting"
echo "fork=true" >> $GITHUB_OUTPUT
else
echo "This is not running from a fork"
echo "fork=false" >> $GITHUB_OUTPUT
fi
- name: Post Comment
if: steps.from_fork.outputs.fork == 'false' && steps.build_results.outputs.has_output == 'true'
uses: actions/github-script@v7
env:
COMMENT_PATH: ${{ runner.temp }}/regression-comment.md
with:
github-token: ${{ secrets.GITHUB_TOKEN }}
script: |
const fs = require('fs')
const commentMarker = '<!-- go-run-output -->'
// GitHub caps comment bodies at 65536 chars.
const MAX_BODY = 65000
let output = fs.readFileSync(process.env.COMMENT_PATH, 'utf8')
let body = `${commentMarker}\n${output}`
if (body.length > MAX_BODY) {
const notice = '\n\n_Output truncated; see workflow logs for the full report._'
body = body.slice(0, MAX_BODY - notice.length) + notice
}
// List comments on the PR
const { data: comments } = await github.rest.issues.listComments({
issue_number: context.issue.number,
owner: context.repo.owner,
repo: context.repo.repo,
})
// Check if a comment already exists
const comment = comments.find(comment => comment.body.includes(commentMarker))
if (comment) {
// Update the existing comment
await github.rest.issues.updateComment({
comment_id: comment.id,
owner: context.repo.owner,
repo: context.repo.repo,
body: body
})
} else {
// Create a new comment
await github.rest.issues.createComment({
issue_number: context.issue.number,
owner: context.repo.owner,
repo: context.repo.repo,
body: body
})
}
+19
View File
@@ -0,0 +1,19 @@
name: Replication Tests
on: [pull_request]
concurrency:
group: ci-replication-tests-${{ github.event.pull_request.number || github.ref }}
cancel-in-progress: true
jobs:
testing-job:
runs-on: ubuntu-22.04
timeout-minutes: 10
name: Run tests
steps:
- name: Check out code
uses: actions/checkout@v6
- name: Build docker image
run: docker build -t local --file testing/ReplicationTestDockerfile .
- name: Run tests
run: docker run --detach=false local
+43
View File
@@ -0,0 +1,43 @@
name: Benchmark SQL Correctness
on:
repository_dispatch:
types: [ release-doltgres ]
workflow_dispatch:
inputs:
version:
description: 'SemVer format release tag, i.e. 0.24.5'
required: true
default: ''
email:
description: 'Email address to receive results'
required: true
default: ''
jobs:
set-version-actor:
name: Set Version and Actor
runs-on: ubuntu-22.04
outputs:
version: ${{ steps.set-vars.outputs.version }}
actor: ${{ steps.set-vars.outputs.actor }}
steps:
- name: Set variables
id: set-vars
run: |
echo "version=$VERSION" >> $GITHUB_OUTPUT
echo "actor=$ACTOR" >> $GITHUB_OUTPUT
env:
VERSION: ${{ github.event.inputs.version || github.event.client_payload.version }}
ACTOR: ${{ github.event.client_payload.actor || github.actor }}
correctness:
runs-on: ubuntu-22.04
needs: set-version-actor
name: Trigger SQL Correctness K8s Workflow
steps:
- uses: peter-evans/repository-dispatch@v2.0.0
with:
token: ${{ secrets.REPO_ACCESS_TOKEN }}
event-type: sql-correctness
client-payload: '{"version": "${{ needs.set-version-actor.outputs.version }}", "mode": "release", "actor": "${{ needs.set-version-actor.outputs.actor }}", "actor_email": "${{ needs.set-version-actor.outputs.actor_email }}", "template_script": "./.github/scripts/sql-correctness/get-doltgres-correctness-job-json.sh"}'
+47
View File
@@ -0,0 +1,47 @@
name: Enginetests
on: [pull_request]
concurrency:
group: test-enginetests-${{ github.event.pull_request.number || github.ref }}
cancel-in-progress: true
jobs:
test:
strategy:
fail-fast: false
matrix:
platform: [ubuntu-latest, macos-latest, windows-latest]
runs-on: ${{ matrix.platform }}
steps:
- name: Checkout code
uses: actions/checkout@v6
- name: Set up Go
uses: actions/setup-go@v5
with:
go-version-file: go.mod
id: go
- name: Install ICU4C (MacOS)
if: ${{ matrix.platform == 'macos-latest' }}
run: |
dir=$(brew --cellar icu4c)
dir="$dir"/$(ls "$dir")
echo CGO_CPPFLAGS=-I$dir/include >> $GITHUB_ENV
echo CGO_LDFLAGS=-L$dir/lib >> $GITHUB_ENV
- name: Install ICU4C (Windows)
if: ${{ matrix.platform == 'windows-latest' }}
uses: msys2/setup-msys2@v2
with:
path-type: inherit
msystem: UCRT64
pacboy: icu:p toolchain:p pkg-config:p
- name: Build SQL Syntax
run: ./build.sh
working-directory: ./postgres/parser
shell: bash
- name: Test (*nix)
if: ${{ matrix.platform != 'windows-latest' }}
run: go test --timeout=20m $(go list ./testing/go/enginetest)
- name: Test (Windows)
if: ${{ matrix.platform == 'windows-latest' }}
shell: msys2 {0}
run: go.exe test --timeout=20m $(go.exe list ./testing/go/enginetest)
+59
View File
@@ -0,0 +1,59 @@
name: Test Import Regressions
on: [pull_request]
concurrency:
group: test-import-dumps-${{ github.event.pull_request.number || github.ref }}
cancel-in-progress: true
jobs:
test-import-dumps:
strategy:
fail-fast: false
matrix:
platform: [ubuntu-latest, macos-latest, windows-latest]
runs-on: ${{ matrix.platform }}
env:
GITHUB_ACTION_IMPORT_DUMPS: true
steps:
- name: Checkout code
uses: actions/checkout@v6
- name: Set up Go
uses: actions/setup-go@v5
with:
go-version-file: go.mod
id: go
- name: Install ICU4C (MacOS)
if: ${{ matrix.platform == 'macos-latest' }}
run: |
dir=$(brew --cellar icu4c)
dir="$dir"/$(ls "$dir")
echo CGO_CPPFLAGS=-I$dir/include >> $GITHUB_ENV
echo CGO_LDFLAGS=-L$dir/lib >> $GITHUB_ENV
- name: Install ICU4C (Windows)
if: ${{ matrix.platform == 'windows-latest' }}
uses: msys2/setup-msys2@v2
with:
path-type: inherit
msystem: UCRT64
pacboy: icu:p toolchain:p pkg-config:p
- name: Build SQL Syntax
run: ./build.sh
working-directory: ./postgres/parser
shell: bash
- name: Setup PostgreSQL
uses: tj-actions/install-postgresql@v3
with:
postgresql-version: 15
- name: Test (MacOS)
if: ${{ matrix.platform == 'macos-latest' }}
working-directory: ./testing/go
run: go test --timeout=30m -run TestImportingDumps .
- name: Test (Windows)
if: ${{ matrix.platform == 'windows-latest' }}
working-directory: ./testing/go
shell: msys2 {0}
run: go.exe test --timeout=30m -run TestImportingDumps .
- name: Test (Linux)
if: ${{ matrix.platform == 'ubuntu-latest' }}
working-directory: ./testing/go
run: go test --timeout=30m -run TestImportingDumps .
+67
View File
@@ -0,0 +1,67 @@
name: Test
on: [pull_request]
concurrency:
group: test-${{ github.event.pull_request.number || github.ref }}
cancel-in-progress: true
jobs:
test:
strategy:
fail-fast: false
matrix:
platform: [ubuntu-latest, macos-latest, windows-latest]
runs-on: ${{ matrix.platform }}
steps:
- name: Set timezone for Unix
uses: szenius/set-timezone@v2.0
with:
timezoneLinux: "America/Los_Angeles"
timezoneMacos: "America/Los_Angeles"
- name: Set timezone for Windows
if: ${{ matrix.platform == 'windows-latest' }}
shell: pwsh
run: |
tzutil /s "Pacific Standard Time"
tzutil /g
Get-TimeZone
- name: Checkout code
uses: actions/checkout@v6
- name: Set up Go
uses: actions/setup-go@v5
with:
go-version-file: go.mod
id: go
- name: Install ICU4C (MacOS)
if: ${{ matrix.platform == 'macos-latest' }}
run: |
dir=$(brew --cellar icu4c)
dir="$dir"/$(ls "$dir")
echo CGO_CPPFLAGS=-I$dir/include >> $GITHUB_ENV
echo CGO_LDFLAGS=-L$dir/lib >> $GITHUB_ENV
- name: Install ICU4C (Windows)
if: ${{ matrix.platform == 'windows-latest' }}
uses: msys2/setup-msys2@v2
with:
path-type: inherit
msystem: UCRT64
pacboy: icu:p toolchain:p pkg-config:p
- name: Build SQL Syntax
run: ./build.sh
working-directory: ./postgres/parser
shell: bash
- name: Setup PostgreSQL
uses: tj-actions/install-postgresql@v3
with:
postgresql-version: 15
- name: Test (MacOS)
if: ${{ matrix.platform == 'macos-latest' }}
run: go test --timeout=20m -skip="TestReplication" $(go list ./... | grep -v enginetest)
- name: Test (Windows)
if: ${{ matrix.platform == 'windows-latest' }}
shell: msys2 {0}
run: go.exe test --timeout=20m -skip="TestReplication" $(go.exe list ./... | grep -v enginetest)
- name: Test (Linux)
if: ${{ matrix.platform == 'ubuntu-latest' }}
# Enginetest harness breaks with race testing, not sure why yet
run: go test --timeout=20m -race -skip="TestReplication" $(go list ./... | grep -v enginetest)
+32
View File
@@ -0,0 +1,32 @@
**/.idea/
**/.dolt/
.vscode
.run
venv
.DS_Store
.sqlhistory
.doltcfg
test.sh
# ignore cp'd sysbench runner testing files
SysbenchDockerfile
SysbenchDockerfile.dockerignore
sysbench-runner-tests-entrypoint.sh
config.json
integration-tests/bats/batsee_results
# ignore log files created from logic test and regression tests
testing/logictest/*.log
testing/go/regression/out
# ignore sysbench
scripts/mini_sysbench
# ignore doltgres db created
doltgres
postgres
auth.db
# ignore extensions output
core/extensions/pg_extension/output
+23
View File
@@ -0,0 +1,23 @@
# Lines starting with '#' are comments.
# Each line is a file pattern followed by one or more owners.
# These owners will be the default owners for everything in the repo.
# * @defunkt
# Order is important. The last matching pattern has the most precedence.
# So if a pull request only touches javascript files, only these owners
# will be requested to review.
# *.js @octocat @github/js
# You can also use email addresses if you prefer.
# docs/* docs@example.com
# Begin dolt repository owners
testing/postgres-client-tests/node/* @tbantle22
+139
View File
@@ -0,0 +1,139 @@
# Contribution Guide for DoltgreSQL
## Introduction
Welcome to the DoltgreSQL repository!
This guide outlines how to get started, in addition to how to modify the code.
We have a standard guideline that we try to adhere to for the project, and this guide will go over all of those.
I will assume familiarity with the Go toolchain.
## Getting Set Up
1. **Star the repository**: Since you're about to contribute to the project, why not give us a star to show your support? ⭐️
2. **Install the latest [Go](https://go.dev/dl/)**: We generally attempt to stay up-to-date with the latest version of Go.
For the specific version in use, you may check the [go.mod](https://github.com/dolthub/doltgresql/blob/main/go.mod) file.
3. **Install PSQL 15**: Although we do not use PostgreSQL, we do use the PSQL client (for [PostgreSQL 15](https://www.postgresql.org/download/)) for testing.
In addition, it is worthwhile to have a local PostgreSQL instance installed to cross-check behavior, especially when writing tests (which should _always_ be verified by a [PostgreSQL 15](https://www.postgresql.org/download/) instance).
4. **Clone the repository**: Clone the repository to a local directory of your choice.
5. **Build the parser**: Run the `doltgresql/postgres/parser/build.sh` script.
This creates a few files within the `doltgresql/postgres/parser/parser` directory that are necessary for parsing PostgreSQL statements.
It is recommended to run this file every time you pull changes into your local repository, as these generated files are not included since they would cause near guaranteed merge conflicts.
6. **Run Go tests**: Before building the project, you should always run all of the tests, which can be done by running `go test ./... --count=1` from the source root directory.
This ensures that all Go tests pass, which also ensures that your Go environment is installed and configured correctly.
7. **Build the binary**: From the source root directory, run `go build -o <bin_name> ./cmd/doltgres`, where `<bin_name>` is the name of the binary (usually `doltgres` or `doltgres.exe`).
To run the program without creating an executable, run `go run ./cmd/doltgres`.
8. **Run Bats tests**: We make use of [Bats](https://github.com/bats-core/bats-core) for all end-user-style tests.
Assuming you have [NPM installed](https://docs.npmjs.com/downloading-and-installing-node-js-and-npm), you can install [Bats](https://www.npmjs.com/package/bats) by running `npm install -g bats`.
Then, navigate to `doltgresql/testing/bats` and run `bats .`, which will run all of the Bats tests in the directory.
An alternative is to use [BashSupport Pro](https://plugins.jetbrains.com/plugin/13841-bashsupport-pro), which is cross-platform and used by several developers.
Additionally, our [Bats](https://github.com/bats-core/bats-core) tests assume that you have a `doltgresql` (not `doltgres`) binary on your PATH.
For Windows users, this means that the binary should _not_ end with the `.exe` file extension.
Remember to recompile the executable on your PATH whenever you want to re-test any [Bats](https://github.com/bats-core/bats-core) tests.
9. **Change the data directory**: This is optional but recommended.
By default, we create databases within the `~/doltgres/databases` directory.
For developmental purposes, you may want to change this behavior. You have two options:
1. Set the `DOLTGRES_DATA_DIR` environment variable to a different directory. A value of `.` causes DoltgreSQL to use the current directory as the data directory, so you can have multiple data directories simply by running the program in different directories. This behavior is more consistent with [Dolt's](https://github.com/dolthub/dolt) behavior. This is the recommended option for development.
2. Specify the directory in the `--data-dir` argument. This overrides the environment variable if it is present.
### Note for Windows Users
All of the tooling in this repository is designed for a Unix-like environment.
Some of the lead project developers are on Windows, however they all prefer the Unix commandline environment over Command Prompt and Powershell.
These are a few alternatives that are used to provide a Unix environment:
- **[Git Bash](https://git-scm.com/downloads)**: The Mintty client that may be installed alongside Git.
- **[WSL 1](https://learn.microsoft.com/en-us/windows/wsl/install)**: The Linux subsystem for Windows, specifically version 1.
We've attempted to use version 2 with [Dolt](https://github.com/dolthub/dolt), however we ran into a few issues.
It's possible that it works, but since we do not test or support it, we heavily suggest sticking to version 1 instead.
- **[Cygwin](https://www.cygwin.com/install.html)**: Cygwin is a full Unix-like environment for Windows.
Unlike WSL, which runs a full Linux kernel within Windows, Cygwin emulates Unix through native Windows processes.
This has not been extensively tested, however some have had success using it for development.
## Submitting Changes
1. **Format the repository**: You can do this by running `doltgresql/scripts/format_repo.sh`.
This will reformat any changes to adhere to our preferred style.
This is required, otherwise the pull request's checks will not pass.
2. **Run all tests**: You should ensure that all tests are running correctly.
All changes are required to have passing tests before they're merged.
Additionally, most changes should have tests.
We will not accept submissions that do not properly test to verify their behavior.
3. **Create a Pull Request**: Create your pull request and base it against the [main](https://github.com/dolthub/doltgresql/tree/main) branch.
All pull requests should have a description stating what was fixed, and possibly why it was broken.
If a specific issue is being fixed, then that issue should be linked to from the PR.
In some cases, the PR is not properly tagged from within the issue, so [refer to this link for manually linking the PR within the issue](https://docs.github.com/en/issues/tracking-your-work-with-issues/linking-a-pull-request-to-an-issue#manually-linking-a-pull-request-to-an-issue-using-the-pull-request-sidebar).
4. **Address Feedback**: If any feedback was provided on the PR, then it must be addressed before it may be merged.
In some cases, an approving review will not be given until we've re-examined the pull request.
5. **Merge the Pull Request**: Congratulations!
You're now an official contributor of the [DoltgreSQL](https://github.com/dolthub/doltgresql) project.
## General Style Rules
Here are a few rules that apply to the entire codebase:
1. Comment your code as thoroughly as possible.
It is better to over-comment your code than under-comment.
All functions require a function comment, even if it's just stating that it implements an interface.
2. Remove dead code.
Don't just put it in a comment block.
It will still be in the source history, so it can be retrieved at a later time if needed.
Commenting it out will potentially confuse another developer, who may think that it has some contextual significance.
If it does, then it should be a proper comment rather than just commented-out code.
3. File names use [snake case](https://en.wikipedia.org/wiki/Snake_case).
4. Leave a `//TODO:` comment when there is more work left to be done in an area.
This allows us to keep track of incomplete implementations, so that we don't assume a function is "complete".
A common source of bugs is when another integrator assumes that a function is completely implemented, and interacts with that function under that assumption.
If they find a `//TODO:`, then it may clue them in, and/or assist with debugging efforts.
## Rules for Specific Packages
Some packages have rules that only apply to those packages.
This is generally due to the size of those packages being larger than average, or some other unique reason that doesn't necessarily apply to the rest of the codebase.
### `postgres/parser/parser`
The nested `parser` naming may be confusing, but they're two distinct levels of the parser.
The first level, `postgres/parser`, contains all of the code for parsing a SQL statement into an AST.
The second level, `postgres/parser/parser`, specifically contains the YACC file, along with the generated build files to support the grammar contained in the YACC file.
The parser has been adapted from [CockroachDB](https://github.com/cockroachdb/cockroach/tree/f559e6a494e2f4b5dd329dc451be8c9695e3f831), using the most recent commit that falls under the Apache 2.0 License (see the [BSL](https://github.com/dolthub/doltgresql/blob/main/licenses/BSL.txt)).
As such, their [README](https://github.com/dolthub/doltgresql/blob/main/postgres/parser/parser/README.md) goes over how it's all structured, and we intend to continue in their style.
This means that we should continue to add the `%HELP` comments for statements, mark statements as errors when needed, etc.
This differs from our [Vitess](https://github.com/dolthub/vitess) fork, which is used by [GMS](https://github.com/dolthub/go-mysql-server) and [Dolt](https://github.com/dolthub/dolt).
### `testing/bats`
All Bats tests must follow this general structure:
```bash
@test "file-name: test name" {
# Test Contents
}
```
`file-name` is the name of the file, without the `.bats` file extension.
`test name` is the unique name of the test within that file.
You may surround the name portion with either single quotes `'` or double quotes `"`, although double quotes may cause issues if the test name includes a backslash.
### `testing/go`
Tests within [`testing/go`](https://github.com/dolthub/doltgresql/tree/main/testing/go) are modeled after [engine tests in GMS](https://github.com/dolthub/go-mysql-server/tree/main/enginetest).
One key deviation is the [Focus](https://github.com/dolthub/doltgresql/blob/2246d40a5ec4b92661e526480b6d4af82f232583/testing/go/framework_test.go#L54) field.
Unlike in [GMS](https://github.com/dolthub/go-mysql-server), we do not have a skipped test that a developer would locally unskip and fill in with their test details.
This is by design, as such tests may never become an actual test within a file.
To support this workflow, the [Focus](https://github.com/dolthub/doltgresql/blob/2246d40a5ec4b92661e526480b6d4af82f232583/testing/go/framework_test.go#L54) field was added.
Whenever this field is set to `true`, _only_ focused tests are run.
This means that the developer is required to write a valid test, and then they may focus on that test as though it were the only test.
Once the developer is done, they'll simply delete the [Focus](https://github.com/dolthub/doltgresql/blob/2246d40a5ec4b92661e526480b6d4af82f232583/testing/go/framework_test.go#L54) line (defaults to `false`), and all of the tests will run.
This extends beyond adding new tests, as it allows a developer to focus on failing tests too.
Let's say that two specific tests are failing, you can apply [Focus](https://github.com/dolthub/doltgresql/blob/2246d40a5ec4b92661e526480b6d4af82f232583/testing/go/framework_test.go#L54) to both of those tests, and _only_ those tests will run.
Yes, [Focus](https://github.com/dolthub/doltgresql/blob/2246d40a5ec4b92661e526480b6d4af82f232583/testing/go/framework_test.go#L54) works on more than a single test.
In fact, if [Focus](https://github.com/dolthub/doltgresql/blob/2246d40a5ec4b92661e526480b6d4af82f232583/testing/go/framework_test.go#L54) is present on any tests, then it changes the testing mode such that only tests with [Focus](https://github.com/dolthub/doltgresql/blob/2246d40a5ec4b92661e526480b6d4af82f232583/testing/go/framework_test.go#L54) are run.
This has a major benefit over commenting out other tests too, in that it's safe for use with GitHub Actions.
If a developer accidentally forgets to disable [Focus](https://github.com/dolthub/doltgresql/blob/2246d40a5ec4b92661e526480b6d4af82f232583/testing/go/framework_test.go#L54), then the actions will fail, and report that [Focus](https://github.com/dolthub/doltgresql/blob/2246d40a5ec4b92661e526480b6d4af82f232583/testing/go/framework_test.go#L54) must be disabled.
Compare this with commenting out tests, where a developer forgetting to uncomment the tests will rely on their code reviewers to catch the mistake.
If this occurs after the code review, then it may make it into the main branch, where it may remain undiscovered for years.
This has almost happened multiple times in [GMS](https://github.com/dolthub/go-mysql-server), so this method of testing is a far safer alternative.
+64
View File
@@ -0,0 +1,64 @@
# syntax=docker/dockerfile:1.3-labs
FROM debian:trixie-slim AS base
ENV DEBIAN_FRONTEND=noninteractive
RUN apt-get update -y && \
apt-get install -y --no-install-recommends \
curl tini ca-certificates postgresql-client && \
rm -rf /var/lib/apt/lists/*
# We use bookworm since the icu dependency ver. between the base and golang images is the same
FROM golang:1.25-trixie AS build-from-source
ENV DEBIAN_FRONTEND=noninteractive
ARG DOLTGRES_VERSION="latest"
RUN mkdir -p /tmp/doltgresql/
COPY . /tmp/doltgresql/
WORKDIR /tmp/doltgresql/
RUN if [ "$DOLTGRES_VERSION" = "source" ]; then \
go mod download; \
./postgres/parser/build.sh; \
./scripts/build_binaries.sh "linux-amd64"; \
mv out/doltgresql-*/bin/doltgres /usr/local/bin; \
fi
FROM base AS download-binary
ARG DOLTGRES_VERSION="latest"
RUN if [ "$DOLTGRES_VERSION" = "latest" ]; then \
version=$(curl -s "https://api.github.com/repos/dolthub/doltgresql/releases/latest" \
| grep '"tag_name"' \
| cut -d'"' -f4 \
| sed 's/^v//'); \
echo "fetching https://github.com/dolthub/doltgresql/releases/download/v${version}/install.sh"; \
curl -L "https://github.com/dolthub/doltgresql/releases/download/v${version}/install.sh" | bash; \
fi
RUN if [ "$DOLTGRES_VERSION" != "latest" ] && [ "$DOLTGRES_VERSION" != "source" ]; then \
echo "fetching https://github.com/dolthub/doltgresql/releases/download/v${DOLTGRES_VERSION}/install.sh"; \
curl -L "https://github.com/dolthub/doltgresql/releases/download/v${DOLTGRES_VERSION}/install.sh" | bash; \
fi
FROM base AS runtime
# Only one binary is possible due to DOLTGRES_VERSION, so we optionally copy from either stage
COPY --from=download-binary /usr/local/bin/doltgres* /usr/local/bin/
COPY --from=build-from-source /usr/local/bin/doltgres* /usr/local/bin/
RUN /usr/local/bin/doltgres --version
RUN mkdir /docker-entrypoint-initdb.d && \
mkdir -p /var/lib/doltgres && \
chmod 755 /var/lib/doltgres
COPY docker-entrypoint.sh /usr/local/bin/
RUN chmod +x /usr/local/bin/docker-entrypoint.sh
VOLUME /var/lib/doltgres
EXPOSE 5432
WORKDIR /var/lib/doltgres
ENTRYPOINT ["tini", "-v", "--", "docker-entrypoint.sh"]
+202
View File
@@ -0,0 +1,202 @@
Apache License
Version 2.0, January 2004
http://www.apache.org/licenses/
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
1. Definitions.
"License" shall mean the terms and conditions for use, reproduction,
and distribution as defined by Sections 1 through 9 of this document.
"Licensor" shall mean the copyright owner or entity authorized by
the copyright owner that is granting the License.
"Legal Entity" shall mean the union of the acting entity and all
other entities that control, are controlled by, or are under common
control with that entity. For the purposes of this definition,
"control" means (i) the power, direct or indirect, to cause the
direction or management of such entity, whether by contract or
otherwise, or (ii) ownership of fifty percent (50%) or more of the
outstanding shares, or (iii) beneficial ownership of such entity.
"You" (or "Your") shall mean an individual or Legal Entity
exercising permissions granted by this License.
"Source" form shall mean the preferred form for making modifications,
including but not limited to software source code, documentation
source, and configuration files.
"Object" form shall mean any form resulting from mechanical
transformation or translation of a Source form, including but
not limited to compiled object code, generated documentation,
and conversions to other media types.
"Work" shall mean the work of authorship, whether in Source or
Object form, made available under the License, as indicated by a
copyright notice that is included in or attached to the work
(an example is provided in the Appendix below).
"Derivative Works" shall mean any work, whether in Source or Object
form, that is based on (or derived from) the Work and for which the
editorial revisions, annotations, elaborations, or other modifications
represent, as a whole, an original work of authorship. For the purposes
of this License, Derivative Works shall not include works that remain
separable from, or merely link (or bind by name) to the interfaces of,
the Work and Derivative Works thereof.
"Contribution" shall mean any work of authorship, including
the original version of the Work and any modifications or additions
to that Work or Derivative Works thereof, that is intentionally
submitted to Licensor for inclusion in the Work by the copyright owner
or by an individual or Legal Entity authorized to submit on behalf of
the copyright owner. For the purposes of this definition, "submitted"
means any form of electronic, verbal, or written communication sent
to the Licensor or its representatives, including but not limited to
communication on electronic mailing lists, source code control systems,
and issue tracking systems that are managed by, or on behalf of, the
Licensor for the purpose of discussing and improving the Work, but
excluding communication that is conspicuously marked or otherwise
designated in writing by the copyright owner as "Not a Contribution."
"Contributor" shall mean Licensor and any individual or Legal Entity
on behalf of whom a Contribution has been received by Licensor and
subsequently incorporated within the Work.
2. Grant of Copyright License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
copyright license to reproduce, prepare Derivative Works of,
publicly display, publicly perform, sublicense, and distribute the
Work and such Derivative Works in Source or Object form.
3. Grant of Patent License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
(except as stated in this section) patent license to make, have made,
use, offer to sell, sell, import, and otherwise transfer the Work,
where such license applies only to those patent claims licensable
by such Contributor that are necessarily infringed by their
Contribution(s) alone or by combination of their Contribution(s)
with the Work to which such Contribution(s) was submitted. If You
institute patent litigation against any entity (including a
cross-claim or counterclaim in a lawsuit) alleging that the Work
or a Contribution incorporated within the Work constitutes direct
or contributory patent infringement, then any patent licenses
granted to You under this License for that Work shall terminate
as of the date such litigation is filed.
4. Redistribution. You may reproduce and distribute copies of the
Work or Derivative Works thereof in any medium, with or without
modifications, and in Source or Object form, provided that You
meet the following conditions:
(a) You must give any other recipients of the Work or
Derivative Works a copy of this License; and
(b) You must cause any modified files to carry prominent notices
stating that You changed the files; and
(c) You must retain, in the Source form of any Derivative Works
that You distribute, all copyright, patent, trademark, and
attribution notices from the Source form of the Work,
excluding those notices that do not pertain to any part of
the Derivative Works; and
(d) If the Work includes a "NOTICE" text file as part of its
distribution, then any Derivative Works that You distribute must
include a readable copy of the attribution notices contained
within such NOTICE file, excluding those notices that do not
pertain to any part of the Derivative Works, in at least one
of the following places: within a NOTICE text file distributed
as part of the Derivative Works; within the Source form or
documentation, if provided along with the Derivative Works; or,
within a display generated by the Derivative Works, if and
wherever such third-party notices normally appear. The contents
of the NOTICE file are for informational purposes only and
do not modify the License. You may add Your own attribution
notices within Derivative Works that You distribute, alongside
or as an addendum to the NOTICE text from the Work, provided
that such additional attribution notices cannot be construed
as modifying the License.
You may add Your own copyright statement to Your modifications and
may provide additional or different license terms and conditions
for use, reproduction, or distribution of Your modifications, or
for any such Derivative Works as a whole, provided Your use,
reproduction, and distribution of the Work otherwise complies with
the conditions stated in this License.
5. Submission of Contributions. Unless You explicitly state otherwise,
any Contribution intentionally submitted for inclusion in the Work
by You to the Licensor shall be under the terms and conditions of
this License, without any additional terms or conditions.
Notwithstanding the above, nothing herein shall supersede or modify
the terms of any separate license agreement you may have executed
with Licensor regarding such Contributions.
6. Trademarks. This License does not grant permission to use the trade
names, trademarks, service marks, or product names of the Licensor,
except as required for reasonable and customary use in describing the
origin of the Work and reproducing the content of the NOTICE file.
7. Disclaimer of Warranty. Unless required by applicable law or
agreed to in writing, Licensor provides the Work (and each
Contributor provides its Contributions) on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
implied, including, without limitation, any warranties or conditions
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
PARTICULAR PURPOSE. You are solely responsible for determining the
appropriateness of using or redistributing the Work and assume any
risks associated with Your exercise of permissions under this License.
8. Limitation of Liability. In no event and under no legal theory,
whether in tort (including negligence), contract, or otherwise,
unless required by applicable law (such as deliberate and grossly
negligent acts) or agreed to in writing, shall any Contributor be
liable to You for damages, including any direct, indirect, special,
incidental, or consequential damages of any character arising as a
result of this License or out of the use or inability to use the
Work (including but not limited to damages for loss of goodwill,
work stoppage, computer failure or malfunction, or any and all
other commercial damages or losses), even if such Contributor
has been advised of the possibility of such damages.
9. Accepting Warranty or Additional Liability. While redistributing
the Work or Derivative Works thereof, You may choose to offer,
and charge a fee for, acceptance of support, warranty, indemnity,
or other liability obligations and/or rights consistent with this
License. However, in accepting such obligations, You may act only
on Your own behalf and on Your sole responsibility, not on behalf
of any other Contributor, and only if You agree to indemnify,
defend, and hold each Contributor harmless for any liability
incurred by, or claims asserted against, such Contributor by reason
of your accepting any such warranty or additional liability.
END OF TERMS AND CONDITIONS
APPENDIX: How to apply the Apache License to your work.
To apply the Apache License to your work, attach the following
boilerplate notice, with the fields enclosed by brackets "[]"
replaced with your own identifying information. (Don't include
the brackets!) The text should be enclosed in the appropriate
comment syntax for the file format. We also recommend that a
file or class name and description of purpose be included on the
same "printed page" as the copyright notice for easier
identification within third-party archives.
Copyright [yyyy] [name of copyright owner]
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
+283
View File
@@ -0,0 +1,283 @@
# Doltgres is Dolt for Postgres!
From the creators of [Dolt](https://www.doltdb.com), the world's first version controlled SQL
database, comes [Doltgres](https://www.doltgres.com), the Postgres-flavored version of Dolt. It's a
SQL database that you can branch and merge, fork and clone, push and pull just like a Git
repository. Connect to your Doltgres server just like any Postgres database to read or modify schema
and data. Version control functionality is exposed in SQL via system tables, functions, and
procedures.
Git versions file, Doltgres versions tables. It's like Git and Postgres had a baby.
# Doltgres is Beta
[Doltgres is now Beta quality](https://dolthub.com/blog/2025-04-16-doltgres-goes-beta/), which means
it's ready for your production use case. There will be bugs and missing features, but [we can fix
most of them in 24 hours](https://www.dolthub.com/blog/2024-05-15-24-hour-bug-fixes/) if you [file
an issue](https://github.com/dolthub/doltgresql/issues).
The wait is over! Now is the time to [try out Doltgres](#getting-started) and let us know what you
think. Import your existing Postgres database into Doltgres with `pg_dump` and `psql`, and let us
know if anything doesn't work.
If you're excited about this project, you can also help speed it along in a few other ways:
- Star this repo
- Create [issues](https://github.com/dolthub/doltgresql/issues) if you find bugs
- Create [issues](https://github.com/dolthub/doltgresql/issues) for missing functionality you want
- Contribute code for features you want (see the [Contribution
Guide](https://github.com/dolthub/doltgresql/blob/main/CONTRIBUTING.md))
- Tell your friends and colleagues
# Full Documentation
Doltgres has a [documentation website](https://doltgres.com/docs) with extensive documentation.
# Installation
To install Doltgres on Linx or Mac based systems run this command in your terminal:
```
sudo bash -c 'curl -L https://github.com/dolthub/doltgresql/releases/latest/download/install.sh | bash'
```
This will download the latest doltgres release and put it in `/usr/local/bin/`, which is probably on
your `$PATH`.
## Windows
Download the latest Microsoft Installer (`.msi` file) in
[releases](https://github.com/dolthub/doltgresql/releases) and run it.
## Docker
Doltgres publishes an official Docker image on every release:
* [dolthub/doltgresql](https://hub.docker.com/r/dolthub/doltgresql)
Run it on your local Docker like this:
```bash
$ docker run -e DOLTGRES_PASSWORD=myPassword -p 5432:5432 dolthub/doltgresql:latest
```
## Building From Source
To produce a binary from source code, run `./scripts/build.sh`.
# Getting Started
1. Run `doltgres`. This will create a `postgres` user and a `postgres` database in the current
directory. The default password will be `password`, just like in Postgres. You can use a
`config.yaml` file or set the `DOLTGRES_DATA_DIR` environment variable to use a different directory
for your databases.
You can change the name and password of the super-user by setting the `DOLTGRES_USER` and
`DOLTGRES_PASSWORD` environment variables before running `doltgres` for the first time.
```bash
$ doltgres
INFO[0000] Server ready. Accepting connections.
```
2. Install Postgres to get the `psql` tool. I used Homebrew to install Postgres on my Mac. This
requires I manually add `/opt/homebrew/opt/postgresql@15/bin` to my path. We only need Postgres in
order to use `psql`, so feel free to skip this step if you already have `psql`, or if you have
another Postgres client you use instead.
```
export PATH="/opt/homebrew/opt/postgresql@15/bin:$PATH"
```
3. Open a new terminal. Connect with the following command: `PGPASSWORD=password psql -h localhost
-U postgres`. This will connect to the `postgres` database with the `postgres` user.
```bash
$ PGPASSWORD=password psql -h localhost
psql (15.4 (Homebrew), server 15.0)
Type "help" for help.
postgres=>
```
4. Create a `getting_started` database. Create the `getting_started` example tables.
```sql
postgres=> create database getting_started;
--
(0 rows)
postgres=> \c getting_started;
psql (15.4 (Homebrew), server 15.0)
You are now connected to database "getting_started" as user "postgres".
getting_started=> create table employees (
id int8,
last_name text,
first_name text,
primary key(id));
--
(0 rows)
getting_started=> create table teams (
id int8,
team_name text,
primary key(id));
--
(0 rows)
getting_started=> create table employees_teams(
team_id int8,
employee_id int8,
primary key(team_id, employee_id),
foreign key (team_id) references teams(id),
foreign key (employee_id) references employees(id));
--
(0 rows)
getting_started=> \d
List of relations
Schema | Name | Type | Owner
--------+-----------------+-------+----------
public | employees | table | postgres
public | employees_teams | table | postgres
public | teams | table | postgres
(3 rows)
```
5. Make a Dolt Commit.
```sql
getting_started=> select * from dolt.status;
table_name | staged | status
------------------------+--------+-----------
public.employees | f | new table
public.employees_teams | f | new table
public.teams | f | new table
(3 rows)
getting_started=> select dolt_add('teams', 'employees', 'employees_teams');
dolt_add
----------
{0}
(1 row)
getting_started=> select * from dolt.status;
table_name | staged | status
-----------------------+--------+-----------
public.employees | t | new table
public.employees_teams | t | new table
public.teams | t | new table
(3 rows)
getting_started=> select dolt_commit('-m', 'Created initial schema');
dolt_commit
------------------------------------
{peqq98e2dl5gscvfvic71e7j6ne34533}
(1 row)
```
6. View the Dolt log.
```sql
getting_started=> select * from dolt.log;
commit_hash | committer | email | date | message
----------------------------------+-----------+--------------------+---------------------+----------------------------
peqq98e2dl5gscvfvic71e7j6ne34533 | postgres | postgres@127.0.0.1 | 2023-11-01 22:08:04 | Created initial schema
in7bk735qa6p6rv6i3s797jjem2pg4ru | timsehn | tim@dolthub.com | 2023-11-01 22:04:03 | Initialize data repository
(2 rows)
```
7. Continue with [Dolt Getting Started](https://dolthub.com/docs/introduction/getting-started/database#insert-some-data)
to test out more Doltgres versioning functionality.
# Limitations and differences from Dolt
- No [Git-style CLI](https://dolthub.com/docs/cli-reference/cli) for version control like in
[Dolt](https://github.com/dolthub/dolt), only a SQL interface.
- Can't push to DoltHub or DoltLab, only custom remotes (such as on the file system or to S3).
- Backup and replication are a work in progress.
- No GSSAPI support.
- No extension support yet.
- Some Postgres syntax, types, functions, and features are not yet implemented. If you encounter a
missing feature you need for your application, please [file an issue to let us
know](https://github.com/dolthub/doltgresql/issues).
# Performance
Dolt is [1.1X slower than MySQL](https://dolthub.com/docs/sql-reference/benchmarks/latency) as
measured by a standard suite of Sysbench tests.
We use these same Sysbench tests to benchmark DoltgreSQL and compare the results to PostgreSQL.
Here are the benchmarks for DoltgreSQL version `0.50.0`. All figures are median latency in
milliseconds.
<!-- START_LATENCY_RESULTS_TABLE -->
| Read Tests | Postgres | Doltgres | Multiple |
| --- | --- | --- | --- |
| covering_index_scan_postgres | 1.89 | 5.28 | 2.8 |
| groupby_scan_postgres | 5.28 | 46.63 | 8.8 |
| index_join_postgres | 1.96 | 10.09 | 5.1 |
| index_join_scan_postgres | 0.67 | 8.9 | 13.3 |
| index_scan_postgres | 17.95 | 130.13 | 7.2 |
| oltp_point_select | 0.14 | 0.52 | 3.7 |
| oltp_read_only | 2.48 | 12.75 | 5.1 |
| select_random_points | 0.21 | 1.12 | 5.3 |
| select_random_ranges | 0.41 | 1.39 | 3.4 |
| table_scan_postgres | 17.95 | 132.49 | 7.4 |
| types_table_scan_postgres | 43.39 | 292.6 | 6.7 |
| reads_mean_multiplier | | | 6.3 |
| Write Tests | Postgres | Doltgres | Multiple |
|------------------------------|----------|----------|----------|
| oltp_delete_insert_postgres | 2.22 | 6.79 | 3.1 |
| oltp_insert | 1.1 | 3.68 | 3.3 |
| oltp_read_write | 4.25 | 20.37 | 4.8 |
| oltp_update_index | 1.12 | 3.55 | 3.2 |
| oltp_update_non_index | 1.12 | 3.43 | 3.1 |
| oltp_write_only | 1.73 | 7.43 | 4.3 |
| types_delete_insert_postgres | 2.3 | 7.04 | 3.1 |
| write_mean_multiplier | | | 3.6 |
| Overall Mean Multiple | 5.2 |
| --------------------- | --- |
<!-- END_LATENCY_RESULTS_TABLE -->
<br/>
# Correctness
Dolt is [100% compatible](https://dolthub.com/docs/sql-reference/benchmarks/correctness) with MySQL
based on a standard suite of correctness tests called `sqllogictest`.
We use these same tests to measure the correctness of DoltgreSQL.
Here are DoltgreSQL's sqllogictest results for version `0.50.0`. Tests that did not run could not
complete due to a timeout earlier in the run.
<!-- START_CORRECTNESS_RESULTS_TABLE -->
| Results | Count |
| -- | -- |
| did not run | 91270 |
| not ok | 411415 |
| ok | 5188604 |
| timeout | 16 |
| Total Tests | 5691305 |
| Correctness Percentage | 91.16721 |
| -- | -- |
<!-- END_CORRECTNESS_RESULTS_TABLE -->
<br/>
# Architecture
Doltgres emulates a Postgres server, including parsing Postgres SQL into an Abstract Syntax Tree (AST). This AST is
converted to a form that can be interpreted by the Dolt engine. Doltgres uses the same SQL engine and storage format as Dolt.
[Dolt has a unique architecture](https://dolthub.com/docs/architecture/architecture) that allows for version control
features at OLTP database performance. Doltgres uses the same architecture.
+7
View File
@@ -0,0 +1,7 @@
# WeHub 来源说明
- 原始项目:`dolthub/doltgresql`
- 原始仓库:https://github.com/dolthub/doltgresql
- 导入方式:上游默认分支的最新快照
- 原作者、版权和许可证信息以原始仓库及本仓库 LICENSE 为准
- 本文件仅用于记录来源,不代表 WeHub 是原项目作者
+16
View File
@@ -0,0 +1,16 @@
# Security Policy
## Supported Versions
By default, the most recent release of DoltgreSQL is the version which is
supported for all security updates. If you need ongoing security
support for an older version of DoltgreSQL, please [contact us](https://www.dolthub.com/contact).
## Reporting a Vulnerability
Any security issues with DoltgreSQL can be reported to [security@dolthub.com](security@dolthub.com).
Reports will be responded to within one business day. The majority of
our team operates on Pacific Time and on a US holiday schedule.
DoltHub does not currently run a security bounty program for DoltgreSQL.
+712
View File
@@ -0,0 +1,712 @@
// Copyright 2026 Dolthub, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package casts
import (
"context"
"fmt"
"io"
"strings"
"github.com/cockroachdb/errors"
"github.com/dolthub/dolt/go/libraries/doltcore/doltdb"
"github.com/dolthub/dolt/go/store/hash"
"github.com/dolthub/dolt/go/store/prolly"
"github.com/dolthub/dolt/go/store/prolly/tree"
"github.com/dolthub/go-mysql-server/sql"
"github.com/dolthub/go-mysql-server/sql/expression"
"github.com/dolthub/go-mysql-server/sql/procedures"
"github.com/dolthub/doltgresql/core/id"
"github.com/dolthub/doltgresql/core/rootobject/objinterface"
pgtypes "github.com/dolthub/doltgresql/server/types"
)
// Collection contains a collection of casts.
type Collection struct {
mapHash hash.Hash // This is cached so that we don't have to calculate the hash every time
underlyingMap prolly.AddressMap
ns tree.NodeStore
}
// CastType is the type of the cast, indicating which contexts it may be called in.
type CastType uint8
const (
CastType_Explicit CastType = 0
CastType_Assignment CastType = 1
CastType_Implicit CastType = 2
)
// builtInCasts contains all casts that are built into the database by default.
var builtInCasts = map[id.Cast]Cast{}
// Cast represents a cast between two types.
type Cast struct {
ID id.Cast
CastType CastType
Function id.Function
BuiltIn pgtypes.TypeCastFunction
UseInOut bool
request CastType // This contains the type of cast that was requested, as we may request an EXPLICIT cast and receive an IMPLICIT (which is valid)
}
var _ objinterface.Collection = (*Collection)(nil)
var _ objinterface.RootObject = Cast{}
// NewCollection returns a new Collection.
func NewCollection(ctx context.Context, underlyingMap prolly.AddressMap, ns tree.NodeStore) (*Collection, error) {
collection := &Collection{
mapHash: underlyingMap.HashOf(),
underlyingMap: underlyingMap,
ns: ns,
}
return collection, nil
}
// GetExplicitCast returns the explicit type cast function that will cast the source type to the target type. Returns
// a Cast with an invalid ID if such a cast is not valid.
func (pgc *Collection) GetExplicitCast(ctx *sql.Context, sourceType *pgtypes.DoltgresType, targetType *pgtypes.DoltgresType) (Cast, error) {
castID := id.NewCast(sourceType.ID, targetType.ID)
c, err := pgc.getCast(ctx, castID, sourceType, targetType, CastType_Explicit)
if err != nil {
return Cast{}, err
}
if c.ID.IsValid() {
return c, nil
}
// We check for the identity and sizing casts after checking the maps, as the identity may be overridden by a user.
if cast := pgc.getSizingOrIdentityCast(castID, sourceType, targetType, CastType_Explicit); cast.ID.IsValid() {
return cast, nil
}
// We then check for a record to composite cast
if recordCast := pgc.getRecordCast(sourceType, targetType, CastType_Explicit); recordCast.ID.IsValid() {
return recordCast, nil
}
// All types have a built-in explicit cast from string types: https://www.postgresql.org/docs/15/sql-createcast.html
if sourceType.TypCategory == pgtypes.TypeCategory_StringTypes {
return Cast{
ID: castID,
CastType: CastType_Explicit,
Function: id.NullFunction,
BuiltIn: nil,
UseInOut: true,
request: CastType_Explicit,
}, nil
} else if targetType.TypCategory == pgtypes.TypeCategory_StringTypes {
// All types have a built-in assignment cast to string types, which we can reference in an explicit cast
return Cast{
ID: castID,
CastType: CastType_Explicit,
Function: id.NullFunction,
BuiltIn: nil,
UseInOut: true,
request: CastType_Explicit,
}, nil
}
// It is always valid to convert from the `unknown` type
if sourceType.ID == pgtypes.Unknown.ID {
return Cast{
ID: castID,
CastType: CastType_Explicit,
Function: id.NullFunction,
BuiltIn: nil,
UseInOut: true,
request: CastType_Explicit,
}, nil
}
return Cast{}, nil
}
// GetAssignmentCast returns the assignment type cast function that will cast the source type to the target type.
// Returns a Cast with an invalid ID if such a cast is not valid.
func (pgc *Collection) GetAssignmentCast(ctx *sql.Context, sourceType *pgtypes.DoltgresType, targetType *pgtypes.DoltgresType) (Cast, error) {
castID := id.NewCast(sourceType.ID, targetType.ID)
c, err := pgc.getCast(ctx, castID, sourceType, targetType, CastType_Assignment)
if err != nil {
return Cast{}, err
}
if c.ID.IsValid() {
if c.CastType == CastType_Explicit {
return Cast{}, nil
}
return c, nil
}
// We check for the identity and sizing casts after checking the maps, as the identity may be overridden by a user.
if cast := pgc.getSizingOrIdentityCast(castID, sourceType, targetType, CastType_Assignment); cast.ID.IsValid() {
return cast, nil
}
// We then check for a record to composite cast
if recordCast := pgc.getRecordCast(sourceType, targetType, CastType_Assignment); recordCast.ID.IsValid() {
return recordCast, nil
}
// All types have a built-in assignment cast to string types: https://www.postgresql.org/docs/15/sql-createcast.html
if targetType.TypCategory == pgtypes.TypeCategory_StringTypes {
return Cast{
ID: castID,
CastType: CastType_Assignment,
Function: id.NullFunction,
BuiltIn: nil,
UseInOut: true,
request: CastType_Assignment,
}, nil
}
// It is always valid to convert from the `unknown` type
if sourceType.ID == pgtypes.Unknown.ID {
return Cast{
ID: castID,
CastType: CastType_Assignment,
Function: id.NullFunction,
BuiltIn: nil,
UseInOut: true,
request: CastType_Assignment,
}, nil
}
return Cast{}, nil
}
// GetImplicitCast returns the implicit type cast function that will cast the source type to the target type. Returns a
// Cast with an invalid ID if such a cast is not valid.
func (pgc *Collection) GetImplicitCast(ctx *sql.Context, sourceType *pgtypes.DoltgresType, targetType *pgtypes.DoltgresType) (Cast, error) {
castID := id.NewCast(sourceType.ID, targetType.ID)
c, err := pgc.getCast(ctx, castID, sourceType, targetType, CastType_Implicit)
if err != nil {
return Cast{}, err
}
if c.ID.IsValid() {
if c.CastType == CastType_Implicit {
return c, nil
}
return Cast{}, nil
}
// We check for the identity and sizing casts after checking the maps, as the identity may be overridden by a user.
if cast := pgc.getSizingOrIdentityCast(castID, sourceType, targetType, CastType_Implicit); cast.ID.IsValid() {
return cast, nil
}
// We then check for a record to composite cast
if recordCast := pgc.getRecordCast(sourceType, targetType, CastType_Implicit); recordCast.ID.IsValid() {
return recordCast, nil
}
// It is always valid to convert from the `unknown` type
if sourceType.ID == pgtypes.Unknown.ID {
return Cast{
ID: castID,
CastType: CastType_Implicit,
Function: id.NullFunction,
BuiltIn: nil,
UseInOut: true,
request: CastType_Implicit,
}, nil
}
return Cast{}, nil
}
// getCast is used by each individual Get function to handle the actual fetching of the cast.
func (pgc *Collection) getCast(ctx context.Context, castID id.Cast, sourceType *pgtypes.DoltgresType, targetType *pgtypes.DoltgresType, castType CastType) (Cast, error) {
if c, ok := builtInCasts[castID]; ok {
return c, nil
}
h, err := pgc.underlyingMap.Get(ctx, string(castID))
if err != nil {
return Cast{}, err
}
if h.IsEmpty() {
// If there isn't a direct mapping, then we need to check if the types are array variants.
// As long as the base types are convertable, the array variants are also convertable.
if sourceType != nil && targetType != nil && sourceType.IsArrayType() && targetType.IsArrayType() {
sqlCtx, ok := ctx.(*sql.Context)
if !ok {
return Cast{}, fmt.Errorf("non *sql.Context provided to Collection.getCast()")
}
fromBaseType := sourceType.ArrayBaseType()
toBaseType := targetType.ArrayBaseType()
var baseCast Cast
switch castType {
case CastType_Explicit:
baseCast, err = pgc.GetExplicitCast(sqlCtx, fromBaseType, toBaseType)
if err != nil {
return Cast{}, err
}
case CastType_Assignment:
baseCast, err = pgc.GetAssignmentCast(sqlCtx, fromBaseType, toBaseType)
if err != nil {
return Cast{}, err
}
case CastType_Implicit:
baseCast, err = pgc.GetImplicitCast(sqlCtx, fromBaseType, toBaseType)
if err != nil {
return Cast{}, err
}
}
if baseCast.ID.IsValid() {
// We use a closure that can unwrap the slice, since conversion functions expect a singular non-nil value
evalFunc := func(ctx *sql.Context, vals any, sourceType *pgtypes.DoltgresType, targetType *pgtypes.DoltgresType) (any, error) {
var err error
oldVals := vals.([]any)
newVals := make([]any, len(oldVals))
for i, oldVal := range oldVals {
if oldVal == nil {
continue
}
// Some errors are optional depending on the context, so we'll still process all values even
// after an error is received.
var nErr error
sourceBaseType := sourceType.ArrayBaseType()
targetBaseType := targetType.ArrayBaseType()
newVals[i], nErr = baseCast.Eval(ctx, oldVal, sourceBaseType, targetBaseType)
if nErr != nil && err == nil {
err = nErr
}
}
return newVals, err
}
return Cast{
ID: castID,
CastType: castType,
Function: id.NullFunction,
BuiltIn: evalFunc,
UseInOut: false,
request: castType,
}, nil
}
}
return Cast{}, nil
}
data, err := pgc.ns.ReadBytes(ctx, h)
if err != nil {
return Cast{}, err
}
c, err := DeserializeCast(ctx, data)
if err != nil {
return Cast{}, err
}
c.request = castType
return c, nil
}
// getSizingOrIdentityCast returns an identity cast if the two types are exactly the same, and a sizing cast if they
// only differ in their atttypmod values. Returns a Cast with an invalid ID if no cast is matched. This mirrors the
// behavior as described in:
// https://www.postgresql.org/docs/15/typeconv-query.html
func (pgc *Collection) getSizingOrIdentityCast(castID id.Cast, sourceType *pgtypes.DoltgresType, targetType *pgtypes.DoltgresType, castType CastType) Cast {
// If we receive different types, then we can return immediately
if sourceType.ID != targetType.ID {
return Cast{}
}
// If we have different atttypmod values, then we need to do a sizing cast only if one exists
// Otherwise, then we simply use the identity cast
// TODO: We don't have any sizing cast functions implemented, so for now we'll approximate using output to input.
// We can use the query below to find all implemented sizing cast functions. It's also detailed in the link above.
// Lastly, not all sizing functions accept a boolean, but for those that do, we need to see whether true is
// used for explicit casts, or whether true is used for implicit casts.
// SELECT
// format_type(c.castsource, NULL) AS source,
// format_type(c.casttarget, NULL) AS target,
// p.oid::regprocedure AS func
// FROM pg_cast c JOIN pg_proc p ON p.oid = c.castfunc WHERE c.castsource = c.casttarget ORDER BY 1,2;
useInOut := sourceType.GetAttTypMod() != targetType.GetAttTypMod()
return Cast{
ID: castID,
CastType: castType,
Function: id.NullFunction,
BuiltIn: nil,
UseInOut: useInOut,
request: castType,
}
}
// getRecordCast handles casting from a record type to a composite type (if applicable). Returns a Cast with an invalid
// ID if not applicable.
func (pgc *Collection) getRecordCast(sourceType *pgtypes.DoltgresType, targetType *pgtypes.DoltgresType, castType CastType) Cast {
// TODO: does casting to a record type always work for any composite type?
// https://www.postgresql.org/docs/15/sql-expressions.html#SQL-SYNTAX-ROW-CONSTRUCTORS seems to suggest so
// Also not sure if we should use the passthrough, or if we always default to implicit, assignment, or explicit
if sourceType.IsRecordType() && targetType.IsCompositeType() {
// When casting to a composite type, then we must match the arity and have valid casts for every position.
if targetType.IsRecordType() {
return Cast{
ID: id.NewCast(sourceType.ID, targetType.ID),
CastType: castType,
Function: id.NullFunction,
BuiltIn: nil,
UseInOut: false,
request: castType,
}
} else {
evalFunc := func(ctx *sql.Context, val any, sourceType *pgtypes.DoltgresType, targetType *pgtypes.DoltgresType) (_ any, err error) {
vals, ok := val.([]pgtypes.RecordValue)
if !ok {
return nil, errors.New("casting input error from record type")
}
if len(targetType.CompositeAttrs) != len(vals) {
// TODO: these should go in DETAIL depending on the size
// Input has too few columns.
// Input has too many columns.
return nil, errors.Errorf("cannot cast type %s to %s", sourceType.Name(), targetType.Name())
}
outputVals := make([]pgtypes.RecordValue, len(vals))
for i := range vals {
valType, ok := vals[i].Type.(*pgtypes.DoltgresType)
if !ok {
return nil, errors.New("cannot cast record containing GMS type")
}
outputType := targetType.CompositeAttrs[i].Type
outputVals[i].Type = outputType
if vals[i].Value != nil {
var positionCast Cast
switch castType {
case CastType_Explicit:
positionCast, err = pgc.GetExplicitCast(ctx, valType, outputType)
if err != nil {
return nil, err
}
case CastType_Assignment:
positionCast, err = pgc.GetAssignmentCast(ctx, valType, outputType)
if err != nil {
return nil, err
}
case CastType_Implicit:
positionCast, err = pgc.GetImplicitCast(ctx, valType, outputType)
if err != nil {
return nil, err
}
}
if !positionCast.ID.IsValid() {
// TODO: this should be the DETAIL, with the actual error being "cannot cast type <FROM_TYPE> to <TO_TYPE>"
return nil, errors.Errorf("Cannot cast type %s to %s in column %d", valType.Name(), outputType.Name(), i+1)
}
outputVals[i].Value, err = positionCast.Eval(ctx, vals[i].Value, valType, outputType)
if err != nil {
return nil, err
}
}
}
return outputVals, nil
}
return Cast{
ID: id.NewCast(sourceType.ID, targetType.ID),
CastType: castType,
Function: id.NullFunction,
BuiltIn: evalFunc,
UseInOut: false,
request: castType,
}
}
}
return Cast{}
}
// HasCast returns whether the given cast exists.
func (pgc *Collection) HasCast(ctx context.Context, castID id.Cast) bool {
if _, ok := builtInCasts[castID]; ok {
return true
}
ok, err := pgc.underlyingMap.Has(ctx, string(castID))
if err == nil && ok {
return true
}
return false
}
// AddCast adds a new cast.
func (pgc *Collection) AddCast(ctx context.Context, cast Cast) error {
// First we'll check to see if it exists
if pgc.HasCast(ctx, cast.ID) {
return errors.Errorf(`cast from type %s to type %s already exists`,
cast.ID.SourceType().TypeName(), cast.ID.TargetType().TypeName())
}
if cast.BuiltIn != nil {
return errors.Errorf(`cannot create a built-in cast from type %s to type %s`,
cast.ID.SourceType().TypeName(), cast.ID.TargetType().TypeName())
}
// Now we'll add the cast to our map
data, err := cast.Serialize(ctx)
if err != nil {
return err
}
h, err := pgc.ns.WriteBytes(ctx, data)
if err != nil {
return err
}
mapEditor := pgc.underlyingMap.Editor()
if err = mapEditor.Add(ctx, string(cast.ID), h); err != nil {
return err
}
newMap, err := mapEditor.Flush(ctx)
if err != nil {
return err
}
pgc.underlyingMap = newMap
pgc.mapHash = pgc.underlyingMap.HashOf()
return nil
}
// DropCast drops an existing cast.
func (pgc *Collection) DropCast(ctx context.Context, castIDs ...id.Cast) error {
if len(castIDs) == 0 {
return nil
}
// Check that each name exists before performing any deletions
for _, castID := range castIDs {
if _, ok := builtInCasts[castID]; ok {
return errors.Errorf(`cannot delete built-in cast from type %s to type %s`,
castID.SourceType().TypeName(), castID.TargetType().TypeName())
}
if ok, err := pgc.underlyingMap.Has(ctx, string(castID)); err != nil {
return err
} else if !ok {
return errors.Errorf(`cast from type %s to type %s does not exist`,
castID.SourceType().TypeName(), castID.TargetType().TypeName())
}
}
// Now we'll remove the casts from the map
mapEditor := pgc.underlyingMap.Editor()
for _, castID := range castIDs {
err := mapEditor.Delete(ctx, string(castID))
if err != nil {
return err
}
}
newMap, err := mapEditor.Flush(ctx)
if err != nil {
return err
}
pgc.underlyingMap = newMap
pgc.mapHash = pgc.underlyingMap.HashOf()
return nil
}
// resolveName returns the fully resolved name of the given cast. Returns an error if the name is ambiguous.
func (pgc *Collection) resolveName(ctx context.Context, schemaName string, formattedName string) (id.Cast, error) {
if len(formattedName) == 0 {
return id.NullCast, nil
}
// Check for an exact match
fullID := pgc.tableNameToID(schemaName, formattedName)
if pgc.HasCast(ctx, fullID) {
return fullID, nil
}
// Otherwise we'll iterate over all the names
var resolvedID id.Cast
err := pgc.IterateCasts(ctx, func(c Cast) (stop bool, err error) {
if !strings.EqualFold(string(c.ID), string(fullID)) {
return false, nil
}
// The above matches, so this counts as a match
if resolvedID.IsValid() {
castTableName := CastIDToTableName(c.ID)
resolvedTableName := CastIDToTableName(resolvedID)
return true, fmt.Errorf("`%s` is ambiguous, matches `%s` and `%s`",
formattedName, castTableName.String(), resolvedTableName.String())
}
resolvedID = c.ID
return false, nil
})
return resolvedID, err
}
// IterateCasts iterates over all casts in the collection.
func (pgc *Collection) IterateCasts(ctx context.Context, callback func(c Cast) (stop bool, err error)) error {
for _, cast := range builtInCasts {
stop, err := callback(cast)
if err != nil {
return err
} else if stop {
return nil
}
}
return pgc.underlyingMap.IterAll(ctx, func(_ string, v hash.Hash) error {
data, err := pgc.ns.ReadBytes(ctx, v)
if err != nil {
return err
}
c, err := DeserializeCast(ctx, data)
if err != nil {
return err
}
stop, err := callback(c)
if err != nil {
return err
} else if stop {
return io.EOF
} else {
return nil
}
})
}
// Clone returns a new *Collection with the same contents as the original.
func (pgc *Collection) Clone(ctx context.Context) *Collection {
return &Collection{
mapHash: pgc.mapHash,
underlyingMap: pgc.underlyingMap,
ns: pgc.ns,
}
}
// Map writes any cached sequences to the underlying map, and then returns the underlying map.
func (pgc *Collection) Map(ctx context.Context) (prolly.AddressMap, error) {
return pgc.underlyingMap, nil
}
// tableNameToID returns the ID that was encoded via the Name() call, as the returned TableName contains additional
// information (which this is able to process).
func (pgc *Collection) tableNameToID(schemaName string, formattedName string) id.Cast {
sections := strings.Split(strings.TrimSuffix(strings.TrimPrefix(formattedName, "("), ")"), ")|(")
if len(sections) != 4 {
return id.NullCast
}
return id.NewCast(id.NewType(sections[0], sections[1]), id.NewType(sections[2], sections[3]))
}
// GetID implements the interface objinterface.RootObject.
func (cast Cast) GetID() id.Id {
return cast.ID.AsId()
}
// DiffersFrom returns true when the hash that is associated with the underlying map for this collection is different
// from the hash in the given root.
func (pgc *Collection) DiffersFrom(ctx context.Context, root objinterface.RootValue) bool {
hashOnGivenRoot, err := pgc.LoadCollectionHash(ctx, root)
if err != nil {
return true
}
if pgc.mapHash.Equal(hashOnGivenRoot) {
return false
}
// An empty map should match an uninitialized collection on the root
count, err := pgc.underlyingMap.Count()
if err == nil && count == 0 && hashOnGivenRoot.IsEmpty() {
return false
}
return true
}
// GetRootObjectID implements the interface objinterface.RootObject.
func (cast Cast) GetRootObjectID() objinterface.RootObjectID {
return objinterface.RootObjectID_Casts
}
// HashOf implements the interface objinterface.RootObject.
func (cast Cast) HashOf(ctx context.Context) (hash.Hash, error) {
data, err := cast.Serialize(ctx)
if err != nil {
return hash.Hash{}, err
}
return hash.Of(data), nil
}
// Name implements the interface rootobject.RootObject.
func (cast Cast) Name() doltdb.TableName {
return CastIDToTableName(cast.ID)
}
// Eval evaluates the cast against the given value.
func (cast Cast) Eval(ctx *sql.Context, val any, sourceType *pgtypes.DoltgresType, targetType *pgtypes.DoltgresType) (any, error) {
if cast.UseInOut {
if val == nil {
return nil, nil
}
output, err := sourceType.IoOutput(ctx, val)
if err != nil {
return nil, err
}
return targetType.IoInput(ctx, output)
}
if cast.BuiltIn != nil {
// It may not be strictly true that all built-in casts are STRICT, but it seems true so we'll hold the assumption
if val == nil {
return nil, nil
}
return cast.BuiltIn(ctx, val, sourceType, targetType)
}
if cast.Function != id.NullFunction {
castFunc, ok := functionProvider.Function(ctx, cast.Function.SchemaName(), cast.Function.FunctionName())
if !ok {
return nil, sql.ErrFunctionNotFound.New(cast.Function.FunctionName())
}
var exprs []sql.Expression
switch cast.Function.ParameterCount() {
case 1:
exprs = []sql.Expression{
expression.NewLiteral(val, sourceType),
}
case 2:
exprs = []sql.Expression{
expression.NewLiteral(val, sourceType),
expression.NewLiteral(targetType.GetAttTypMod(), pgtypes.Int32),
}
case 3:
exprs = []sql.Expression{
expression.NewLiteral(val, sourceType),
expression.NewLiteral(targetType.GetAttTypMod(), pgtypes.Int32),
expression.NewLiteral(cast.request == CastType_Explicit, pgtypes.Bool),
}
default:
return nil, errors.New("invalid parameter count for cast function") // TODO: figure out the actual error
}
castFuncInstance, err := castFunc.NewInstance(ctx, exprs)
if err != nil {
return nil, err
}
if val == nil {
if getIsStrictFromFunction(castFuncInstance) {
return nil, nil
}
}
if setRunner, ok := castFuncInstance.(procedures.InterpreterExpr); ok {
runner, err := getRunnerFromContext(ctx)
if err != nil {
return nil, err
}
castFuncInstance = setRunner.SetStatementRunner(ctx, runner)
}
return castFuncInstance.Eval(ctx, nil)
}
// In this case, the values are binary-coercible, but we still check as we may deviate from Postgres for some reason
if _, _, err := targetType.Convert(ctx, val); err != nil {
return nil, errors.Errorf(`cast from type %s to type %s is mislabeled as binary-coercible`,
cast.ID.SourceType().TypeName(), cast.ID.TargetType().TypeName())
}
return val, nil
}
// CastIDToTableName returns the ID in a format that's better for user consumption.
func CastIDToTableName(castID id.Cast) doltdb.TableName {
name := fmt.Sprintf(`(%s)|(%s)|(%s)|(%s)`,
castID.SourceType().SchemaName(),
castID.SourceType().TypeName(),
castID.TargetType().SchemaName(),
castID.TargetType().TypeName())
return doltdb.TableName{
Name: name,
Schema: "",
}
}
// functionProvider is set by init and is used to avoid import cycles
var functionProvider sql.FunctionProvider
// getRunnerFromContext is set by init and is used to avoid import cycles
var getRunnerFromContext func(ctx *sql.Context) (sql.StatementRunner, error)
// getIsStrictFromFunction is set by init and is used to avoid import cycles
var getIsStrictFromFunction func(f sql.Expression) bool
+133
View File
@@ -0,0 +1,133 @@
// Copyright 2026 Dolthub, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package casts
import (
"context"
"github.com/cockroachdb/errors"
"github.com/dolthub/dolt/go/libraries/doltcore/doltdb"
"github.com/dolthub/dolt/go/libraries/doltcore/merge"
"github.com/dolthub/dolt/go/store/hash"
"github.com/dolthub/dolt/go/store/prolly"
"github.com/dolthub/dolt/go/store/prolly/tree"
"github.com/dolthub/doltgresql/core/id"
"github.com/dolthub/doltgresql/core/rootobject/objinterface"
"github.com/dolthub/doltgresql/flatbuffers/gen/serial"
)
// storage is used to read from and write to the root.
var storage = objinterface.RootObjectSerializer{
Bytes: (*serial.RootValue).CastsBytes,
RootValueAdd: serial.RootValueAddCasts,
}
// HandleMerge implements the interface objinterface.Collection.
func (*Collection) HandleMerge(ctx context.Context, mro merge.MergeRootObject) (doltdb.RootObject, *merge.MergeStats, error) {
ourCast := mro.OurRootObj.(Cast)
theirCast := mro.TheirRootObj.(Cast)
// Ensure that they have the same identifier
if ourCast.ID != theirCast.ID {
return nil, nil, errors.Newf("attempted to merge different casts: `%s` and `%s`",
ourCast.Name().String(), theirCast.Name().String())
}
ourHash, err := ourCast.HashOf(ctx)
if err != nil {
return nil, nil, err
}
theirHash, err := theirCast.HashOf(ctx)
if err != nil {
return nil, nil, err
}
if ourHash.Equal(theirHash) {
return mro.OurRootObj, &merge.MergeStats{
Operation: merge.TableUnmodified,
Adds: 0,
Deletes: 0,
Modifications: 0,
DataConflicts: 0,
SchemaConflicts: 0,
ConstraintViolations: 0,
}, nil
}
// TODO: figure out a decent merge strategy
return nil, nil, errors.Errorf("unable to merge `%s`", theirCast.Name().String())
}
// LoadCollection implements the interface objinterface.Collection.
func (*Collection) LoadCollection(ctx context.Context, root objinterface.RootValue) (objinterface.Collection, error) {
return LoadCasts(ctx, root)
}
// LoadCollectionHash implements the interface objinterface.Collection.
func (*Collection) LoadCollectionHash(ctx context.Context, root objinterface.RootValue) (hash.Hash, error) {
m, ok, err := storage.GetProllyMap(ctx, root)
if err != nil || !ok {
return hash.Hash{}, err
}
return m.HashOf(), nil
}
// LoadCasts loads the casts collection from the given root.
func LoadCasts(ctx context.Context, root objinterface.RootValue) (*Collection, error) {
m, ok, err := storage.GetProllyMap(ctx, root)
if err != nil {
return nil, err
}
if !ok {
m, err = prolly.NewEmptyAddressMap(root.NodeStore())
if err != nil {
return nil, err
}
}
return NewCollection(ctx, m, root.NodeStore())
}
// ResolveNameFromObjects implements the interface objinterface.Collection.
func (*Collection) ResolveNameFromObjects(ctx context.Context, name doltdb.TableName, rootObjects []objinterface.RootObject) (doltdb.TableName, id.Id, error) {
// There are root objects to search through, so we'll create a temporary store
ns := tree.NewTestNodeStore()
addressMap, err := prolly.NewEmptyAddressMap(ns)
if err != nil {
return doltdb.TableName{}, id.Null, err
}
tempCollection, err := NewCollection(ctx, addressMap, ns)
if err != nil {
return doltdb.TableName{}, id.Null, err
}
for _, rootObject := range rootObjects {
if c, ok := rootObject.(Cast); ok {
if err = tempCollection.AddCast(ctx, c); err != nil {
return doltdb.TableName{}, id.Null, err
}
}
}
return tempCollection.ResolveName(ctx, name)
}
// Serializer implements the interface objinterface.Collection.
func (*Collection) Serializer() objinterface.RootObjectSerializer {
return storage
}
// UpdateRoot implements the interface objinterface.Collection.
func (pgc *Collection) UpdateRoot(ctx context.Context, root objinterface.RootValue) (objinterface.RootValue, error) {
m, err := pgc.Map(ctx)
if err != nil {
return nil, err
}
return storage.WriteProllyMap(ctx, root, m)
}
+33
View File
@@ -0,0 +1,33 @@
// Copyright 2026 Dolthub, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package casts
import (
"github.com/dolthub/go-mysql-server/sql"
"github.com/dolthub/doltgresql/core/id"
)
// Init initializes this package.
func Init(
getRunnerFromCtx func(ctx *sql.Context) (sql.StatementRunner, error),
getIsStrictFromFunc func(f sql.Expression) bool,
provider sql.FunctionProvider,
) map[id.Cast]Cast {
functionProvider = provider
getRunnerFromContext = getRunnerFromCtx
getIsStrictFromFunction = getIsStrictFromFunc
return builtInCasts
}
+149
View File
@@ -0,0 +1,149 @@
// Copyright 2026 Dolthub, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package casts
import (
"context"
"io"
"github.com/cockroachdb/errors"
"github.com/dolthub/dolt/go/libraries/doltcore/doltdb"
"github.com/dolthub/dolt/go/store/hash"
"github.com/dolthub/doltgresql/core/id"
"github.com/dolthub/doltgresql/core/rootobject/objinterface"
pgtypes "github.com/dolthub/doltgresql/server/types"
)
// DeserializeRootObject implements the interface objinterface.Collection.
func (pgc *Collection) DeserializeRootObject(ctx context.Context, data []byte) (objinterface.RootObject, error) {
return DeserializeCast(ctx, data)
}
// DiffRootObjects implements the interface objinterface.Collection.
func (pgc *Collection) DiffRootObjects(ctx context.Context, fromHash string, ours objinterface.RootObject, theirs objinterface.RootObject, ancestor objinterface.RootObject) ([]objinterface.RootObjectDiff, objinterface.RootObject, error) {
return nil, nil, errors.New("cast conflict detection has not yet been implemented")
}
// DropRootObject implements the interface objinterface.Collection.
func (pgc *Collection) DropRootObject(ctx context.Context, identifier id.Id) error {
if identifier.Section() != id.Section_Cast {
return errors.Errorf(`cast %s does not exist`, identifier.String())
}
return pgc.DropCast(ctx, id.Cast(identifier))
}
// GetFieldType implements the interface objinterface.Collection.
func (pgc *Collection) GetFieldType(ctx context.Context, fieldName string) *pgtypes.DoltgresType {
return nil
}
// GetID implements the interface objinterface.Collection.
func (pgc *Collection) GetID() objinterface.RootObjectID {
return objinterface.RootObjectID_Casts
}
// GetRootObject implements the interface objinterface.Collection.
func (pgc *Collection) GetRootObject(ctx context.Context, identifier id.Id) (objinterface.RootObject, bool, error) {
if identifier.Section() != id.Section_Cast {
return nil, false, nil
}
c, err := pgc.getCast(ctx, id.Cast(identifier), nil, nil, CastType_Explicit)
return c, err == nil && c.ID.IsValid(), err
}
// HasRootObject implements the interface objinterface.Collection.
func (pgc *Collection) HasRootObject(ctx context.Context, identifier id.Id) (bool, error) {
if identifier.Section() != id.Section_Cast {
return false, nil
}
return pgc.HasCast(ctx, id.Cast(identifier)), nil
}
// IDToTableName implements the interface objinterface.Collection.
func (pgc *Collection) IDToTableName(identifier id.Id) doltdb.TableName {
if identifier.Section() != id.Section_Cast {
return doltdb.TableName{}
}
return CastIDToTableName(id.Cast(identifier))
}
// IterAll implements the interface objinterface.Collection.
func (pgc *Collection) IterAll(ctx context.Context, callback func(rootObj objinterface.RootObject) (stop bool, err error)) error {
return pgc.IterateCasts(ctx, func(c Cast) (stop bool, err error) {
return callback(c)
})
}
// IterIDs implements the interface objinterface.Collection.
func (pgc *Collection) IterIDs(ctx context.Context, callback func(identifier id.Id) (stop bool, err error)) error {
err := pgc.underlyingMap.IterAll(ctx, func(k string, _ hash.Hash) error {
stop, err := callback(id.Id(k))
if err != nil {
return err
} else if stop {
return io.EOF
} else {
return nil
}
})
return err
}
// PutRootObject implements the interface objinterface.Collection.
func (pgc *Collection) PutRootObject(ctx context.Context, rootObj objinterface.RootObject) error {
c, ok := rootObj.(Cast)
if !ok {
return errors.Newf("invalid cast root object: %T", rootObj)
}
return pgc.AddCast(ctx, c)
}
// RenameRootObject implements the interface objinterface.Collection.
func (pgc *Collection) RenameRootObject(ctx context.Context, oldName id.Id, newName id.Id) error {
if !oldName.IsValid() || !newName.IsValid() || oldName.Section() != newName.Section() || oldName.Section() != id.Section_Cast {
return errors.New("cannot rename cast due to invalid id")
}
oldCastName := id.Cast(oldName)
newCastName := id.Cast(newName)
c, err := pgc.getCast(ctx, oldCastName, nil, nil, CastType_Explicit)
if err != nil {
return err
}
if err = pgc.DropCast(ctx, newCastName); err != nil {
return err
}
c.ID = newCastName
return pgc.AddCast(ctx, c)
}
// ResolveName implements the interface objinterface.Collection.
func (pgc *Collection) ResolveName(ctx context.Context, name doltdb.TableName) (doltdb.TableName, id.Id, error) {
rawID, err := pgc.resolveName(ctx, name.Schema, name.Name)
if err != nil || !rawID.IsValid() {
return doltdb.TableName{}, id.Null, err
}
return CastIDToTableName(rawID), rawID.AsId(), nil
}
// TableNameToID implements the interface objinterface.Collection.
func (pgc *Collection) TableNameToID(name doltdb.TableName) id.Id {
return pgc.tableNameToID(name.Schema, name.Name).AsId()
}
// UpdateField implements the interface objinterface.Collection.
func (pgc *Collection) UpdateField(ctx context.Context, rootObject objinterface.RootObject, fieldName string, newValue any) (objinterface.RootObject, error) {
return nil, errors.New("updating through the conflicts table for this object type is not yet supported")
}
+67
View File
@@ -0,0 +1,67 @@
// Copyright 2026 Dolthub, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package casts
import (
"context"
"github.com/cockroachdb/errors"
"github.com/dolthub/doltgresql/core/id"
"github.com/dolthub/doltgresql/utils"
)
// Serialize returns the Cast as a byte slice. If the Cast is invalid, then this returns a nil slice.
func (cast Cast) Serialize(ctx context.Context) ([]byte, error) {
if !cast.ID.IsValid() {
return nil, nil
}
// Initialize the writer and version
writer := utils.NewWriter(256)
writer.VariableUint(0) // Version
// Write the cast data
writer.Id(cast.ID.AsId())
writer.Uint8(uint8(cast.CastType))
writer.Id(cast.Function.AsId())
writer.Bool(cast.UseInOut)
// Returns the data
return writer.Data(), nil
}
// DeserializeCast returns the Cast that was serialized in the byte slice. Returns an empty Cast (invalid ID) if data is
// nil or empty.
func DeserializeCast(ctx context.Context, data []byte) (Cast, error) {
if len(data) == 0 {
return Cast{}, nil
}
reader := utils.NewReader(data)
version := reader.VariableUint()
if version != 0 {
return Cast{}, errors.Errorf("version %d of casts is not supported, please upgrade the server", version)
}
// Read from the reader
t := Cast{}
t.ID = id.Cast(reader.Id())
t.CastType = CastType(reader.Uint8())
t.Function = id.Function(reader.Id())
t.UseInOut = reader.Bool()
if !reader.IsEmpty() {
return Cast{}, errors.Errorf("extra data found while deserializing a cast")
}
// Return the deserialized object
return t, nil
}
+401
View File
@@ -0,0 +1,401 @@
// Copyright 2025 Dolthub, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package conflicts
import (
"context"
"maps"
"slices"
"github.com/cockroachdb/errors"
"github.com/dolthub/dolt/go/libraries/doltcore/doltdb"
"github.com/dolthub/dolt/go/store/hash"
"github.com/dolthub/dolt/go/store/prolly"
"github.com/dolthub/dolt/go/store/prolly/tree"
"github.com/dolthub/go-mysql-server/sql"
"github.com/dolthub/doltgresql/core/id"
"github.com/dolthub/doltgresql/core/rootobject/objinterface"
pgtypes "github.com/dolthub/doltgresql/server/types"
)
// Collection contains a collection of conflicts.
type Collection struct {
accessCache map[id.Id]Conflict // This cache is used for general access when you know the exact ID
nameCache map[doltdb.TableName]id.Id // This cache is used for ID resolution, since exact table names are always used
idCache []id.Id // This cache simply contains the name of every root object
mapHash hash.Hash // This is cached so that we don't have to calculate the hash every time
underlyingMap prolly.AddressMap
ns tree.NodeStore
}
// Conflict represents a root object conflict.
type Conflict struct {
ID id.Id
FromHash string
RootObjectID objinterface.RootObjectID
Ours objinterface.RootObject
Theirs objinterface.RootObject
Ancestor objinterface.RootObject
}
var _ objinterface.Collection = (*Collection)(nil)
var _ objinterface.RootObject = Conflict{}
var _ objinterface.Conflict = Conflict{}
var _ doltdb.ConflictRootObject = Conflict{}
// NewCollection returns a new Collection.
func NewCollection(ctx context.Context, underlyingMap prolly.AddressMap, ns tree.NodeStore) (*Collection, error) {
collection := &Collection{
accessCache: make(map[id.Id]Conflict),
nameCache: make(map[doltdb.TableName]id.Id),
idCache: nil,
mapHash: hash.Hash{},
underlyingMap: underlyingMap,
ns: ns,
}
return collection, collection.reloadCaches(ctx)
}
// GetConflict returns the conflict with the given ID. Returns a conflict with an invalid ID if it cannot be found
// (Conflict.ID.IsValid() == false).
func (pgc *Collection) GetConflict(ctx context.Context, conflictID id.Id) (Conflict, error) {
if conflict, ok := pgc.accessCache[conflictID]; ok {
return conflict, nil
}
return Conflict{}, nil
}
// HasConflict returns whether the conflict is present.
func (pgc *Collection) HasConflict(ctx context.Context, conflictID id.Id) bool {
_, ok := pgc.accessCache[conflictID]
return ok
}
// AddConflict adds a new conflict.
func (pgc *Collection) AddConflict(ctx context.Context, conflict Conflict) error {
// First we'll check to see if it exists
if _, ok := pgc.accessCache[conflict.ID]; ok {
return errors.Errorf(`"%s" already has a conflict`, conflict.Ours.Name())
}
// Now we'll add the conflict to our map
data, err := conflict.Serialize(ctx)
if err != nil {
return err
}
h, err := pgc.ns.WriteBytes(ctx, data)
if err != nil {
return err
}
mapEditor := pgc.underlyingMap.Editor()
if err = mapEditor.Add(ctx, string(conflict.ID), h); err != nil {
return err
}
newMap, err := mapEditor.Flush(ctx)
if err != nil {
return err
}
pgc.underlyingMap = newMap
pgc.mapHash = pgc.underlyingMap.HashOf()
return pgc.reloadCaches(ctx)
}
// DropConflict drops an existing conflict.
func (pgc *Collection) DropConflict(ctx context.Context, conflictIDs ...id.Id) error {
if len(conflictIDs) == 0 {
return nil
}
// Check that each name exists before performing any deletions
for _, conflictID := range conflictIDs {
if _, ok := pgc.accessCache[conflictID]; !ok {
return errors.Errorf(`conflict %s does not exist`, conflictID.String())
}
}
// Now we'll remove the conflicts from the map
mapEditor := pgc.underlyingMap.Editor()
for _, conflictID := range conflictIDs {
err := mapEditor.Delete(ctx, string(conflictID))
if err != nil {
return err
}
}
newMap, err := mapEditor.Flush(ctx)
if err != nil {
return err
}
pgc.underlyingMap = newMap
pgc.mapHash = pgc.underlyingMap.HashOf()
return pgc.reloadCaches(ctx)
}
// iterateIDs iterates over all conflict IDs in the collection.
func (pgc *Collection) iterateIDs(ctx context.Context, callback func(conflictID id.Id) (stop bool, err error)) error {
for _, conflictID := range pgc.idCache {
stop, err := callback(conflictID)
if err != nil {
return err
} else if stop {
return nil
}
}
return nil
}
// IterateConflicts iterates over all conflicts in the collection.
func (pgc *Collection) IterateConflicts(ctx context.Context, callback func(conflict Conflict) (stop bool, err error)) error {
for _, conflictID := range pgc.idCache {
stop, err := callback(pgc.accessCache[conflictID])
if err != nil {
return err
} else if stop {
return nil
}
}
return nil
}
// Clone returns a new *Collection with the same contents as the original.
func (pgc *Collection) Clone(ctx context.Context) *Collection {
return &Collection{
accessCache: maps.Clone(pgc.accessCache),
nameCache: maps.Clone(pgc.nameCache),
idCache: slices.Clone(pgc.idCache),
underlyingMap: pgc.underlyingMap,
mapHash: pgc.mapHash,
ns: pgc.ns,
}
}
// Map writes any cached sequences to the underlying map, and then returns the underlying map.
func (pgc *Collection) Map(ctx context.Context) (prolly.AddressMap, error) {
return pgc.underlyingMap, nil
}
// DiffersFrom returns true when the hash that is associated with the underlying map for this collection is different
// from the hash in the given root.
func (pgc *Collection) DiffersFrom(ctx context.Context, root objinterface.RootValue) bool {
hashOnGivenRoot, err := pgc.LoadCollectionHash(ctx, root)
if err != nil {
return true
}
if pgc.mapHash.Equal(hashOnGivenRoot) {
return false
}
// An empty map should match an uninitialized collection on the root
count, err := pgc.underlyingMap.Count()
if err == nil && count == 0 && hashOnGivenRoot.IsEmpty() {
return false
}
return true
}
// reloadCaches writes the underlying map's contents to the caches.
func (pgc *Collection) reloadCaches(ctx context.Context) error {
count, err := pgc.underlyingMap.Count()
if err != nil {
return err
}
clear(pgc.accessCache)
clear(pgc.nameCache)
pgc.mapHash = pgc.underlyingMap.HashOf()
pgc.idCache = make([]id.Id, 0, count)
return pgc.underlyingMap.IterAll(ctx, func(_ string, h hash.Hash) error {
if h.IsEmpty() {
return nil
}
data, err := pgc.ns.ReadBytes(ctx, h)
if err != nil {
return err
}
conflict, err := DeserializeConflict(ctx, data)
if err != nil {
return err
}
pgc.accessCache[conflict.ID] = conflict
pgc.nameCache[conflict.Name()] = conflict.ID
pgc.idCache = append(pgc.idCache, conflict.ID)
return nil
})
}
// DiffCount implements the interface objinterface.Conflict.
func (conflict Conflict) DiffCount(ctx *sql.Context) (int, error) {
diffs, _, err := conflict.Diffs(ctx)
return len(diffs), err
}
// Diffs implements the interface objinterface.Conflict.
func (conflict Conflict) Diffs(ctx context.Context) ([]objinterface.RootObjectDiff, objinterface.RootObject, error) {
return DiffRootObjects(ctx, conflict.RootObjectID, conflict.FromHash, conflict.Ours, conflict.Theirs, conflict.Ancestor)
}
// FieldType implements the interface objinterface.Conflict.
func (conflict Conflict) FieldType(ctx context.Context, name string) *pgtypes.DoltgresType {
return GetFieldType(ctx, conflict.RootObjectID, name)
}
// GetContainedRootObjectID implements the interface objinterface.RootObject.
func (conflict Conflict) GetContainedRootObjectID() objinterface.RootObjectID {
return conflict.RootObjectID
}
// GetID implements the interface objinterface.RootObject.
func (conflict Conflict) GetID() id.Id {
return conflict.ID
}
// GetRootObjectID implements the interface objinterface.RootObject.
func (conflict Conflict) GetRootObjectID() objinterface.RootObjectID {
return objinterface.RootObjectID_Conflicts
}
// HashOf implements the interface objinterface.RootObject.
func (conflict Conflict) HashOf(ctx context.Context) (hash.Hash, error) {
data, err := conflict.Serialize(ctx)
if err != nil {
return hash.Hash{}, err
}
return hash.Of(data), nil
}
// Name implements the interface objinterface.RootObject.
func (conflict Conflict) Name() doltdb.TableName {
if conflict.Ours != nil {
return conflict.Ours.Name()
} else {
return conflict.Theirs.Name()
}
}
// RemoveDiffs implements the interface doltdb.ConflictRootObject.
func (conflict Conflict) RemoveDiffs(ctx *sql.Context, diffs []doltdb.RootObjectDiff) (doltdb.ConflictRootObject, error) {
// This is only called from within Dolt, and it specifically deals with modifying the conflict collection on the
// root. It is therefore safe to clear context values, to ensure the root changes are not overwritten.
ClearContextValues(ctx)
// We need to handle the root object field name in a special way, since it's how we select which side to use in full
if len(diffs) == 1 {
diff := diffs[0].(objinterface.RootObjectDiff)
if diff.FieldName == objinterface.FIELD_NAME_ROOT_OBJECT {
if conflict.Ours == nil {
conflict.Theirs = nil
} else {
conflict.Theirs = conflict.Ours
}
conflict.Ancestor = nil
return conflict, nil
}
}
// Deletion is equivalent to setting the value in "theirs" to "ours", as the same value cannot conflict with itself.
// We will only reach this point if both "ours" and "theirs" are non-nil entries, so the above relies on safe assumptions.
for _, doltDiff := range diffs {
diff := doltDiff.(objinterface.RootObjectDiff)
newTheirs, err := UpdateField(ctx, conflict.RootObjectID, conflict.Theirs, diff.FieldName, diff.OurValue)
if err != nil {
return nil, err
}
conflict.Theirs = newTheirs
}
return conflict, nil
}
// Rows implements the interface doltdb.ConflictRootObject.
func (conflict Conflict) Rows(ctx *sql.Context) (sql.RowIter, error) {
diffs, _, err := conflict.Diffs(ctx)
if err != nil {
return nil, err
}
rows := make([]sql.Row, len(diffs))
for i, diff := range diffs {
rows[i], err = diff.ToRow(ctx)
if err != nil {
return nil, err
}
}
return sql.RowsToRowIter(rows...), nil
}
// Schema implements the interface doltdb.ConflictRootObject.
func (conflict Conflict) Schema(originatingTableName string) sql.Schema {
sch := objinterface.RootObjectDiffSchema.Copy()
for _, col := range sch {
col.Source = originatingTableName
}
return sch
}
// UpdateField implements the interface doltdb.ConflictRootObject.
func (conflict Conflict) UpdateField(ctx *sql.Context, o doltdb.RootObjectDiff, n doltdb.RootObjectDiff) (doltdb.ConflictRootObject, error) {
// This is only called from within Dolt, and it specifically deals with modifying the conflict collection on the
// root. It is therefore safe to clear context values, to ensure the root changes are not overwritten.
ClearContextValues(ctx)
oldDiff := o.(objinterface.RootObjectDiff)
newDiff := n.(objinterface.RootObjectDiff)
// We need to handle the root object field name in a special way, since it's how we select which side to use in full
if oldDiff.FieldName == objinterface.FIELD_NAME_ROOT_OBJECT {
switch newDiff.OurValue.(string) {
case objinterface.FIELD_NAME_OURS:
conflict.Theirs = conflict.Ours
case objinterface.FIELD_NAME_THEIRS:
conflict.Ours = conflict.Theirs
case objinterface.FIELD_NAME_ANCESTOR:
conflict.Ours = conflict.Ancestor
conflict.Theirs = conflict.Ancestor
default:
return nil, errors.Newf("cannot replace the object with `%s`", newDiff.OurValue.(string))
}
conflict.Ancestor = nil
return conflict, nil
}
newOurs, err := UpdateField(ctx, conflict.RootObjectID, conflict.Ours, oldDiff.FieldName, newDiff.OurValue)
if err != nil {
return nil, err
}
conflict.Ours = newOurs
return conflict, nil
}
// ClearContextValues clears context values, and is declared in a different package. It is assigned here by an Init
// function to get around import cycles.
var ClearContextValues = func(ctx *sql.Context) {
panic("ClearContextValues was never initialized")
}
// DiffRootObjects handles conflict diffs, and is declared in a different package. It is assigned here by an Init
// function to get around import cycles.
var DiffRootObjects = func(ctx context.Context, rootObjID objinterface.RootObjectID, fromHash string, ours, theirs, ancestor objinterface.RootObject) ([]objinterface.RootObjectDiff, objinterface.RootObject, error) {
return nil, nil, errors.New("DiffRootObjects was never initialized")
}
// GetFieldType handles type fetching for fields, and is declared in a different package. It is assigned here by an Init
// function to get around import cycles.
var GetFieldType = func(ctx context.Context, rootObjID objinterface.RootObjectID, fieldName string) *pgtypes.DoltgresType {
panic("GetFieldType was never initialized")
}
// ResolveNameExternal handles name resolution across all collection types, and is declared in a different package. It
// is assigned here by an Init function to get around import cycles.
var ResolveNameExternal = func(ctx context.Context, name doltdb.TableName, rootObjects []objinterface.RootObject) (doltdb.TableName, id.Id, error) {
panic("ResolveNameExternal was never initialized")
}
// UpdateField handles updating fields in a root object, and is declared in a different package. It is assigned here by
// an Init function to get around import cycles.
var UpdateField = func(ctx context.Context, rootObjID objinterface.RootObjectID, rootObject objinterface.RootObject, fieldName string, newValue any) (objinterface.RootObject, error) {
return nil, errors.New("UpdateField was never initialized")
}
+117
View File
@@ -0,0 +1,117 @@
// Copyright 2025 Dolthub, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package conflicts
import (
"context"
"github.com/cockroachdb/errors"
"github.com/dolthub/dolt/go/libraries/doltcore/doltdb"
"github.com/dolthub/dolt/go/libraries/doltcore/merge"
"github.com/dolthub/dolt/go/store/hash"
"github.com/dolthub/dolt/go/store/prolly"
"github.com/dolthub/doltgresql/core/id"
"github.com/dolthub/doltgresql/core/rootobject/objinterface"
"github.com/dolthub/doltgresql/flatbuffers/gen/serial"
)
// storage is used to read from and write to the root.
var storage = objinterface.RootObjectSerializer{
Bytes: (*serial.RootValue).ConflictsBytes,
RootValueAdd: serial.RootValueAddConflicts,
}
// HandleMerge implements the interface objinterface.Collection.
func (*Collection) HandleMerge(ctx context.Context, mro merge.MergeRootObject) (doltdb.RootObject, *merge.MergeStats, error) {
// It technically doesn't make sense to merge conflicts, but we'll only error if there are differences
ourConflict := mro.OurRootObj.(Conflict)
theirConflict := mro.TheirRootObj.(Conflict)
// Ensure that they have the same identifier
if ourConflict.ID != theirConflict.ID {
return nil, nil, errors.Newf("attempted to merge different conflicts: `%s` and `%s`",
ourConflict.Name().String(), theirConflict.Name().String())
}
ourHash, err := ourConflict.HashOf(ctx)
if err != nil {
return nil, nil, err
}
theirHash, err := theirConflict.HashOf(ctx)
if err != nil {
return nil, nil, err
}
if ourHash.Equal(theirHash) {
return mro.OurRootObj, &merge.MergeStats{
Operation: merge.TableUnmodified,
Adds: 0,
Deletes: 0,
Modifications: 0,
DataConflicts: 0,
SchemaConflicts: 0,
RootObjectConflicts: 0,
ConstraintViolations: 0,
}, nil
}
return nil, nil, errors.New("attempted to merge conflicts")
}
// LoadCollection implements the interface objinterface.Collection.
func (*Collection) LoadCollection(ctx context.Context, root objinterface.RootValue) (objinterface.Collection, error) {
return LoadConflicts(ctx, root)
}
// LoadCollectionHash implements the interface objinterface.Collection.
func (*Collection) LoadCollectionHash(ctx context.Context, root objinterface.RootValue) (hash.Hash, error) {
m, ok, err := storage.GetProllyMap(ctx, root)
if err != nil || !ok {
return hash.Hash{}, err
}
return m.HashOf(), nil
}
// LoadConflicts loads the conflicts collection from the given root.
func LoadConflicts(ctx context.Context, root objinterface.RootValue) (*Collection, error) {
m, ok, err := storage.GetProllyMap(ctx, root)
if err != nil {
return nil, err
}
if !ok {
m, err = prolly.NewEmptyAddressMap(root.NodeStore())
if err != nil {
return nil, err
}
}
return NewCollection(ctx, m, root.NodeStore())
}
// ResolveNameFromObjects implements the interface objinterface.Collection.
func (*Collection) ResolveNameFromObjects(ctx context.Context, name doltdb.TableName, rootObjects []objinterface.RootObject) (doltdb.TableName, id.Id, error) {
// Conflicts make use of this interface function, so we must return nil to prevent infinite recursion
return doltdb.TableName{}, id.Null, nil
}
// Serializer implements the interface objinterface.Collection.
func (*Collection) Serializer() objinterface.RootObjectSerializer {
return storage
}
// UpdateRoot implements the interface objinterface.Collection.
func (pgc *Collection) UpdateRoot(ctx context.Context, root objinterface.RootValue) (objinterface.RootValue, error) {
m, err := pgc.Map(ctx)
if err != nil {
return nil, err
}
return storage.WriteProllyMap(ctx, root, m)
}
+126
View File
@@ -0,0 +1,126 @@
// Copyright 2025 Dolthub, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package conflicts
import (
"context"
"github.com/cockroachdb/errors"
"github.com/dolthub/dolt/go/libraries/doltcore/doltdb"
"github.com/dolthub/doltgresql/core/id"
"github.com/dolthub/doltgresql/core/rootobject/objinterface"
pgtypes "github.com/dolthub/doltgresql/server/types"
)
// DeserializeRootObject implements the interface objinterface.Collection.
func (pgc *Collection) DeserializeRootObject(ctx context.Context, data []byte) (objinterface.RootObject, error) {
return DeserializeConflict(ctx, data)
}
// DiffRootObjects implements the interface objinterface.Collection.
func (pgc *Collection) DiffRootObjects(ctx context.Context, fromHash string, ours objinterface.RootObject, theirs objinterface.RootObject, ancestor objinterface.RootObject) ([]objinterface.RootObjectDiff, objinterface.RootObject, error) {
return nil, nil, errors.Errorf("cannot diff the conflict root objects themselves")
}
// DropRootObject implements the interface objinterface.Collection.
func (pgc *Collection) DropRootObject(ctx context.Context, identifier id.Id) error {
return pgc.DropConflict(ctx, identifier)
}
// GetFieldType implements the interface objinterface.Collection.
func (pgc *Collection) GetFieldType(ctx context.Context, fieldName string) *pgtypes.DoltgresType {
return nil
}
// GetID implements the interface objinterface.Collection.
func (pgc *Collection) GetID() objinterface.RootObjectID {
return objinterface.RootObjectID_Conflicts
}
// GetRootObject implements the interface objinterface.Collection.
func (pgc *Collection) GetRootObject(ctx context.Context, identifier id.Id) (objinterface.RootObject, bool, error) {
conflict, err := pgc.GetConflict(ctx, identifier)
return conflict, err == nil && conflict.ID.IsValid(), err
}
// HasRootObject implements the interface objinterface.Collection.
func (pgc *Collection) HasRootObject(ctx context.Context, identifier id.Id) (bool, error) {
return pgc.HasConflict(ctx, identifier), nil
}
// IDToTableName implements the interface objinterface.Collection.
func (pgc *Collection) IDToTableName(identifier id.Id) doltdb.TableName {
return doltdb.TableName{Name: string(identifier)}
}
// IterAll implements the interface objinterface.Collection.
func (pgc *Collection) IterAll(ctx context.Context, callback func(rootObj objinterface.RootObject) (stop bool, err error)) error {
return pgc.IterateConflicts(ctx, func(conflict Conflict) (stop bool, err error) {
return callback(conflict)
})
}
// IterIDs implements the interface objinterface.Collection.
func (pgc *Collection) IterIDs(ctx context.Context, callback func(identifier id.Id) (stop bool, err error)) error {
return pgc.iterateIDs(ctx, func(conflictID id.Id) (stop bool, err error) {
return callback(conflictID)
})
}
// PutRootObject implements the interface objinterface.Collection.
func (pgc *Collection) PutRootObject(ctx context.Context, rootObj objinterface.RootObject) error {
conflict, ok := rootObj.(Conflict)
if !ok {
return errors.Newf("invalid conflict root object: %T", rootObj)
}
return pgc.AddConflict(ctx, conflict)
}
// RenameRootObject implements the interface objinterface.Collection.
func (pgc *Collection) RenameRootObject(ctx context.Context, oldName id.Id, newName id.Id) error {
return errors.New("cannot rename root object conflicts")
}
// ResolveName implements the interface objinterface.Collection.
func (pgc *Collection) ResolveName(ctx context.Context, name doltdb.TableName) (doltdb.TableName, id.Id, error) {
if len(pgc.idCache) == 0 {
return doltdb.TableName{}, id.Null, nil
}
objs := make([]objinterface.RootObject, 0, len(pgc.idCache))
for _, conflict := range pgc.accessCache {
if conflict.Ours != nil {
objs = append(objs, conflict.Ours)
} else if conflict.Theirs != nil {
objs = append(objs, conflict.Theirs)
} else if conflict.Ancestor != nil {
objs = append(objs, conflict.Ancestor)
}
}
return ResolveNameExternal(ctx, name, objs)
}
// TableNameToID implements the interface objinterface.Collection.
func (pgc *Collection) TableNameToID(name doltdb.TableName) id.Id {
if resolvedId, ok := pgc.nameCache[name]; ok {
return resolvedId
}
return id.Null
}
// UpdateField implements the interface objinterface.Collection.
func (pgc *Collection) UpdateField(ctx context.Context, rootObject objinterface.RootObject, fieldName string, newValue any) (objinterface.RootObject, error) {
return nil, errors.New("conflicts should not have conflict tables themselves, so this update should be impossible")
}
+122
View File
@@ -0,0 +1,122 @@
// Copyright 2025 Dolthub, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package conflicts
import (
"context"
"github.com/cockroachdb/errors"
"github.com/dolthub/doltgresql/core/rootobject/objinterface"
"github.com/dolthub/doltgresql/utils"
)
// Serialize returns the Conflict as a byte slice. If the Conflict is invalid, then this returns a nil slice.
func (conflict Conflict) Serialize(ctx context.Context) (_ []byte, err error) {
if !conflict.ID.IsValid() {
return nil, nil
}
// Write all of the conflicts to the writer
writer := utils.NewWriter(512)
writer.VariableUint(0) // Version
// Serialize "ours", "theirs", and "ancestor"
var ours, theirs, ancestor []byte
if conflict.Ours != nil {
ours, err = conflict.Ours.Serialize(ctx)
if err != nil {
return nil, err
}
}
if conflict.Theirs != nil {
theirs, err = conflict.Theirs.Serialize(ctx)
if err != nil {
return nil, err
}
}
if conflict.Ancestor != nil {
ancestor, err = conflict.Ancestor.Serialize(ctx)
if err != nil {
return nil, err
}
}
// Write the conflict data
writer.Id(conflict.ID)
writer.String(conflict.FromHash)
writer.Int64(int64(conflict.RootObjectID))
writer.Bool(conflict.Ours != nil)
writer.Bool(conflict.Theirs != nil)
writer.Bool(conflict.Ancestor != nil)
writer.ByteSlice(ours)
writer.ByteSlice(theirs)
writer.ByteSlice(ancestor)
// Returns the data
return writer.Data(), nil
}
// DeserializeConflict returns the Conflict that was serialized in the byte slice. Returns an empty Conflict (invalid
// ID) if data is nil or empty.
func DeserializeConflict(ctx context.Context, data []byte) (_ Conflict, err error) {
if len(data) == 0 {
return Conflict{}, nil
}
reader := utils.NewReader(data)
version := reader.VariableUint()
if version > 0 {
return Conflict{}, errors.Errorf("version %d of conflicts is not supported, please upgrade the server", version)
}
// Read from the reader
conflict := Conflict{}
conflict.ID = reader.Id()
conflict.FromHash = reader.String()
conflict.RootObjectID = objinterface.RootObjectID(reader.Int64())
hasOurs := reader.Bool()
hasTheirs := reader.Bool()
hasAncestor := reader.Bool()
ours := reader.ByteSlice()
theirs := reader.ByteSlice()
ancestor := reader.ByteSlice()
// Deserialize "ours", "theirs", and "ancestor"
if hasOurs {
conflict.Ours, err = DeserializeRootObject(ctx, conflict.RootObjectID, ours)
if err != nil {
return Conflict{}, err
}
}
if hasTheirs {
conflict.Theirs, err = DeserializeRootObject(ctx, conflict.RootObjectID, theirs)
if err != nil {
return Conflict{}, err
}
}
if hasAncestor {
conflict.Ancestor, err = DeserializeRootObject(ctx, conflict.RootObjectID, ancestor)
if err != nil {
return Conflict{}, err
}
}
if !reader.IsEmpty() {
return Conflict{}, errors.Errorf("extra data found while deserializing a conflict")
}
// Return the deserialized object
return conflict, nil
}
// DeserializeRootObject handles root object deserialization, and is declared in a different package. It is assigned
// here by an Init function to get around import cycles.
var DeserializeRootObject = func(ctx context.Context, rootObjID objinterface.RootObjectID, data []byte) (objinterface.RootObject, error) {
return nil, errors.New("DeserializeRootObject was never initialized")
}
+672
View File
@@ -0,0 +1,672 @@
// Copyright 2024 Dolthub, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package core
import (
"maps"
"slices"
"github.com/cockroachdb/errors"
"github.com/dolthub/dolt/go/libraries/doltcore/doltdb"
"github.com/dolthub/dolt/go/libraries/doltcore/sqle/dsess"
"github.com/dolthub/dolt/go/libraries/doltcore/sqle/resolve"
"github.com/dolthub/go-mysql-server/sql"
"github.com/dolthub/doltgresql/core/casts"
"github.com/dolthub/doltgresql/core/extensions"
"github.com/dolthub/doltgresql/core/functions"
"github.com/dolthub/doltgresql/core/procedures"
"github.com/dolthub/doltgresql/core/rootobject/objinterface"
"github.com/dolthub/doltgresql/core/sequences"
"github.com/dolthub/doltgresql/core/triggers"
"github.com/dolthub/doltgresql/core/typecollection"
)
// contextValues contains a set of cached data passed alongside the context. This data is considered temporary
// and may be refreshed at any point, including during the middle of a query. Callers should not assume that
// data stored in contextValues is persisted, and other types of data should not be added to contextValues.
type contextValues struct {
seqs map[string]*sequences.Collection
types map[string]*typecollection.TypeCollection
funcs map[string]*functions.Collection
procs map[string]*procedures.Collection
trigs map[string]*triggers.Collection
exts map[string]*extensions.Collection
casts map[string]*casts.Collection
pgCatalogCache any
runner sql.StatementRunner
// cache the dateOutputFormat, this is refreshed on SET
dateOutputFormat string
}
// getContextValues accesses the contextValues in the given context. If the context does not have a contextValues, then
// it creates one and adds it to the context.
func getContextValues(ctx *sql.Context) (*contextValues, error) {
if ctx == nil {
return nil, errors.New("context is nil")
}
sess := dsess.DSessFromSess(ctx.Session)
if sess.DoltgresSessObj == nil {
cv := &contextValues{}
sess.DoltgresSessObj = cv
return cv, nil
}
cv, ok := sess.DoltgresSessObj.(*contextValues)
if !ok {
return nil, errors.Errorf("context contains an unknown values struct of type: %T", sess.DoltgresSessObj)
}
return cv, nil
}
// ClearContextValues clears all context values. This is primarily for operations that are directly called from Dolt, as
// Dolt does not have the Doltgres concept of context values. Care must be taken to ensure that intermediate state
// written to the context values are not overwritten.
func ClearContextValues(ctx *sql.Context) {
sess := dsess.DSessFromSess(ctx.Session)
if sess.DoltgresSessObj != nil {
// We want to persist the runner between clears since it's static
var runner sql.StatementRunner
if cv, ok := sess.DoltgresSessObj.(*contextValues); ok {
runner = cv.runner
}
sess.DoltgresSessObj = &contextValues{
runner: runner,
}
}
}
// GetRootFromContext returns the working session's root from the context, along with the session.
func GetRootFromContext(ctx *sql.Context) (*dsess.DoltSession, *RootValue, error) {
return getRootFromContextForDatabase(ctx, "")
}
// getRootFromContextForDatabase returns the working session's root from the context for a specific database, along with the session.
func getRootFromContextForDatabase(ctx *sql.Context, database string) (*dsess.DoltSession, *RootValue, error) {
session := dsess.DSessFromSess(ctx.Session)
if len(database) == 0 {
database = ctx.GetCurrentDatabase()
}
state, ok, err := session.LookupDbState(ctx, database)
if err != nil {
return nil, nil, err
}
if !ok {
return nil, nil, sql.ErrDatabaseNotFound.New(database)
}
return session, state.WorkingRoot().(*RootValue), nil
}
// IsContextValid returns whether the context is valid for use with any of the functions in the package. If this is not
// false, then there's a high likelihood that the context is being used in a temporary scenario (such as setting up the
// database, etc.).
func IsContextValid(ctx *sql.Context) bool {
if ctx == nil {
return false
}
_, ok := ctx.Session.(*dsess.DoltSession)
return ok
}
// GetPgCatalogCache returns a cache of data for pg_catalog tables. This function should only be used by
// pg_catalog table handlers. The catalog cache instance stores generated pg_catalog table data so that
// it only has to generated table data once per query.
//
// TODO: The return type here is currently any, to avoid a package import cycle. This could be cleaned up by
// introducing a new interface type, in a package that does not depend on core or pgcatalog packages.
func GetPgCatalogCache(ctx *sql.Context) (any, error) {
cv, err := getContextValues(ctx)
if err != nil {
return nil, err
}
return cv.pgCatalogCache, nil
}
// SetPgCatalogCache sets |pgCatalogCache| as the catalog cache instance for this session.
//
// TODO: The input type here is currently any, to avoid a package import cycle. This could be cleaned up by
// introducing a new interface type, in a package that does not depend on core or pgcatalog packages.
func SetPgCatalogCache(ctx *sql.Context, pgCatalogCache any) error {
cv, err := getContextValues(ctx)
if err != nil {
return err
}
cv.pgCatalogCache = pgCatalogCache
return nil
}
// SetRunnerOnContext sets the given runner within the context values.
func SetRunnerOnContext(ctx *sql.Context, runner sql.StatementRunner) error {
if runner == nil {
return nil
}
cv, err := getContextValues(ctx)
if err != nil {
return err
}
cv.runner = runner
return nil
}
// GetRunnerFromContext returns the sql.StatementRunner from within the context.
func GetRunnerFromContext(ctx *sql.Context) (sql.StatementRunner, error) {
cv, err := getContextValues(ctx)
if err != nil {
return nil, err
}
return cv.runner, nil
}
// GetDoltTableFromContext returns the Dolt table from the context. Returns nil if no table was found.
func GetDoltTableFromContext(ctx *sql.Context, tableName doltdb.TableName) (*doltdb.Table, error) {
_, root, err := GetRootFromContext(ctx)
if err != nil {
return nil, err
}
var table *doltdb.Table
if tableName.Schema == "" {
_, table, _, err = resolve.Table(ctx, root, tableName.Name)
if err != nil {
return nil, err
}
} else {
table, _, err = root.GetTable(ctx, tableName)
}
if err != nil {
return nil, err
}
return table, nil
}
// GetSqlDatabaseFromContext returns the database from the context. Uses the context's current database if an empty
// string is provided. Returns nil if the database was not found.
func GetSqlDatabaseFromContext(ctx *sql.Context, database string) (sql.Database, error) {
session := dsess.DSessFromSess(ctx.Session)
if len(database) == 0 {
database = ctx.GetCurrentDatabase()
}
db, err := session.Provider().Database(ctx, database)
if err != nil {
if sql.ErrDatabaseNotFound.Is(err) {
return nil, nil
}
return nil, err
}
return db, nil
}
// GetSqlTableFromContext returns the table from the context. Uses the context's current database if an empty database
// name is provided. Returns nil if no table was found.
func GetSqlTableFromContext(ctx *sql.Context, databaseName string, tableName doltdb.TableName) (sql.Table, error) {
db, err := GetSqlDatabaseFromContext(ctx, databaseName)
if err != nil || db == nil {
return nil, err
}
schemaDb, ok := db.(sql.SchemaDatabase)
if !ok {
// Fairly confident that Dolt only has database implementations that inherit sql.SchemaDatabase, so only GMS
// databases may fail here (like information_schema). In this scenario, we expect that no schema will be passed,
// so we'll special-case it here.
if len(tableName.Schema) == 0 {
tbl, ok, err := db.GetTableInsensitive(ctx, tableName.Name)
if err != nil || !ok {
return nil, err
}
return tbl, nil
}
return nil, nil
}
var searchPath []string
if len(tableName.Schema) == 0 {
// If a schema was not provided, then we'll use the search path
searchPath, err = SearchPath(ctx)
if err != nil {
return nil, err
}
} else {
// A specific schema is given, so we'll only use that one for the search path
searchPath = []string{tableName.Schema}
}
for _, schema := range searchPath {
db, ok, err = schemaDb.GetSchema(ctx, schema)
if err != nil {
return nil, err
}
if !ok {
continue
}
tbl, ok, err := db.GetTableInsensitive(ctx, tableName.Name)
if err != nil {
return nil, err
}
if !ok {
continue
}
return tbl, nil
}
return nil, nil
}
// SearchPath returns the effective schema search path for the current session
func SearchPath(ctx *sql.Context) ([]string, error) {
path, err := resolve.SearchPath(ctx)
if err != nil {
return nil, err
}
// pg_catalog is *always* implicitly part of the search path as the first element, unless it's specifically
// included later. This allows users to override built-in names with user-defined names, but they have to
// opt in to that behavior.
hasPgCatalog := false
for _, schema := range path {
if schema == "pg_catalog" {
hasPgCatalog = true
break
}
}
if !hasPgCatalog {
path = append([]string{"pg_catalog"}, path...)
}
return path, nil
}
// GetExtensionsCollectionFromContext returns the extensions collection from the given context. Will always return a
// collection if no error is returned.
func GetExtensionsCollectionFromContext(ctx *sql.Context, database string) (*extensions.Collection, error) {
cv, err := getContextValues(ctx)
if err != nil {
return nil, err
}
if cv.exts == nil {
cv.exts = make(map[string]*extensions.Collection)
}
if len(database) == 0 {
database = ctx.GetCurrentDatabase()
}
if cv.exts[database] == nil {
_, root, err := getRootFromContextForDatabase(ctx, database)
if err != nil {
return nil, err
}
cv.exts[database], err = extensions.LoadExtensions(ctx, root)
if err != nil {
return nil, err
}
}
return cv.exts[database], nil
}
// GetFunctionsCollectionFromContext returns the functions collection from the given context. Will always return a
// collection if no error is returned.
func GetFunctionsCollectionFromContext(ctx *sql.Context, database string) (*functions.Collection, error) {
cv, err := getContextValues(ctx)
if err != nil {
return nil, err
}
if cv.funcs == nil {
cv.funcs = make(map[string]*functions.Collection)
}
if len(database) == 0 {
database = ctx.GetCurrentDatabase()
}
if cv.funcs[database] == nil {
_, root, err := getRootFromContextForDatabase(ctx, database)
if err != nil {
return nil, err
}
cv.funcs[database], err = functions.LoadFunctions(ctx, root)
if err != nil {
return nil, err
}
}
return cv.funcs[database], nil
}
// GetProceduresCollectionFromContext returns the procedures collection from the given context. Will always return a
// collection if no error is returned.
func GetProceduresCollectionFromContext(ctx *sql.Context, database string) (*procedures.Collection, error) {
cv, err := getContextValues(ctx)
if err != nil {
return nil, err
}
if cv.procs == nil {
cv.procs = make(map[string]*procedures.Collection)
}
if len(database) == 0 {
database = ctx.GetCurrentDatabase()
}
if cv.procs[database] == nil {
_, root, err := getRootFromContextForDatabase(ctx, database)
if err != nil {
return nil, err
}
cv.procs[database], err = procedures.LoadProcedures(ctx, root)
if err != nil {
return nil, err
}
}
return cv.procs[database], nil
}
// GetSequencesCollectionFromContext returns the given sequence collection from the context for the database
// named. If no database is provided, the context's current database is used.
// Will always return a collection if no error is returned.
func GetSequencesCollectionFromContext(ctx *sql.Context, database string) (*sequences.Collection, error) {
cv, err := getContextValues(ctx)
if err != nil {
return nil, err
}
if cv.seqs == nil {
cv.seqs = make(map[string]*sequences.Collection)
}
if len(database) == 0 {
database = ctx.GetCurrentDatabase()
}
if cv.seqs[database] == nil {
_, root, err := getRootFromContextForDatabase(ctx, database)
if err != nil {
return nil, err
}
cv.seqs[database], err = sequences.LoadSequences(ctx, root)
if err != nil {
return nil, err
}
}
return cv.seqs[database], nil
}
// GetTriggersCollectionFromContext returns the triggers collection from the given context. Will always return a
// collection if no error is returned.
func GetTriggersCollectionFromContext(ctx *sql.Context, database string) (*triggers.Collection, error) {
cv, err := getContextValues(ctx)
if err != nil {
return nil, err
}
if cv.trigs == nil {
cv.trigs = make(map[string]*triggers.Collection)
}
if len(database) == 0 {
database = ctx.GetCurrentDatabase()
}
if cv.trigs[database] == nil {
_, root, err := getRootFromContextForDatabase(ctx, database)
if err != nil {
return nil, err
}
cv.trigs[database], err = triggers.LoadTriggers(ctx, root)
if err != nil {
return nil, err
}
}
return cv.trigs[database], nil
}
// GetTypesCollectionFromContext returns the given type collection from the context.
// Will always return a collection if no error is returned.
func GetTypesCollectionFromContext(ctx *sql.Context, database string) (*typecollection.TypeCollection, error) {
cv, err := getContextValues(ctx)
if err != nil {
return nil, err
}
if cv.types == nil {
cv.types = make(map[string]*typecollection.TypeCollection)
}
if len(database) == 0 {
database = ctx.GetCurrentDatabase()
}
if cv.types[database] == nil {
_, root, err := getRootFromContextForDatabase(ctx, database)
if err != nil {
return nil, err
}
cv.types[database], err = typecollection.LoadTypes(ctx, root)
if err != nil {
return nil, err
}
}
return cv.types[database], nil
}
// GetCastsCollectionFromContext returns the given casts collection from the context.
// Will always return a collection if no error is returned.
func GetCastsCollectionFromContext(ctx *sql.Context, database string) (*casts.Collection, error) {
cv, err := getContextValues(ctx)
if err != nil {
return nil, err
}
if cv.casts == nil {
cv.casts = make(map[string]*casts.Collection)
}
if len(database) == 0 {
database = ctx.GetCurrentDatabase()
}
if cv.casts[database] == nil {
_, root, err := getRootFromContextForDatabase(ctx, database)
if err != nil {
return nil, err
}
cv.casts[database], err = casts.LoadCasts(ctx, root)
if err != nil {
return nil, err
}
}
return cv.casts[database], nil
}
// CloseContextRootFinalizer finalizes any changes persisted within the context by writing them to the working root.
// This should ONLY be called by the ContextRootFinalizer node.
func CloseContextRootFinalizer(ctx *sql.Context) error {
sess := dsess.DSessFromSess(ctx.Session)
if sess.DoltgresSessObj == nil {
return nil
}
cv, ok := sess.DoltgresSessObj.(*contextValues)
if !ok {
return nil
}
// We need to update the root for all databases used by this context. This logic parallels what happens during
// transaction commit in the dolt/sqle layer, where we check each branch state to see if it's dirty
for _, db := range databasesInContext(ctx, cv) {
err := updateSessionRootForDatabase(ctx, db, cv)
if err != nil {
return err
}
}
return nil
}
// GetDateStyleOutputFormat returns the cached DateOutputFormat
func GetDateStyleOutputFormat(ctx *sql.Context) (string, error) {
cv, err := getContextValues(ctx)
if err != nil {
return "", err
}
return cv.dateOutputFormat, nil
}
// SetDateStyleOutputFormat cached the provided dateOutputFormat
func SetDateStyleOutputFormat(ctx *sql.Context, dateOutputFormat string) error {
cv, err := getContextValues(ctx)
if err != nil {
return err
}
cv.dateOutputFormat = dateOutputFormat
return nil
}
// updateSessionRootForDatabase updates the root for all changes made to root object collections within the context
// values.
func updateSessionRootForDatabase(ctx *sql.Context, db string, cv *contextValues) error {
session, root, err := getRootFromContextForDatabase(ctx, db)
if err != nil {
return err
}
newRoot := root
if cv.seqs != nil && cv.seqs[db] != nil {
retRoot, err := cv.seqs[db].UpdateRoot(ctx, newRoot)
if err != nil {
return err
}
newRoot = retRoot.(*RootValue)
delete(cv.seqs, db)
}
if cv.funcs != nil && cv.funcs[db] != nil && cv.funcs[db].DiffersFrom(ctx, root) {
retRoot, err := cv.funcs[db].UpdateRoot(ctx, newRoot)
if err != nil {
return err
}
newRoot = retRoot.(*RootValue)
delete(cv.funcs, db)
}
if cv.procs != nil && cv.procs[db] != nil && cv.procs[db].DiffersFrom(ctx, root) {
retRoot, err := cv.procs[db].UpdateRoot(ctx, newRoot)
if err != nil {
return err
}
newRoot = retRoot.(*RootValue)
delete(cv.procs, db)
}
if cv.trigs != nil && cv.trigs[db] != nil && cv.trigs[db].DiffersFrom(ctx, root) {
retRoot, err := cv.trigs[db].UpdateRoot(ctx, newRoot)
if err != nil {
return err
}
newRoot = retRoot.(*RootValue)
delete(cv.trigs, db)
}
if cv.exts != nil && cv.exts[db] != nil && cv.exts[db].DiffersFrom(ctx, root) {
retRoot, err := cv.exts[db].UpdateRoot(ctx, newRoot)
if err != nil {
return err
}
newRoot = retRoot.(*RootValue)
delete(cv.exts, db)
}
if cv.types != nil && cv.types[db] != nil {
retRoot, err := cv.types[db].UpdateRoot(ctx, newRoot)
if err != nil {
return err
}
newRoot = retRoot.(*RootValue)
delete(cv.types, db)
}
if cv.casts != nil && cv.casts[db] != nil && cv.casts[db].DiffersFrom(ctx, root) {
retRoot, err := cv.casts[db].UpdateRoot(ctx, newRoot)
if err != nil {
return err
}
newRoot = retRoot.(*RootValue)
delete(cv.casts, db)
}
// Setting the session working root doesn't do a check to see if anything actually changed or not before marking that
// branch state dirty, and dolt only allows a single dirty working set per commit. So it's important here to only
// update the session root if something actually changed for that db.
if err, rootChanged := rootValueChanged(newRoot, root); rootChanged {
if err = session.SetWorkingRoot(ctx, db, newRoot); err != nil {
// TODO: We need a way to see if the session has a writeable working root
// (new interface method on session probably), and avoid setting it if so
if errors.Is(err, doltdb.ErrOperationNotSupportedInDetachedHead) {
return nil
}
return err
}
} else if err != nil {
return err
}
return nil
}
// rootValueChanged returns whether the new root value is different from the old one
func rootValueChanged(newRoot *RootValue, root *RootValue) (error, bool) {
if newRoot == root {
return nil, false
}
newHash, err := newRoot.HashOf()
if err != nil {
return err, false
}
oldHash, err := root.HashOf()
if err != nil {
return err, false
}
if newHash == oldHash {
return nil, false
}
return nil, true
}
// databasesInContext returns all databases found within the context values.
func databasesInContext(ctx *sql.Context, cv *contextValues) []string {
dbs := make(map[string]struct{})
if cv.seqs != nil {
for db := range cv.seqs {
dbs[db] = struct{}{}
}
}
currentDb := ctx.GetCurrentDatabase()
if len(currentDb) > 0 {
dbs[currentDb] = struct{}{}
}
return slices.Sorted(maps.Keys(dbs))
}
// clear removes the collection from the cache.
func (cv *contextValues) clear(objID objinterface.RootObjectID) {
switch objID {
case objinterface.RootObjectID_None:
// Nothing to cache with this
case objinterface.RootObjectID_Sequences:
cv.seqs = nil
case objinterface.RootObjectID_Types:
cv.types = nil
case objinterface.RootObjectID_Functions:
cv.funcs = nil
case objinterface.RootObjectID_Triggers:
cv.trigs = nil
case objinterface.RootObjectID_Extensions:
// We don't cache these
case objinterface.RootObjectID_Conflicts:
// We don't cache these
case objinterface.RootObjectID_Procedures:
cv.procs = nil
case objinterface.RootObjectID_Casts:
cv.casts = nil
default:
panic("unhandled context clear object ID")
}
}
+212
View File
@@ -0,0 +1,212 @@
// Copyright 2024 Dolthub, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package dataloader
import (
"bufio"
"fmt"
"io"
"strings"
"github.com/cockroachdb/errors"
"github.com/dolthub/dolt/go/libraries/doltcore/table"
"github.com/dolthub/go-mysql-server/sql"
"github.com/dolthub/doltgresql/server/types"
)
// CsvDataLoader is an implementation of DataLoader that reads data from chunks of CSV files and inserts them into a table.
type CsvDataLoader struct {
results LoadDataResults
partialRecord string
nextDataChunk *bufio.Reader
colTypes []*types.DoltgresType
sch sql.Schema
removeHeader bool
delimiter string
}
func (cdl *CsvDataLoader) SetNextDataChunk(ctx *sql.Context, data *bufio.Reader) error {
cdl.nextDataChunk = data
return nil
}
var _ DataLoader = (*CsvDataLoader)(nil)
const defaultCsvDelimiter = ","
// NewCsvDataLoader creates a new DataLoader instance that will produce rows for the schema provided.
// |header| is true, the first line of the data will be treated as a header and ignored. If |delimiter| is not the empty
// string, it will be used as the delimiter separating value.
func NewCsvDataLoader(colNames []string, sch sql.Schema, delimiter string, header bool) (*CsvDataLoader, error) {
colTypes, reducedSch, err := getColumnTypes(colNames, sch)
if err != nil {
return nil, err
}
if delimiter == "" {
delimiter = defaultCsvDelimiter
}
return &CsvDataLoader{
colTypes: colTypes,
sch: reducedSch,
removeHeader: header,
delimiter: delimiter,
}, nil
}
// nextRow attempts to read the next row from the data and return it, and returns true if a row was read
func (cdl *CsvDataLoader) nextRow(ctx *sql.Context, reader *csvReader) (sql.Row, bool, error) {
if cdl.removeHeader {
_, err := reader.readLine()
cdl.removeHeader = false
if err != nil {
return nil, false, err
}
}
record, err := reader.ReadSqlRow()
if err != nil {
if ple, ok := err.(*partialLineError); ok {
cdl.partialRecord = ple.partialLine
return nil, false, nil
}
// csvReader will return a BadRow error if it encounters an input line without the
// correct number of columns. If we see the end of data marker, then break out of the
// loop and return from this function without returning an error.
if _, ok := err.(*table.BadRow); ok {
if len(record) == 1 && record[0] == "\\." {
return nil, false, nil
}
}
if err != io.EOF {
return nil, false, err
}
recordValues := make([]string, 0, len(record))
for _, v := range record {
recordValues = append(recordValues, fmt.Sprintf("%v", v))
}
cdl.partialRecord = strings.Join(recordValues, ",")
return nil, false, nil
}
// If we see the end of data marker, then break out of the loop. Normally this will happen in the code
// above when we receive a BadRow error, since there won't be enough values, but if a table only has
// one column, we won't get a BadRow error, and we'll handle the end of data marker here.
if len(record) == 1 && record[0] == "\\." {
return nil, false, nil
}
if len(record) > len(cdl.colTypes) {
return nil, false, errors.Errorf("extra data after last expected column")
} else if len(record) < len(cdl.colTypes) {
return nil, false, errors.Errorf(`missing data for column "%s"`, cdl.sch[len(record)].Name)
}
// Cast the values using I/O input
row := make(sql.Row, len(cdl.colTypes))
for i := range cdl.colTypes {
if record[i] == nil {
row[i] = nil
} else {
row[i], err = cdl.colTypes[i].IoInput(ctx, fmt.Sprintf("%v", record[i]))
if err != nil {
return nil, false, err
}
}
}
return row, true, nil
}
// Finish implements the DataLoader interface
func (cdl *CsvDataLoader) Finish(ctx *sql.Context) (*LoadDataResults, error) {
// If there is partial data from the last chunk that hasn't been inserted, return an error.
if cdl.partialRecord != "" {
return nil, errors.Errorf("partial record (%s) found at end of data load", cdl.partialRecord)
}
return &cdl.results, nil
}
func (cdl *CsvDataLoader) Resolved() bool {
return true
}
func (cdl *CsvDataLoader) String() string {
return "CsvDataLoader"
}
func (cdl *CsvDataLoader) Schema(ctx *sql.Context) sql.Schema {
return cdl.sch
}
func (cdl *CsvDataLoader) Children() []sql.Node {
return nil
}
func (cdl *CsvDataLoader) WithChildren(ctx *sql.Context, children ...sql.Node) (sql.Node, error) {
if len(children) != 0 {
return nil, sql.ErrInvalidChildrenNumber.New(cdl, len(children), 0)
}
return cdl, nil
}
func (cdl *CsvDataLoader) IsReadOnly() bool {
return true
}
type csvRowIter struct {
cdl *CsvDataLoader
reader *csvReader
}
func (c csvRowIter) Next(ctx *sql.Context) (sql.Row, error) {
row, hasNext, err := c.cdl.nextRow(ctx, c.reader)
if err != nil {
return nil, err
}
// TODO: this isn't the best way to handle the count of rows, something like a RowUpdateAccumulator would be better
if hasNext {
c.cdl.results.RowsLoaded++
} else {
return nil, io.EOF
}
return row, nil
}
func (c csvRowIter) Close(context *sql.Context) error {
return nil
}
var _ sql.RowIter = (*csvRowIter)(nil)
func (cdl *CsvDataLoader) RowIter(ctx *sql.Context, r sql.Row) (sql.RowIter, error) {
combinedReader := NewStringPrefixReader(cdl.partialRecord, cdl.nextDataChunk)
cdl.partialRecord = ""
csvReader, err := newCsvReaderWithDelimiter(combinedReader, cdl.delimiter)
if err != nil {
return nil, err
}
return &csvRowIter{cdl: cdl, reader: csvReader}, nil
}
+364
View File
@@ -0,0 +1,364 @@
// Copyright 2024 Dolthub, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package dataloader
import (
"bufio"
"bytes"
"context"
"encoding/csv"
"io"
"unicode/utf8"
"github.com/dolthub/dolt/go/libraries/doltcore/table"
"github.com/dolthub/go-mysql-server/sql"
textunicode "golang.org/x/text/encoding/unicode"
"golang.org/x/text/transform"
)
// csvReadBufSize is the size of the buffer used when reading the csv file.
var csvReadBufSize = 256 * 1024
// partialLineError is an error type that is returned when an incomplete record is read from a CSV
// file. This can occur when a CSV document is split across multiple messages and the message
// boundaries don't line up with CSV record boundaries. Callers should use this error to record the
// partial line, so that it can be prepended to the next message.
type partialLineError struct {
partialLine string
}
var _ error = partialLineError{}
func (ple partialLineError) Error() string {
return "incomplete record found at end of CSV data: " + ple.partialLine
}
// csvReader implements TableReader. It reads csv files and returns rows.
//
// This implementation is adapted from the CSVReader in dolt, which is a fork
// of the standard Golang CSV reader. The main differences with the Golang std
// library implementation are that this parser has been adapted to differentiate
// between quoted and unquoted empty strings (for distinguishing between the empty
// string and NULL), and to use multi-rune delimiters. This adaptation removes the
// comment feature and the lazyQuotes option.
//
// Additionally, this fork of the dolt implementation removes some dolt specific
// features and adds support for a few Postgres requirements, such as allowing for
// the full CSV document to be arbitrarily split into multiple messages and for
// incomplete/partial lines to be communicated to the caller.
type csvReader struct {
closer io.Closer
bRd *bufio.Reader
isDone bool
delim []byte
numLine int
fieldsPerRecord int
}
// NewCsvReader creates a csvReader from a given ReadCloser.
//
// The interpretation of the bytes of the supplied reader is a little murky. If
// there is a UTF8, UTF16LE or UTF16BE BOM as the first bytes read, then the
// BOM is stripped and the remaining contents of the reader are treated as that
// encoding. If we are not in any of those marked encodings, then some of the
// bytes go uninterpreted until we get to the SQL layer. It is currently the
// case that newlines must be encoded as a '0xa' byte.
func NewCsvReader(r io.ReadCloser) (*csvReader, error) {
return newCsvReaderWithDelimiter(r, ",")
}
// newCsvReaderWithDelimiter creates a csvReader from a given ReadCloser, |r|, using
// the |delimiter| as the field delimiter in the parsed data.
func newCsvReaderWithDelimiter(r io.ReadCloser, delimiter string) (*csvReader, error) {
textReader := transform.NewReader(r, textunicode.BOMOverride(transform.Nop))
br := bufio.NewReaderSize(textReader, csvReadBufSize)
return &csvReader{
closer: r,
bRd: br,
isDone: false,
delim: []byte(delimiter),
}, nil
}
func (csvr *csvReader) ReadSqlRow() (sql.Row, error) {
if csvr.isDone {
return nil, io.EOF
}
rowVals, err := csvr.csvReadRecords(nil)
if err == io.EOF {
csvr.isDone = true
return nil, io.EOF
}
sqlRows := rowValsToSQLRows(rowVals)
if err != nil {
if _, ok := err.(*partialLineError); ok {
return nil, err
}
return sqlRows, table.NewBadRow(nil, err.Error())
}
return sqlRows, nil
}
func rowValsToSQLRows(rowVals []*string) sql.Row {
var sqlRow sql.Row
for _, rowVal := range rowVals {
if rowVal == nil {
sqlRow = append(sqlRow, nil)
} else {
sqlRow = append(sqlRow, *rowVal)
}
}
return sqlRow
}
// Close should release resources being held
func (csvr *csvReader) Close(ctx context.Context) error {
if csvr.closer != nil {
err := csvr.closer.Close()
csvr.closer = nil
return err
}
return nil
}
// Functions below this line are borrowed or adapted from encoding/csv/reader.go
// lengthNL returns 1 if the last byte in b is a newline, 0 otherwise.
func lengthNL(b []byte) int {
if len(b) > 0 && b[len(b)-1] == '\n' {
return 1
}
return 0
}
// readLine reads the next line (with the trailing endline).
// If EOF is hit without a trailing endline, it will be omitted.
// If some bytes were read, then the error is never io.EOF.
// The result is only valid until the next call to readLine.
func (csvr *csvReader) readLine() ([]byte, error) {
var rawBuffer []byte
line, err := csvr.bRd.ReadSlice('\n')
if err == bufio.ErrBufferFull {
rawBuffer = append(rawBuffer[:0], line...)
for err == bufio.ErrBufferFull {
line, err = csvr.bRd.ReadSlice('\n')
rawBuffer = append(rawBuffer, line...)
}
line = rawBuffer
}
if len(line) > 0 && err == io.EOF {
err = nil
// For backwards compatibility, drop trailing \r before EOF.
if line[len(line)-1] == '\r' {
line = line[:len(line)-1]
}
}
csvr.numLine++
// Normalize \r\n to \n on all input lines.
if n := len(line); n >= 2 && line[n-2] == '\r' && line[n-1] == '\n' {
line[n-2] = '\n'
line = line[:n-1]
}
// If the line does NOT end with a newline, then we must have read a partial record
if len(line) > 0 && lengthNL(line) == 0 {
return nil, &partialLineError{string(line)}
}
return line, err
}
type recordState struct {
line []byte
// recordBuffer holds the unescaped fields, one after another.
// The fields can be accessed by using the indexes in fieldIndexes.
// E.g., For the row `a,"b","c""d",e`, recordBuffer will contain `abc"de`
// and fieldIndexes will contain the indexes [1, 2, 5, 6].
recordBuffer []byte
fieldIndexes []int
rawData []byte
}
func (csvr *csvReader) csvReadRecords(dst []*string) ([]*string, error) {
recordStartline := csvr.numLine // Starting line for record
var rs recordState
var err error
for err == nil {
rs = recordState{}
rs.line, err = csvr.readLine()
rs.rawData = append(rs.rawData, rs.line...)
if err == nil && len(rs.line) == lengthNL(rs.line) {
continue // Skip empty lines
}
break
}
if err != nil {
return nil, err
}
// nullString indicates whether to interpret an empty string as a NULL
// only empty strings escaped with double quotes will be non-null
nullString := make(map[int]bool)
fieldIdx := 0
kontinue := true
for kontinue {
// Parse each field in the record.
keep := true
if len(rs.line) == 0 || rs.line[0] != '"' {
kontinue, keep, err = csvr.parseField(&rs)
if !keep {
nullString[fieldIdx] = true
}
} else {
kontinue, err = csvr.parseQuotedField(&rs)
if err != nil {
return nil, err
}
}
fieldIdx++
}
// Create a single string and create slices out of it.
// This pins the memory of the fields together, but allocates once.
str := string(rs.recordBuffer) // Convert to string once to batch allocations
dst = dst[:0]
if cap(dst) < len(rs.fieldIndexes) {
dst = make([]*string, len(rs.fieldIndexes))
}
dst = dst[:len(rs.fieldIndexes)]
var preIdx int
for i, idx := range rs.fieldIndexes {
_, ok := nullString[i]
if ok {
dst[i] = nil
} else {
s := str[preIdx:idx]
dst[i] = &s
}
preIdx = idx
}
// Check or update the expected fields per record.
if csvr.fieldsPerRecord > 0 {
if len(dst) != csvr.fieldsPerRecord && err == nil {
err = &csv.ParseError{StartLine: recordStartline, Line: csvr.numLine, Err: csv.ErrFieldCount}
}
} else if csvr.fieldsPerRecord == 0 {
csvr.fieldsPerRecord = len(dst)
}
return dst, err
}
func (csvr *csvReader) parseField(rs *recordState) (kontinue bool, keep bool, err error) {
i := bytes.Index(rs.line, csvr.delim)
field := rs.line
if i >= 0 {
field = field[:i]
} else {
field = field[:len(field)-lengthNL(field)]
}
rs.recordBuffer = append(rs.recordBuffer, field...)
rs.fieldIndexes = append(rs.fieldIndexes, len(rs.recordBuffer))
keep = len(field) != 0 // discard unquoted empty strings
if i >= 0 {
dl := len(csvr.delim)
rs.line = rs.line[i+dl:]
return true, keep, err
}
return false, keep, err
}
func (csvr *csvReader) parseQuotedField(rs *recordState) (kontinue bool, err error) {
const quoteLen = len(`"`)
dl := len(csvr.delim)
recordStartLine := csvr.numLine
// full copy needed here because we append rs.line to fullField, and this can result in buffer corruption in
// some cases (namely when windows line endings are present)
fullField := make([]byte, len(rs.line))
copy(fullField, rs.line)
// Quoted string field
rs.line = rs.line[quoteLen:]
for {
i := bytes.IndexByte(rs.line, '"')
if i >= 0 {
// Hit next quote.
rs.recordBuffer = append(rs.recordBuffer, rs.line[:i]...)
rs.line = rs.line[i+quoteLen:]
atDelimiter := len(rs.line) >= dl && bytes.Equal(rs.line[:dl], csvr.delim)
nextRune, _ := utf8.DecodeRune(rs.line)
switch {
case atDelimiter:
// `"<delimiter>` sequence (end of field).
rs.line = rs.line[dl:]
rs.fieldIndexes = append(rs.fieldIndexes, len(rs.recordBuffer))
return true, err
case nextRune == '"':
// `""` sequence (append quote).
rs.recordBuffer = append(rs.recordBuffer, '"')
rs.line = rs.line[quoteLen:]
case lengthNL(rs.line) == len(rs.line):
// `"\n` sequence (end of line).
rs.fieldIndexes = append(rs.fieldIndexes, len(rs.recordBuffer))
return false, err
default:
// `"*` sequence (invalid non-escaped quote).
col := utf8.RuneCount(fullField[:len(fullField)-len(rs.line)-quoteLen])
err = &csv.ParseError{StartLine: recordStartLine, Line: csvr.numLine, Column: col, Err: csv.ErrQuote}
return false, err
}
} else if len(rs.line) > 0 {
// Hit end of line (copy all data so far).
rs.recordBuffer = append(rs.recordBuffer, rs.line...)
if err != nil {
return false, err
}
rs.line, err = csvr.readLine()
rs.rawData = append(rs.rawData, rs.line...)
if err == io.EOF {
err = nil
}
// If we get a partialLineError, populate the partialLine field with the full record data
// since quoted fields can span multiple lines, otherwise we wouldn't capture the initial
// lines of this record.
if ple, ok := err.(*partialLineError); ok {
ple.partialLine = string(rs.rawData)
return true, ple
}
fullField = append(fullField, rs.line...)
} else {
// Abrupt end of file
if err == nil {
return false, &partialLineError{string(rs.rawData)}
}
rs.fieldIndexes = append(rs.fieldIndexes, len(rs.recordBuffer))
return false, err
}
}
}
+73
View File
@@ -0,0 +1,73 @@
// Copyright 2024 Dolthub, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package dataloader
import (
"bufio"
"github.com/cockroachdb/errors"
"github.com/dolthub/go-mysql-server/sql"
"github.com/dolthub/doltgresql/server/types"
)
// DataLoader allows callers to insert rows from multiple chunks into a table. Rows encoded in each chunk will not
// necessarily end cleanly on a chunk boundary, so DataLoader implementations must handle recognizing partial, or
// incomplete records, and saving that partial record until the next call to LoadChunk, so that it may be prefixed
// with the incomplete record.
type DataLoader interface {
sql.ExecSourceRel
// SetNextDataChunk sets the next data chunk to be processed by the DataLoader. Data records
// are not guaranteed to start and end cleanly on chunk boundaries, so implementations must recognize incomplete
// records and save them to prepend on the next processed chunk.
SetNextDataChunk(ctx *sql.Context, data *bufio.Reader) error
// Finish finalizes the current load operation and cleans up any resources used. Implementations should check that
// the last call to LoadChunk did not end with an incomplete record and return an error to the caller if so. The
// returned LoadDataResults describe the load operation, including how many rows were inserted.
Finish(ctx *sql.Context) (*LoadDataResults, error)
}
// LoadDataResults contains the results of a load data operation, including the number of rows loaded.
type LoadDataResults struct {
// RowsLoaded contains the total number of rows inserted during a load data operation.
RowsLoaded int32
}
// getColumnTypes returns the types of the columns in the schema that match the provided column names, in the order
// they are provided. If a subset of column names are provided, the returned types will only contain those columns.
// If the column names are not found in the schema, an error is returned.
func getColumnTypes(colNames []string, sch sql.Schema) ([]*types.DoltgresType, sql.Schema, error) {
colTypes := make([]*types.DoltgresType, len(colNames))
reducedSch := make(sql.Schema, len(colNames))
for i, colName := range colNames {
colIdx := sch.IndexOfColName(colName)
if colIdx < 0 {
// should be impossible
return nil, nil, errors.Errorf("column %s not found in schema", colName)
}
col := sch[colIdx]
var ok bool
colTypes[i], ok = col.Type.(*types.DoltgresType)
if !ok {
return nil, nil, errors.Errorf("unsupported column type: name: %s, type: %T", col.Name, col.Type)
}
reducedSch[i] = col
}
return colTypes, reducedSch, nil
}
+59
View File
@@ -0,0 +1,59 @@
// Copyright 2024 Dolthub, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package dataloader
import "io"
// stringPrefixReader is an io.ReadCloser that reads from a string prefix before reading from
// another io.Reader. This is used for reassembling partial records across multi-message
// exchanges, since the end of a wire message does not typically line up with the end of a record.
type stringPrefixReader struct {
prefix string
prefixPosition uint
reader io.Reader
}
var _ io.ReadCloser = (*stringPrefixReader)(nil)
// NewStringPrefixReader creates a new stringPrefixReader that first returns the data in |prefix| and
// then returns data from |reader|.
func NewStringPrefixReader(prefix string, reader io.Reader) *stringPrefixReader {
return &stringPrefixReader{
prefix: prefix,
reader: reader,
}
}
// Read implements the io.Reader interface
func (spr *stringPrefixReader) Read(p []byte) (n int, err error) {
if spr.prefixPosition < uint(len(spr.prefix)) {
n = copy(p, spr.prefix[spr.prefixPosition:])
spr.prefixPosition += uint(n)
if n == len(p) {
return n, nil
}
}
read, err := spr.reader.Read(p[n:])
return n + read, err
}
// Close implements the io.Closer interface
func (spr *stringPrefixReader) Close() error {
if closer, ok := spr.reader.(io.Closer); ok {
return closer.Close()
}
return nil
}
+215
View File
@@ -0,0 +1,215 @@
// Copyright 2024 Dolthub, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package dataloader
import (
"bufio"
"io"
"strings"
"github.com/cockroachdb/errors"
"github.com/dolthub/go-mysql-server/sql"
"github.com/dolthub/doltgresql/server/types"
)
const defaultTextDelimiter = "\t"
const defaultNullChar = "\\N"
// TabularDataLoader tracks the state of a load data operation from a tabular data source.
type TabularDataLoader struct {
results LoadDataResults
partialLine strings.Builder
nextDataChunk *bufio.Reader
colTypes []*types.DoltgresType
sch sql.Schema
delimiterChar string
nullChar string
removeHeader bool
}
var _ DataLoader = (*TabularDataLoader)(nil)
// NewTabularDataLoader creates a new TabularDataLoader to insert into the specified |table| using the specified
// |delimiterChar| and |nullChar|. If |header| is true, the first line of the data will be treated as a header and
// ignored.
func NewTabularDataLoader(colNames []string, tableSch sql.Schema, delimiterChar, nullChar string, header bool) (*TabularDataLoader, error) {
colTypes, reducedSch, err := getColumnTypes(colNames, tableSch)
if err != nil {
return nil, err
}
if delimiterChar == "" {
delimiterChar = defaultTextDelimiter
}
if nullChar == "" {
nullChar = defaultNullChar
}
return &TabularDataLoader{
colTypes: colTypes,
sch: reducedSch,
delimiterChar: delimiterChar,
nullChar: nullChar,
removeHeader: header,
}, nil
}
// nextRow returns the next SQL row from the reader provided, using any previously saved partial line. Returns true if
// there was another row.
func (tdl *TabularDataLoader) nextRow(ctx *sql.Context, data *bufio.Reader) (sql.Row, bool, error) {
if tdl.removeHeader {
_, err := data.ReadString('\n')
tdl.removeHeader = false
if err != nil {
return nil, false, err
}
}
for {
// Read the next line from the file
line, err := data.ReadString('\n')
if err != nil {
if err != io.EOF {
return nil, false, err
}
// bufio.Reader.ReadString will return an error AND a line
// if the final contents of the data does NOT end in the
// delimiter. In this case, that means that we need to save
// the partial line and use it in the next chunk.
tdl.partialLine.WriteString(line)
return nil, false, nil
}
// If we've not reached EOF, then there will be a newline appended to the end that we must remove.
line = strings.TrimSuffix(line, "\n")
// Data with windows line endings will also have a carriage return character that we need to remove.
line = strings.TrimSuffix(line, "\r")
if tdl.partialLine.Len() > 0 {
tdl.partialLine.WriteString(line)
line = tdl.partialLine.String()
tdl.partialLine.Reset()
}
// If we see the end of data marker, return early
if line == `\.` {
return nil, false, nil
}
// Skip over empty lines
if len(line) == 0 {
continue
}
// Split the values by the delimiter, ensuring the correct number of values have been read
values := strings.Split(line, tdl.delimiterChar)
if len(values) > len(tdl.colTypes) {
return nil, false, errors.Errorf("extra data after last expected column")
} else if len(values) < len(tdl.colTypes) {
return nil, false, errors.Errorf(`missing data for column "%s"`, tdl.sch[len(values)].Name)
}
// Cast the values using I/O input
row := make(sql.Row, len(tdl.colTypes))
for i := range tdl.colTypes {
if values[i] == tdl.nullChar {
row[i] = nil
} else {
// We must un-escape strings here since we're receiving everything verbatim
values[i] = strings.ReplaceAll(values[i], `\\`, `\`)
row[i], err = tdl.colTypes[i].IoInput(ctx, values[i])
if err != nil {
return nil, false, err
}
}
}
return row, true, nil
}
}
func (tdl *TabularDataLoader) SetNextDataChunk(ctx *sql.Context, data *bufio.Reader) error {
tdl.nextDataChunk = data
return nil
}
// Finish completes the current load data operation and finalizes the data that has been inserted.
func (tdl *TabularDataLoader) Finish(ctx *sql.Context) (*LoadDataResults, error) {
// If there is partial data from the last chunk that hasn't been inserted, return an error.
if tdl.partialLine.Len() > 0 {
return nil, errors.Errorf("partial line found at end of data load")
}
return &tdl.results, nil
}
func (tdl *TabularDataLoader) Resolved() bool {
return true
}
func (tdl *TabularDataLoader) String() string {
return "TabularDataLoader"
}
func (tdl *TabularDataLoader) Schema(ctx *sql.Context) sql.Schema {
return tdl.sch
}
func (tdl *TabularDataLoader) Children() []sql.Node {
return nil
}
func (tdl *TabularDataLoader) WithChildren(ctx *sql.Context, children ...sql.Node) (sql.Node, error) {
if len(children) != 0 {
return nil, sql.ErrInvalidChildrenNumber.New(tdl, len(children), 0)
}
return tdl, nil
}
func (tdl *TabularDataLoader) IsReadOnly() bool {
return true
}
type tabularRowIter struct {
tdl *TabularDataLoader
reader *bufio.Reader
}
func (t tabularRowIter) Next(ctx *sql.Context) (sql.Row, error) {
row, hasNext, err := t.tdl.nextRow(ctx, t.reader)
if err != nil {
return nil, err
}
// TODO: this isn't the best way to handle the count of rows, something like a RowUpdateAccumulator would be better
if hasNext {
t.tdl.results.RowsLoaded++
} else {
return nil, io.EOF
}
return row, nil
}
func (t tabularRowIter) Close(context *sql.Context) error {
return nil
}
func (tdl *TabularDataLoader) RowIter(ctx *sql.Context, r sql.Row) (sql.RowIter, error) {
return &tabularRowIter{tdl: tdl, reader: tdl.nextDataChunk}, nil
}
+234
View File
@@ -0,0 +1,234 @@
// Copyright 2025 Dolthub, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package extensions
import (
"cmp"
"context"
"maps"
"slices"
"github.com/cockroachdb/errors"
"github.com/dolthub/dolt/go/libraries/doltcore/doltdb"
"github.com/dolthub/dolt/go/store/hash"
"github.com/dolthub/dolt/go/store/prolly"
"github.com/dolthub/dolt/go/store/prolly/tree"
"github.com/dolthub/doltgresql/core/id"
"github.com/dolthub/doltgresql/core/rootobject/objinterface"
)
// Collection contains a collection of loaded extensions.
type Collection struct {
accessCache map[id.Extension]Extension // This cache is used for general access
idCache []id.Extension // This cache simply contains the name of every loaded extension
mapHash hash.Hash // This is cached so that we don't have to calculate the hash every time
underlyingMap prolly.AddressMap
ns tree.NodeStore
}
// Extension represents a loaded extension.
type Extension struct {
ExtName id.Extension
Namespace id.Namespace
Relocatable bool
LibIdentifier LibraryIdentifier
// TODO: keep track of what it references so I can later delete them
}
var _ objinterface.Collection = (*Collection)(nil)
var _ objinterface.RootObject = Extension{}
// NewCollection returns a new Collection.
func NewCollection(ctx context.Context, underlyingMap prolly.AddressMap, ns tree.NodeStore) (*Collection, error) {
collection := &Collection{
accessCache: make(map[id.Extension]Extension),
idCache: nil,
mapHash: hash.Hash{},
underlyingMap: underlyingMap,
ns: ns,
}
return collection, collection.reloadCaches(ctx)
}
// GetLoadedExtension returns the loaded extension with the given name. Returns an extension with an invalid ID if it
// cannot be found.
func (pge *Collection) GetLoadedExtension(ctx context.Context, name id.Extension) (Extension, error) {
if f, ok := pge.accessCache[name]; ok {
return f, nil
}
return Extension{}, nil
}
// HasLoadedExtension returns whether the extension has been loaded.
func (pge *Collection) HasLoadedExtension(ctx context.Context, name id.Extension) bool {
_, ok := pge.accessCache[name]
return ok
}
// AddLoadedExtension adds a new extension, that has already been loaded, to the collection.
func (pge *Collection) AddLoadedExtension(ctx context.Context, ext Extension) error {
// First we'll check to see if it exists
if _, ok := pge.accessCache[ext.ExtName]; ok {
return errors.Errorf(`extension "%s" already exists`, ext.ExtName)
}
// Now we'll add the extension to our map
data, err := ext.Serialize(ctx)
if err != nil {
return err
}
h, err := pge.ns.WriteBytes(ctx, data)
if err != nil {
return err
}
mapEditor := pge.underlyingMap.Editor()
if err = mapEditor.Add(ctx, string(ext.ExtName), h); err != nil {
return err
}
newMap, err := mapEditor.Flush(ctx)
if err != nil {
return err
}
pge.underlyingMap = newMap
pge.mapHash = pge.underlyingMap.HashOf()
return pge.reloadCaches(ctx)
}
// DropLoadedExtension drops a loaded extension. This should be called when unloading an extension, but this function
// itself does not perform the necessary logic.
func (pge *Collection) DropLoadedExtension(ctx context.Context, names ...id.Extension) error {
// TODO: should this also handle the unloading logic?
if len(names) == 0 {
return nil
}
// Check that each name exists before performing any deletions
for _, name := range names {
if _, ok := pge.accessCache[name]; !ok {
return errors.Errorf(`extension "%s" does not exist`, name)
}
}
// Now we'll remove the extensions from the map
mapEditor := pge.underlyingMap.Editor()
for _, name := range names {
err := mapEditor.Delete(ctx, string(name))
if err != nil {
return err
}
}
newMap, err := mapEditor.Flush(ctx)
if err != nil {
return err
}
pge.underlyingMap = newMap
pge.mapHash = pge.underlyingMap.HashOf()
return pge.reloadCaches(ctx)
}
// Clone returns a new *Collection with the same contents as the original.
func (pge *Collection) Clone(ctx context.Context) *Collection {
return &Collection{
accessCache: maps.Clone(pge.accessCache),
idCache: slices.Clone(pge.idCache),
mapHash: pge.mapHash,
underlyingMap: pge.underlyingMap,
ns: pge.ns,
}
}
// Map writes any cached sequences to the underlying map, and then returns the underlying map.
func (pge *Collection) Map(ctx context.Context) (prolly.AddressMap, error) {
return pge.underlyingMap, nil
}
// DiffersFrom returns true when the hash that is associated with the underlying map for this collection is different
// from the hash in the given root.
func (pge *Collection) DiffersFrom(ctx context.Context, root objinterface.RootValue) bool {
hashOnGivenRoot, err := pge.LoadCollectionHash(ctx, root)
if err != nil {
return true
}
if pge.mapHash.Equal(hashOnGivenRoot) {
return false
}
// An empty map should match an uninitialized collection on the root
count, err := pge.underlyingMap.Count()
if err == nil && count == 0 && hashOnGivenRoot.IsEmpty() {
return false
}
return true
}
// reloadCaches writes the underlying map's contents to the caches.
func (pge *Collection) reloadCaches(ctx context.Context) error {
count, err := pge.underlyingMap.Count()
if err != nil {
return err
}
clear(pge.accessCache)
pge.mapHash = pge.underlyingMap.HashOf()
pge.idCache = make([]id.Extension, 0, count)
return pge.underlyingMap.IterAll(ctx, func(_ string, h hash.Hash) error {
if h.IsEmpty() {
return nil
}
data, err := pge.ns.ReadBytes(ctx, h)
if err != nil {
return err
}
ext, err := DeserializeExtension(ctx, data)
if err != nil {
return err
}
pge.accessCache[ext.ExtName] = ext
pge.idCache = append(pge.idCache, ext.ExtName)
return nil
})
}
// CompareVersions compares the major and minor version of the extension versus the given extension.
func (ext Extension) CompareVersions(other Extension) int {
return cmp.Or(
cmp.Compare(ext.LibIdentifier.Version().Major(), other.LibIdentifier.Version().Major()),
cmp.Compare(ext.LibIdentifier.Version().Minor(), other.LibIdentifier.Version().Minor()),
)
}
// GetID implements the interface objinterface.RootObject.
func (ext Extension) GetID() id.Id {
return ext.ExtName.AsId()
}
// GetRootObjectID implements the interface objinterface.RootObject.
func (ext Extension) GetRootObjectID() objinterface.RootObjectID {
return objinterface.RootObjectID_Extensions
}
// HashOf implements the interface objinterface.RootObject.
func (ext Extension) HashOf(ctx context.Context) (hash.Hash, error) {
data, err := ext.Serialize(ctx)
if err != nil {
return hash.Hash{}, err
}
return hash.Of(data), nil
}
// Name implements the interface objinterface.RootObject.
func (ext Extension) Name() doltdb.TableName {
return doltdb.TableName{Name: ext.ExtName.Name()}
}
+134
View File
@@ -0,0 +1,134 @@
// Copyright 2025 Dolthub, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package extensions
import (
"context"
"github.com/cockroachdb/errors"
"github.com/dolthub/dolt/go/libraries/doltcore/doltdb"
"github.com/dolthub/dolt/go/libraries/doltcore/merge"
"github.com/dolthub/dolt/go/store/hash"
"github.com/dolthub/dolt/go/store/prolly"
"github.com/dolthub/doltgresql/core/id"
"github.com/dolthub/doltgresql/core/rootobject/objinterface"
"github.com/dolthub/doltgresql/flatbuffers/gen/serial"
)
// storage is used to read from and write to the root.
var storage = objinterface.RootObjectSerializer{
Bytes: (*serial.RootValue).ExtensionsBytes,
RootValueAdd: serial.RootValueAddExtensions,
}
// HandleMerge implements the interface objinterface.Collection.
func (*Collection) HandleMerge(ctx context.Context, mro merge.MergeRootObject) (doltdb.RootObject, *merge.MergeStats, error) {
ourExt := mro.OurRootObj.(Extension)
theirExt := mro.TheirRootObj.(Extension)
// Ensure that they have the same ID
if ourExt.ExtName != theirExt.ExtName {
return nil, nil, errors.Newf("attempted to merge different extensions: `%s` and `%s`",
ourExt.Name().String(), theirExt.Name().String())
}
ourHash, err := ourExt.HashOf(ctx)
if err != nil {
return nil, nil, err
}
theirHash, err := theirExt.HashOf(ctx)
if err != nil {
return nil, nil, err
}
// We always keep the newest extension. I don't think this is actually valid since old extensions are very likely
// to have invalid function signatures, but this is a start for now.
// TODO: figure out a better method
if ourHash.Equal(theirHash) || ourExt.CompareVersions(theirExt) >= 0 {
return mro.OurRootObj, &merge.MergeStats{
Operation: merge.TableUnmodified,
Adds: 0,
Deletes: 0,
Modifications: 0,
DataConflicts: 0,
SchemaConflicts: 0,
ConstraintViolations: 0,
}, nil
} else {
return mro.TheirRootObj, &merge.MergeStats{
Operation: merge.TableModified,
Adds: 0,
Deletes: 0,
Modifications: 1,
DataConflicts: 0,
SchemaConflicts: 0,
ConstraintViolations: 0,
}, nil
}
}
// LoadCollection implements the interface objinterface.Collection.
func (*Collection) LoadCollection(ctx context.Context, root objinterface.RootValue) (objinterface.Collection, error) {
return LoadExtensions(ctx, root)
}
// LoadCollectionHash implements the interface objinterface.Collection.
func (*Collection) LoadCollectionHash(ctx context.Context, root objinterface.RootValue) (hash.Hash, error) {
m, ok, err := storage.GetProllyMap(ctx, root)
if err != nil || !ok {
return hash.Hash{}, err
}
return m.HashOf(), nil
}
// LoadExtensions loads the extensions collection from the given root.
func LoadExtensions(ctx context.Context, root objinterface.RootValue) (*Collection, error) {
m, ok, err := storage.GetProllyMap(ctx, root)
if err != nil {
return nil, err
}
if !ok {
m, err = prolly.NewEmptyAddressMap(root.NodeStore())
if err != nil {
return nil, err
}
}
return NewCollection(ctx, m, root.NodeStore())
}
// ResolveNameFromObjects implements the interface objinterface.Collection.
func (*Collection) ResolveNameFromObjects(ctx context.Context, name doltdb.TableName, rootObjects []objinterface.RootObject) (doltdb.TableName, id.Id, error) {
tempCollection := Collection{
accessCache: make(map[id.Extension]Extension),
}
for _, rootObject := range rootObjects {
if obj, ok := rootObject.(Extension); ok {
tempCollection.accessCache[obj.ExtName] = obj
}
}
return tempCollection.ResolveName(ctx, name)
}
// Serializer implements the interface objinterface.Collection.
func (*Collection) Serializer() objinterface.RootObjectSerializer {
return storage
}
// UpdateRoot implements the interface objinterface.Collection.
func (pge *Collection) UpdateRoot(ctx context.Context, root objinterface.RootValue) (objinterface.RootValue, error) {
m, err := pge.Map(ctx)
if err != nil {
return nil, err
}
return storage.WriteProllyMap(ctx, root, m)
}
+199
View File
@@ -0,0 +1,199 @@
// Copyright 2025 Dolthub, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package extensions
import (
"fmt"
"strconv"
"sync"
"github.com/cockroachdb/errors"
"github.com/dolthub/doltgresql/core/extensions/pg_extension"
)
var (
cachedError error // TODO: is it better for this to be a local object instead of a global since it's always returned?
allExtensions map[string]*pg_extension.ExtensionFiles // TODO: should use id.Extension instead of a string
allLibraries map[string]*pg_extension.Library // TODO: close these at some point
extMutex = &sync.Mutex{}
libMutex = &sync.Mutex{}
)
// Version 0 identifiers are encoded like the following:
// 00AAA1111111111BBB
// `00` is a two-digit version specifier, and we only have version 0 for now
// `AAA` is the three-letter platform specifier
// `1111111111` is the ten-digit library version specifier (i.e. an encoded form of 1.0, 1.5, etc.)
// `BBB` is the extension name, and may be of any length (will have at least 1 character)
// LibraryIdentifier points to a specific extension, as extension functions are dependent on the extension name,
// version, and originating platform (as different platforms may encode data differently).
type LibraryIdentifier string
// InvalidIdentifierReason gives a reason as to why an Identifier is invalid.
type InvalidIdentifierReason uint8
const (
InvalidIdentifierReason_MismatchedPlatform InvalidIdentifierReason = iota
InvalidIdentifierReason_MissingLibrary
InvalidIdentifierReason_InvalidVersion
)
// InvalidIdentifier represents an invalid LibraryIdentifier, providing both the LibraryIdentifier and the reason that
// it is invalid.
type InvalidIdentifier struct {
Identifier LibraryIdentifier
Reason InvalidIdentifierReason
}
// GetExtension returns the extension matching the given name. Returns an error if the extension cannot be found.
func GetExtension(name string) (_ *pg_extension.ExtensionFiles, err error) {
extMutex.Lock()
defer extMutex.Unlock()
if cachedError != nil {
return nil, cachedError
}
if allExtensions == nil {
allLibraries = make(map[string]*pg_extension.Library)
allExtensions, err = pg_extension.LoadExtensions()
if err != nil {
allExtensions = make(map[string]*pg_extension.ExtensionFiles)
cachedError = err
return nil, err
}
}
ext, ok := allExtensions[name]
if !ok {
return nil, errors.Errorf(`could not open extension control file "%s.control"`, name)
}
return ext, nil
}
// GetExtensionFunction returns the function inside the extension matching the given names. Returns an error if the
// extension or function cannot be found.
func GetExtensionFunction(identifier LibraryIdentifier, funcName string) (_ pg_extension.Function, err error) {
libMutex.Lock()
defer libMutex.Unlock()
extName := identifier.ExtensionName()
if identifier.Platform() != pg_extension.PLATFORM {
return pg_extension.Function{}, errors.Errorf(
`function "%s" was initialized through "%s" on a different platform, which is not supported`, funcName, extName)
}
lib, ok := allLibraries[extName]
if !ok {
ext, err := GetExtension(extName)
if err != nil {
return pg_extension.Function{}, err
}
lib, err = ext.LoadLibrary()
if err != nil {
return pg_extension.Function{}, err
}
lib.Version = ext.Control.DefaultVersion
allLibraries[extName] = lib
}
if lib.Version != identifier.Version() {
return pg_extension.Function{}, errors.Errorf(
`function "%s" was initialized through "%s" v%s, the current platform only supports v%s"`,
funcName, extName, identifier.Version().String(), lib.Version.String())
}
libFunc, ok := lib.Funcs[funcName]
if !ok {
return pg_extension.Function{}, errors.Errorf(`extension "%s" does not declare the function "%s"`, extName, funcName)
}
return libFunc, nil
}
// FindInvalidIdentifiers returns all identifiers that are not valid for the current environment. If the return is
// empty, then that means all of the given identifiers are valid.
func FindInvalidIdentifiers(identifiers ...LibraryIdentifier) []InvalidIdentifier {
extMutex.Lock()
defer extMutex.Unlock()
var invalidIdentifiers []InvalidIdentifier
if cachedError != nil {
invalidIdentifiers = make([]InvalidIdentifier, 0, len(identifiers))
for _, identifier := range identifiers {
invalidIdentifiers = append(invalidIdentifiers, InvalidIdentifier{
Identifier: identifier,
Reason: InvalidIdentifierReason_MissingLibrary,
})
}
return invalidIdentifiers
}
for _, identifier := range identifiers {
if identifier.Platform() != pg_extension.PLATFORM {
invalidIdentifiers = append(invalidIdentifiers, InvalidIdentifier{
Identifier: identifier,
Reason: InvalidIdentifierReason_MismatchedPlatform,
})
continue
}
ext, ok := allExtensions[identifier.ExtensionName()]
if !ok {
invalidIdentifiers = append(invalidIdentifiers, InvalidIdentifier{
Identifier: identifier,
Reason: InvalidIdentifierReason_MissingLibrary,
})
continue
}
if ext.Control.DefaultVersion != identifier.Version() {
invalidIdentifiers = append(invalidIdentifiers, InvalidIdentifier{
Identifier: identifier,
Reason: InvalidIdentifierReason_InvalidVersion,
})
continue
}
}
return invalidIdentifiers
}
// GetPlatform returns the current platform that Doltgres is being executed on. This is encoded as a three-letter string.
func GetPlatform() string {
return pg_extension.PLATFORM
}
// CreateLibraryIdentifier creates a LibraryIdentifier using the given information.
func CreateLibraryIdentifier(name string, version pg_extension.Version) LibraryIdentifier {
return LibraryIdentifier(fmt.Sprintf("00%s%010d%s", pg_extension.PLATFORM, version, name))
}
// Platform returns the platform that this LibraryIdentifier was created on.
func (id LibraryIdentifier) Platform() string {
return string(id[2:5])
}
// Version returns the library version that this LibraryIdentifier references.
func (id LibraryIdentifier) Version() pg_extension.Version {
val, err := strconv.ParseUint(string(id[5:15]), 10, 32)
if err != nil {
// We'll panic for now since this should never happen
panic(err)
}
return pg_extension.Version(val)
}
// ExtensionName returns the extension referenced by this LibraryIdentifier.
func (id LibraryIdentifier) ExtensionName() string {
return string(id[15:])
}
// DisplayString returns the identifier as a human-readable string.
func (id LibraryIdentifier) DisplayString() string {
return fmt.Sprintf("%s--%s:%s", id.ExtensionName(), id.Version().String(), id.Platform())
}
+12
View File
@@ -0,0 +1,12 @@
# Finding Extension Function Imports
These are commands that can be used to find the functions that an extension imports, so that we know which ones we need to implement for the extension to load.
## Windows
On Windows, we make use of `dumpbin`, which is installed alongside Visual Studio (the full version, _not_ Code). We are generally only interested in the functions under `postgres.exe`, as the library should load other DLLs as necessary.
```cmd
dumpbin /imports "C:/Program Files/PostgreSQL/15/lib/LIBRARY_NAME.dll"
```
## Linux
On Linux, we make use of the built-in `nm` command. We are interested in the `U` functions that do not have an `@` near the end (as those are usually implemented in external libraries).
```bash
nm -D -u /usr/lib/postgresql/15/lib/LIBRARY_NAME.so
```
+56
View File
@@ -0,0 +1,56 @@
// Copyright 2025 Dolthub, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package pg_extension
/*
#cgo CFLAGS: "-I${SRCDIR}/library"
#include "exports.h"
static inline Datum CallFmgrFunctionC(FunctionCallInfo fcinfo) {
return ((PGFunction)fcinfo->flinfo->fn_addr)(fcinfo);
}
*/
import "C"
import "unsafe"
// Datum is a C pointer to some data. Depending on the function being called, it may not be a pointer that should be
// freed, as some functions return pointers to static memory.
type Datum uintptr
// NullableDatum is used for arguments to Fmgr function calls.
type NullableDatum struct {
Value Datum
IsNull bool
}
// CallFmgrFunction calls the given function and forwards the arguments.
func CallFmgrFunction(fn uintptr, args ...NullableDatum) (result Datum, isNotNull bool) {
fi := Malloc[C.FmgrInfo]()
defer Free(fi)
ZeroMemory(fi)
fc := Malloc[C.FunctionCallInfoBaseData]()
defer Free(fc)
ZeroMemory(fc)
fi.fn_addr = unsafe.Pointer(fn)
fc.flinfo = fi
fc.nargs = C.int16_t(len(args))
for i, arg := range args {
fc.args[i].value = C.Datum(arg.Value)
fc.args[i].isnull = C.bool(arg.IsNull)
}
result = Datum(C.CallFmgrFunctionC(fc))
return result, !bool(fc.isnull) && result != 0
}
@@ -0,0 +1,380 @@
// Copyright 2025 Dolthub, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package pg_extension
import (
"cmp"
"fmt"
"maps"
"os"
"regexp"
"slices"
"strconv"
"strings"
)
// sqlFunctionCapture is a regex to capture the function name as defined in the library. We'll eventually replace this
// and use the nodes from the parser, but this is good enough for the default extensions.
var sqlFunctionCapture = regexp.MustCompile(`(?is)create\s+(?:or\s+replace\s+)?function\s+(.*?)\s*\(.*?\)\s+(?:.*?language c.*?as\s+'.*?'\s*,\s*'(.*?)'.*?;|.*?as\s+'.*?'\s*,\s*'(.*?)'.*?language c.*?;|.*?language c.*?;)`)
// createFunctionStart is a regex to find the beginning of a CREATE FUNCTION statement.
var createFunctionStart = regexp.MustCompile(`(?is)create\s+(?:or\s+replace\s+)?function`)
// ExtensionFiles contains all of the files that are related to or used by an extension.
type ExtensionFiles struct {
Name string
ControlFileName string
SQLFileNames []string
LibraryFileName string
ControlFileDir string
LibraryFileDir string
Control Control
}
// Control contains the contents of the control file.
// https://www.postgresql.org/docs/15/extend-extensions.html#id-1.8.3.20.11
type Control struct {
Directory string
DefaultVersion Version
Comment string
Encoding string
ModulePathname string
Requires []string
Superuser bool
Trusted bool
Relocatable bool
Schema string
Extra map[string]string // All entries in here could not be matched to an expected field
}
// Version specifies the major and minor version numbers for an extension.
type Version uint32
// FilenameVersions returns the versions that were encoded in a filename. `From` is the first number, while `To` is the
// second number. If a filename only specifies a single version, this both `From` and `To` will equal one another.
type FilenameVersions struct {
From Version
To Version
}
// LoadExtensions loads information for all extensions that are in the extensions directory of a local Postgres installation.
func LoadExtensions() (map[string]*ExtensionFiles, error) {
libDir, extDir, err := PostgresDirectories()
if err != nil {
return nil, err
}
dirEntries, err := os.ReadDir(extDir)
if err != nil {
return nil, err
}
libEntries, err := os.ReadDir(libDir)
if err != nil {
return nil, err
}
extensionFiles := make(map[string]*ExtensionFiles)
// Look for the control files first
for _, dirEntry := range dirEntries {
fileName := dirEntry.Name()
if !dirEntry.IsDir() && strings.HasSuffix(fileName, ".control") {
extensionName := strings.TrimSuffix(fileName, ".control")
extensionFiles[extensionName] = &ExtensionFiles{
Name: extensionName,
ControlFileName: fileName,
ControlFileDir: extDir,
}
}
}
// Associate the SQL files and libraries
for _, extFile := range extensionFiles {
for _, dirEntry := range dirEntries {
fileName := dirEntry.Name()
if !dirEntry.IsDir() && strings.HasPrefix(fileName, extFile.Name+"--") && strings.HasSuffix(fileName, ".sql") {
extFile.SQLFileNames = append(extFile.SQLFileNames, fileName)
}
}
for _, libEntry := range libEntries {
fileName := libEntry.Name()
if !libEntry.IsDir() && strings.HasPrefix(fileName, extFile.Name+".") {
extFile.LibraryFileName = fileName
extFile.LibraryFileDir = libDir
}
}
slices.SortFunc(extFile.SQLFileNames, func(aStr, bStr string) int {
a := DecodeFilenameVersions(extFile.Name, aStr)
b := DecodeFilenameVersions(extFile.Name, bStr)
return cmp.Or(
cmp.Compare(a.From, b.From),
cmp.Compare(a.To, b.To),
)
})
// Some SQL files are old migration files that won't apply to us, so we can remove them by starting at the first
// non-migration file.
for nextLoop := true; nextLoop; {
nextLoop = false
for i := 1; i < len(extFile.SQLFileNames); i++ {
if strings.Count(extFile.SQLFileNames[i], "--") == 1 {
extFile.SQLFileNames = extFile.SQLFileNames[i:]
nextLoop = true
break
}
}
}
// Load the control file
if err = extFile.loadControl(); err != nil {
return nil, err
}
}
return extensionFiles, nil
}
// loadControl loads the control file of an extension.
func (extFile *ExtensionFiles) loadControl() error {
data, err := os.ReadFile(fmt.Sprintf("%s/%s", extFile.ControlFileDir, extFile.ControlFileName))
if err != nil {
return err
}
extFile.Control = Control{ // These are the default values
Directory: "",
DefaultVersion: 0,
Comment: "",
Encoding: "",
ModulePathname: "",
Requires: nil,
Superuser: true,
Trusted: false,
Relocatable: false,
Schema: "",
Extra: make(map[string]string),
}
lines := strings.Split(strings.ReplaceAll(string(data), "\r", ""), "\n")
for _, originalLine := range lines {
line := strings.TrimSpace(originalLine)
if commentIdx := strings.Index(line, "#"); commentIdx != -1 {
line = line[:commentIdx]
}
// Line may be empty if it only contained a comment
if len(line) == 0 {
continue
}
equalsSplit := strings.Index(line, "=")
if equalsSplit == -1 {
return fmt.Errorf("malformed `%s.control`:\n%s", extFile.Name, string(data))
}
name := strings.ToLower(strings.TrimSpace(line[:equalsSplit]))
value := line[equalsSplit+1:]
switch name {
case "directory":
extFile.Control.Directory = removeStringQuotations(value)
case "default_version":
value = removeStringQuotations(value)
separator := strings.Index(value, ".")
if separator == -1 {
return fmt.Errorf("malformed `%s.control` line:\n%s", extFile.Name, originalLine)
}
major, err := strconv.Atoi(value[:separator])
if err != nil {
return fmt.Errorf("malformed `%s.control` line:\n%s", extFile.Name, originalLine)
}
minor, err := strconv.Atoi(value[separator+1:])
if err != nil {
return fmt.Errorf("malformed `%s.control` line:\n%s", extFile.Name, originalLine)
}
extFile.Control.DefaultVersion = ToVersion(uint16(major), uint16(minor))
case "comment":
extFile.Control.Comment = removeStringQuotations(value)
case "encoding":
extFile.Control.Encoding = removeStringQuotations(value)
case "module_pathname":
extFile.Control.ModulePathname = removeStringQuotations(value)
case "requires":
value = removeStringQuotations(value)
var entries []string
for _, entry := range strings.Split(value, ",") {
entries = append(entries, strings.TrimSpace(entry))
}
extFile.Control.Requires = entries
case "superuser", "trusted", "relocatable":
value = strings.ToLower(strings.TrimSpace(value))
var boolValue bool
if value == "true" {
boolValue = true
} else if value == "false" {
boolValue = false
} else {
return fmt.Errorf("malformed `%s.control` line:\n%s", extFile.Name, originalLine)
}
switch name {
case "superuser":
extFile.Control.Superuser = boolValue
case "trusted":
extFile.Control.Trusted = boolValue
case "relocatable":
extFile.Control.Relocatable = boolValue
}
case "schema":
extFile.Control.Schema = removeStringQuotations(value)
default:
extFile.Control.Extra[name] = value
}
}
return nil
}
// LoadSQLFiles loads the contents of the SQL files used by the extension. These will be in the order that they need to
// be executed.
func (extFile *ExtensionFiles) LoadSQLFiles() ([]string, error) {
sqlFiles := make([]string, len(extFile.SQLFileNames))
for i, sqlFileName := range extFile.SQLFileNames {
data, err := os.ReadFile(fmt.Sprintf("%s/%s", extFile.ControlFileDir, sqlFileName))
if err != nil {
return nil, err
}
sqlFiles[i] = string(data)
}
return sqlFiles, nil
}
// LoadSQLFunctionNames loads all of the library function names that are used by the extension.
func (extFile *ExtensionFiles) LoadSQLFunctionNames() ([]string, error) {
funcNames := make(map[string]struct{})
for _, sqlFileName := range extFile.SQLFileNames {
data, err := os.ReadFile(fmt.Sprintf("%s/%s", extFile.ControlFileDir, sqlFileName))
if err != nil {
return nil, err
}
fileRemaining := string(data)
OuterLoop:
for {
// We want to advance the file to the start of the next CREATE FUNCTION if one is present
startIdx := createFunctionStart.FindStringIndex(fileRemaining)
if startIdx == nil {
break
}
fileRemaining = fileRemaining[startIdx[0]:]
// We capture the ending semicolon so the regex doesn't match beyond the function definition's boundaries.
endIdx := strings.IndexRune(fileRemaining, ';')
if endIdx == -1 {
break
}
matches := sqlFunctionCapture.FindStringSubmatch(fileRemaining[:endIdx+1])
switch len(matches) {
case 0:
break OuterLoop
case 4:
if len(matches[2]) > 0 {
funcNames[matches[2]] = struct{}{}
} else if len(matches[3]) > 0 {
funcNames[matches[3]] = struct{}{}
} else {
funcNames[matches[1]] = struct{}{}
}
default:
return nil, fmt.Errorf("invalid CREATE FUNCTION string: %s", string(data))
}
// We nudge it forward to guarantee that our next CREATE FUNCTION search will grab the next one
fileRemaining = fileRemaining[6:]
}
}
sortedFuncNames := slices.Sorted(maps.Keys(funcNames))
return sortedFuncNames, nil
}
// LoadLibrary loads the extension as a library.
func (extFile *ExtensionFiles) LoadLibrary() (*Library, error) {
if len(extFile.LibraryFileName) == 0 {
return nil, fmt.Errorf("extension `%s` does not reference a library", extFile.Name)
}
funcNames, err := extFile.LoadSQLFunctionNames()
if err != nil {
return nil, err
}
return LoadLibrary(fmt.Sprintf("%s/%s", extFile.LibraryFileDir, extFile.LibraryFileName), funcNames)
}
// ToVersion creates a version from the given major and minor version numbers.
func ToVersion(major uint16, minor uint16) Version {
return (Version(major) << 16) + Version(minor)
}
// Major returns the encoded major version number.
func (v Version) Major() uint16 {
return uint16(v >> 16)
}
// Minor returns the encoded minor version number.
func (v Version) Minor() uint16 {
return uint16(v)
}
// String returns the version in the `major.minor` format.
func (v Version) String() string {
return fmt.Sprintf("%d.%d", v.Major(), v.Minor())
}
// DecodeFilenameVersions decodes the version information within the file name. The `sqlFileName` should be the full
// file name (excluding the path), and the SQL file name should contain only the name as
func DecodeFilenameVersions(name string, fileName string) FilenameVersions {
var versionSubsection string
if strings.HasSuffix(fileName, ".sql") {
versionSubsection = strings.TrimSuffix(fileName[len(name)+2: /* We add 2 to account for the -- */], ".sql")
} else if strings.HasSuffix(fileName, ".control") {
versionSubsection = strings.TrimSuffix(fileName[len(name)+2: /* We add 2 to account for the -- */], ".control")
} else {
// The given name is not a .SQL or .CONTROL file, so we'll just return
return FilenameVersions{}
}
var from, to string
if dashIdx := strings.Index(versionSubsection, "--"); dashIdx == -1 {
from = versionSubsection
to = versionSubsection
} else {
from = versionSubsection[:dashIdx]
to = versionSubsection[dashIdx+2:]
}
fromSplit := strings.Index(from, ".")
toSplit := strings.Index(to, ".")
if fromSplit == -1 || toSplit == -1 {
return FilenameVersions{}
}
fromMajor, err := strconv.Atoi(from[:fromSplit])
if err != nil {
return FilenameVersions{}
}
fromMinor, err := strconv.Atoi(from[fromSplit+1:])
if err != nil {
return FilenameVersions{}
}
toMajor, err := strconv.Atoi(to[:toSplit])
if err != nil {
return FilenameVersions{}
}
toMinor, err := strconv.Atoi(to[toSplit+1:])
if err != nil {
return FilenameVersions{}
}
return FilenameVersions{
From: ToVersion(uint16(fromMajor), uint16(fromMinor)),
To: ToVersion(uint16(toMajor), uint16(toMinor)),
}
}
// removeStringQuotations removes the single quotes that are used to specify that a value is a string.
func removeStringQuotations(str string) string {
str = strings.TrimSpace(str)
if strings.HasPrefix(str, "'") {
return (str[:len(str)-1])[1:]
}
return str
}
@@ -0,0 +1,40 @@
// Copyright 2025 Dolthub, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package pg_extension
import (
"bytes"
"os/exec"
"strings"
)
// PostgresDirectories returns the installation directories of a local Postgres instance.
func PostgresDirectories() (libDir string, extensionDir string, err error) {
var buffer bytes.Buffer
cmd := exec.Command("pg_config", "--pkglibdir")
cmd.Stdout = &buffer
if err := cmd.Run(); err != nil {
return "", "", err
}
libDir = strings.TrimSpace(buffer.String())
buffer.Reset()
cmd = exec.Command("pg_config", "--sharedir")
cmd.Stdout = &buffer
if err := cmd.Run(); err != nil {
return "", "", err
}
extensionDir = strings.TrimSpace(buffer.String()) + "/extension"
return libDir, extensionDir, nil
}
@@ -0,0 +1,58 @@
$defFile = "postgres.def"
$outDir = Resolve-Path '..\output' -ErrorAction SilentlyContinue -ErrorVariable _dummy
if (-not $outDir) { $outDir = (New-Item -ItemType Directory -Path '..\output').FullName }
$outFile = Join-Path $outDir 'postgres.exe'
function TryVS {
$vswhere = "$Env:ProgramFiles (x86)\Microsoft Visual Studio\Installer\vswhere.exe"
if (-not (Test-Path $vswhere)) { return $false }
$vsRoot = & $vswhere -latest `
-products * `
-requires Microsoft.VisualStudio.Component.VC.Tools.x86.x64 `
-property installationPath |
Select-Object -First 1
if (-not $vsRoot) { return $false }
$msvcDir = Get-ChildItem -Path (Join-Path $vsRoot 'VC\Tools\MSVC') |
Sort-Object Name -Descending |
Select-Object -First 1
$linkExe = Join-Path $msvcDir.FullName 'bin\Hostx64\x64\link.exe'
& cmd /c "`"$vsRoot\VC\Auxiliary\Build\vcvars64.bat`" >nul `&`& `"$linkExe`" /DLL /NOENTRY /DEF:$defFile /OUT:`"$outFile`""
return $true
}
function TryGCC {
$gcc = (& where.exe gcc.exe 2>$null | Select-Object -First 1)
if (-not $gcc) { return $false }
$args = @(
"-shared",
"-nostdlib",
$defFile,
"-o", $outFile
)
& $gcc @args
return $true
}
function TryClang {
$lld = (& where.exe lld-link.exe 2>$null | Select-Object -First 1)
if ($lld) {
& $lld /DLL /NOENTRY /DEF:$defFile /OUT:"$outFile"
return $true
}
$clang = (& where.exe clang.exe 2>$null | Select-Object -First 1)
if (-not $clang) { return $false }
$args = @(
"-shared",
"-nostdlib",
$defFile,
"-o", $outFile
)
& $clang @args
return $true
}
if (TryVS) { Write-Host "Definition file built using Visual Studio"; exit 0 }
if (TryGCC) { Write-Host "Definition file built using GCC"; exit 0 }
if (TryClang) { Write-Host "Definition file built using Clang"; exit 0 }
throw "Could not build the definition file"
+40
View File
@@ -0,0 +1,40 @@
#!/usr/bin/env bash
set -euo pipefail
cd "$(dirname "$0")"
case "$(go env GOOS)" in
windows) ext="dll" ;;
darwin) ext="dylib" ;;
*) ext="so" ;;
esac
# For now, other platforms directly embed the library, but removing this check will allow them to build a shared library as well
if [[ "$(go env GOOS)" != "windows" ]]; then
exit 0
fi
# MacOS requires that exported functions are present in the calling binary, but other platforms use a dynamic library.
# To account for this, we put the exported functions in a normal package by default.
# For MacOS, this works just fine as we import the package.
# To make a dynamic library, we copy the files into a temporary directory and modify the files to create a valid library.
# This lets us use the same code for both scenarios.
mkdir -p temp_lib
trap 'rm -rf temp_lib' EXIT
cp ./*.* ./temp_lib
# For Windows, we also need to build the definition file
if [[ "$(go env GOOS)" == "windows" ]]; then
powershell.exe -File "build_definitions.ps1"
fi
for f in temp_lib/*.go; do
sed 's/^package extension_cgo$/package main/' "$f" > "$f".tmp
mv "$f".tmp "$f"
done
printf "module github.com/dolthub/pg_extension\n\ngo 1.24" > ./temp_lib/go.mod
(
cd temp_lib
CGO_ENABLED=1 go build -buildmode=c-shared -o "../../output/pg_extension.${ext}" .
)
@@ -0,0 +1,57 @@
// Copyright 2025 Dolthub, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include <stdarg.h>
#include <stdio.h>
#include <stdbool.h>
#if defined(_WIN32) || defined(_WIN64)
#define DLLEXPORT __declspec(dllexport)
#else
#define DLLEXPORT __attribute__((visibility("default")))
#endif
static char last_error[512];
DLLEXPORT bool errstart(int elevel, const char* domain) {
last_error[0] = '\0';
return 1;
}
DLLEXPORT bool errstart_cold(int elevel, const char *domain) {
return errstart(elevel, domain);
}
DLLEXPORT int errmsg(const char *fmt, ...) {
va_list ap;
va_start(ap, fmt);
vsnprintf(last_error, sizeof(last_error), fmt, ap);
va_end(ap);
return 0;
}
DLLEXPORT int errmsg_internal(const char *fmt, ...) {
va_list ap;
va_start(ap, fmt);
vsnprintf(last_error, sizeof(last_error), fmt, ap);
va_end(ap);
return 0;
}
DLLEXPORT int errfinish(int dummy, ...) {
if (last_error[0]) {
fprintf(stderr, "Postgres ERROR: %s\n", last_error);
}
return 0;
}
@@ -0,0 +1,145 @@
// Copyright 2025 Dolthub, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package extension_cgo
/*
#include "exports.h"
static inline Datum FunctionPassthrough(PGFunction f, FunctionCallInfoBaseData *fcinfo) {
return (*f)(fcinfo);
}
*/
import "C"
import (
"encoding/hex"
"fmt"
"os"
"unsafe"
)
func main() {}
//export errcode
func errcode(code C.int) C.int {
return code
}
//export palloc
func palloc(sz C.size_t) unsafe.Pointer {
// TODO: should track this pointer so we know to free it later
return C.malloc(sz)
}
//export palloc0
func palloc0(sz C.size_t) unsafe.Pointer {
// TODO: should track this pointer so we know to free it later
ptr := C.malloc(sz)
if ptr != nil {
C.memset(ptr, 0, sz)
}
return ptr
}
//export MemoryContextAlloc
func MemoryContextAlloc(c unsafe.Pointer, sz C.size_t) unsafe.Pointer {
// TODO: should track this pointer so we know to free it later, could use the memory context
return C.malloc(sz)
}
//export MemoryContextAllocExtended
func MemoryContextAllocExtended(c unsafe.Pointer, sz C.size_t, f C.int) unsafe.Pointer {
// TODO: should track this pointer so we know to free it later, could use the memory context
return C.malloc(sz)
}
//export pg_detoast_datum_packed
func pg_detoast_datum_packed(d unsafe.Pointer) unsafe.Pointer {
return d
}
//export text_to_cstring
func text_to_cstring(t unsafe.Pointer) *C.char {
return C.CString(C.GoString((*C.char)(t)))
}
//export uuid_in
func uuid_in(fc C.FunctionCallInfo) C.Datum {
uuidInputStr := (*C.pgext_const_char)(unsafe.Pointer(uintptr(fc.args[0].value)))
uuidInputBytes := decodeUuidStr([]byte(C.GoString(uuidInputStr)))
outputBytes := (*C.pgext_unsigned_char)(C.malloc(C.size_t(len(uuidInputBytes))))
C.memcpy(unsafe.Pointer(outputBytes), unsafe.Pointer(&uuidInputBytes[0]), C.size_t(len(uuidInputBytes)))
return C.Datum(uintptr(unsafe.Pointer(outputBytes)))
}
// decodeUuidStr is a helper function for uuid_in, which converts the given byte slice to the static array representation.
func decodeUuidStr(strBytes []byte) [16]byte {
if strBytes[8] != '-' || strBytes[13] != '-' || strBytes[18] != '-' || strBytes[23] != '-' {
return [16]byte{255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255}
}
u := [16]byte{}
src := strBytes
dst := u[:]
for i, byteGroup := range []int{8, 4, 4, 4, 12} {
if i > 0 {
src = src[1:]
}
_, err := hex.Decode(dst[:byteGroup/2], src[:byteGroup])
if err != nil {
return [16]byte{255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255}
}
src = src[byteGroup:]
dst = dst[byteGroup/2:]
}
return u
}
//export uuid_out
func uuid_out(ptr unsafe.Pointer) C.Datum {
uuidInputBytes := C.GoBytes(ptr, 16)
textBuffer := make([]byte, 36)
_ = hex.Encode(textBuffer[0:8], uuidInputBytes[0:4])
textBuffer[8] = '-'
_ = hex.Encode(textBuffer[9:13], uuidInputBytes[4:6])
textBuffer[13] = '-'
_ = hex.Encode(textBuffer[14:18], uuidInputBytes[6:8])
textBuffer[18] = '-'
_ = hex.Encode(textBuffer[19:23], uuidInputBytes[8:10])
textBuffer[23] = '-'
_ = hex.Encode(textBuffer[24:], uuidInputBytes[10:])
return C.Datum(uintptr(unsafe.Pointer(C.CString(string(textBuffer)))))
}
//export DirectFunctionCall1Coll
func DirectFunctionCall1Coll(fn unsafe.Pointer, collation C.uint32_t, arg1 C.Datum) C.Datum {
fc := (*C.FunctionCallInfoBaseData)(C.malloc(C.SZ_FCINFO))
if fc == nil {
_, _ = fmt.Fprintln(os.Stderr, "DirectFunctionCall1Coll: out of memory")
return 0
}
defer C.free(unsafe.Pointer(fc))
C.memset(unsafe.Pointer(fc), 0, C.SZ_FCINFO)
fc.isnull = false
fc.fncollation = collation
fc.nargs = 1
fc.args[0].value = arg1
fc.args[0].isnull = false
result := C.FunctionPassthrough(C.PGFunction(fn), fc)
if fc.isnull {
_, _ = fmt.Fprintf(os.Stderr, "function %p returned NULL\n", fn)
}
return result
}
@@ -0,0 +1,68 @@
// Copyright 2025 Dolthub, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#ifndef PG_EXT_EXPORTS_H
#define PG_EXT_EXPORTS_H
#include <stdlib.h>
#include <string.h>
#include <stdint.h>
#include <stdarg.h>
#include <stdio.h>
#include <stdbool.h>
// This doesn't compile unless it has a value, but Postgres defines this as an empty value intentionally
#define FLEXIBLE_ARRAY_MEMBER 8
typedef uintptr_t Datum;
typedef struct FunctionCallInfoBaseData* FunctionCallInfo;
typedef Datum (*PGFunction) (FunctionCallInfo fcinfo);
typedef struct NullableDatum {
Datum value;
bool isnull;
} NullableDatum;
typedef struct FmgrInfo {
void* fn_addr;
uint32_t fn_oid;
short fn_nargs;
bool fn_strict;
bool fn_retset;
unsigned char fn_stats;
void* fn_extra;
void* fn_mcxt;
void* fn_expr;
} FmgrInfo;
typedef struct FunctionCallInfoBaseData {
FmgrInfo* flinfo;
void* context;
void* resultinfo;
uint32_t fncollation;
bool isnull;
short nargs;
NullableDatum args[FLEXIBLE_ARRAY_MEMBER];
} FunctionCallInfoBaseData;
enum {
SZ_FMGRINFO = sizeof(FmgrInfo),
SZ_FCINFO = sizeof(FunctionCallInfoBaseData)
};
typedef const char pgext_const_char;
typedef unsigned char pgext_unsigned_char;
typedef const uint8_t pgext_const_uint8;
#endif //PG_EXT_EXPORTS_H
@@ -0,0 +1,132 @@
// Copyright 2025 Dolthub, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package extension_cgo
/*
#include "exports.h"
typedef enum
{
PG_MD5 = 0,
PG_SHA1,
PG_SHA224,
PG_SHA256,
PG_SHA384,
PG_SHA512,
} pg_cryptohash_type;
typedef struct pg_cryptohash_ctx {
pg_cryptohash_type hashType;
} pg_cryptohash_ctx;
*/
import "C"
import (
"crypto/md5"
"crypto/sha1"
"crypto/sha256"
"crypto/sha512"
"hash"
"sync"
"unsafe"
)
var pg_cryptohash_store sync.Map
//export pg_cryptohash_create
func pg_cryptohash_create(typ C.pg_cryptohash_type) *C.pg_cryptohash_ctx {
ctx := (*C.pg_cryptohash_ctx)(C.malloc(C.size_t(unsafe.Sizeof(C.pg_cryptohash_ctx{}))))
ctx.hashType = typ
ctxPtr := uintptr(unsafe.Pointer(ctx))
switch typ {
case 1:
pg_cryptohash_store.Store(ctxPtr, sha1.New())
case C.PG_SHA224:
pg_cryptohash_store.Store(ctxPtr, sha512.New512_224())
case C.PG_SHA256:
pg_cryptohash_store.Store(ctxPtr, sha256.New())
case C.PG_SHA384:
pg_cryptohash_store.Store(ctxPtr, sha512.New384())
case C.PG_SHA512:
pg_cryptohash_store.Store(ctxPtr, sha512.New())
default:
// Default to MD5
pg_cryptohash_store.Store(ctxPtr, md5.New())
}
return ctx
}
//export pg_cryptohash_init
func pg_cryptohash_init(ctx *C.pg_cryptohash_ctx) C.int {
if ctx == nil {
return -1
}
return 0
}
//export pg_cryptohash_update
func pg_cryptohash_update(ctx *C.pg_cryptohash_ctx, data *C.pgext_const_uint8, len C.size_t) C.int {
if ctx == nil {
return -1
}
if len == 0 {
return 0
}
ctxPtr := uintptr(unsafe.Pointer(ctx))
storedHashAny, ok := pg_cryptohash_store.Load(ctxPtr)
if !ok {
return -1
}
storedHash := storedHashAny.(hash.Hash)
dataSlice := unsafe.Slice((*byte)(unsafe.Pointer(data)), int(len))
if _, err := storedHash.Write(dataSlice); err != nil {
return 1
}
return 0
}
//export pg_cryptohash_final
func pg_cryptohash_final(ctx *C.pg_cryptohash_ctx, dest *C.uint8_t, destLen C.size_t) C.int {
if ctx == nil {
return -1
}
ctxPtr := uintptr(unsafe.Pointer(ctx))
storedHashAny, ok := pg_cryptohash_store.Load(ctxPtr)
if !ok {
return -1
}
storedHash := storedHashAny.(hash.Hash)
sum := storedHash.Sum(nil)
destSlice := unsafe.Slice((*byte)(unsafe.Pointer(dest)), int(destLen))
// If the destination slice is too small, then it's invalid
if len(sum) > len(destSlice) {
return -1
}
copy(destSlice, sum)
return 0
}
//export pg_cryptohash_free
func pg_cryptohash_free(ctx *C.pg_cryptohash_ctx) {
if ctx != nil {
ctxPtr := uintptr(unsafe.Pointer(ctx))
pg_cryptohash_store.Delete(ctxPtr)
C.free(unsafe.Pointer(ctx))
}
}
//export pg_cryptohash_error
func pg_cryptohash_error(ctx *C.pg_cryptohash_ctx) *C.pgext_const_char {
return C.CString("")
}
@@ -0,0 +1,26 @@
LIBRARY "postgres.exe"
EXPORTS
; ---- functions ----
DirectFunctionCall1Coll = pg_extension.DirectFunctionCall1Coll
errcode = pg_extension.errcode
errfinish = pg_extension.errfinish
errmsg = pg_extension.errmsg
errmsg_internal = pg_extension.errmsg_internal
errstart = pg_extension.errstart
errstart_cold = pg_extension.errstart_cold
MemoryContextAlloc = pg_extension.MemoryContextAlloc
MemoryContextAllocExtended = pg_extension.MemoryContextAllocExtended
palloc = pg_extension.palloc
palloc0 = pg_extension.palloc0
palloc_extended = pg_extension.palloc_extended
pg_cryptohash_create = pg_extension.pg_cryptohash_create
pg_cryptohash_error = pg_extension.pg_cryptohash_error
pg_cryptohash_final = pg_extension.pg_cryptohash_final
pg_cryptohash_free = pg_extension.pg_cryptohash_free
pg_cryptohash_init = pg_extension.pg_cryptohash_init
pg_cryptohash_update = pg_extension.pg_cryptohash_update
pg_detoast_datum_packed = pg_extension.pg_detoast_datum_packed
strlcpy = pg_extension.strlcpy
text_to_cstring = pg_extension.text_to_cstring
uuid_in = pg_extension.uuid_in
uuid_out = pg_extension.uuid_out
@@ -0,0 +1,45 @@
// Copyright 2025 Dolthub, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//go:build !darwin
package extension_cgo
/*
#include "exports.h"
*/
import "C"
import "unsafe"
//export strlcpy
func strlcpy(dst *C.char, src *C.pgext_const_char, size C.size_t) C.size_t {
var srcLen C.size_t
for {
if *(*C.char)(unsafe.Pointer(uintptr(unsafe.Pointer(src)) + uintptr(srcLen))) == 0 {
break
}
srcLen++
}
if size != 0 {
n := srcLen
if n >= size {
n = size - 1
}
dstSlice := unsafe.Slice((*byte)(unsafe.Pointer(dst)), int(n+1))
srcSlice := unsafe.Slice((*byte)(unsafe.Pointer(src)), int(n))
copy(dstSlice, srcSlice)
dstSlice[n] = 0
}
return srcLen
}
@@ -0,0 +1,26 @@
// Copyright 2025 Dolthub, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//go:build darwin
package extension_cgo
/*
#include "exports.h"
*/
import "C"
func strlcpy(dst *C.char, src *C.pgext_const_char, size C.size_t) C.size_t {
return C.strlcpy(dst, src, size)
}
@@ -0,0 +1,120 @@
// Copyright 2025 Dolthub, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package pg_extension
import (
"fmt"
"sync"
)
// Library is a fully-loaded extension library.
type Library struct {
Magic PgMagicStruct
Funcs map[string]Function
Version Version
internal InternalLoadedLibrary
}
// InternalLoadedLibrary is an interface that is implemented by the specific platform to handle library operations.
type InternalLoadedLibrary interface {
Lookup(sym string) (uintptr, error)
Close() error
}
// Function represents an internal library function.
type Function struct {
Name string
Ptr uintptr
Args []int
APIVersion int
// TODO: return type?
}
// PgFunctionInfo is a stand-in for the C struct that reports the function information.
type PgFunctionInfo struct {
APIVersion int32
}
// PgMagicStruct is a stand-in for the C struct that reports the information of the library.
type PgMagicStruct struct {
Len int32
Version int32
FuncMaxArgs int32
IndexMaxKeys int32
NameDataLen int32
Float4ByVal int32
Float8ByVal int32
}
var (
// loadedLibraries contains all of the loaded libraries.
// TODO: need to close all of these before the program ends
loadedLibraries = make(map[string]*Library)
// loadedLibrariesMutex gates access to the cached libraries.
loadedLibrariesMutex = &sync.Mutex{}
)
// LoadLibrary loads the library of the extension, along with preloading all of the functions given.
func LoadLibrary(path string, funcNames []string) (*Library, error) {
loadedLibrariesMutex.Lock()
defer loadedLibrariesMutex.Unlock()
if lib, ok := loadedLibraries[path]; ok {
return lib, nil
}
internalLib, err := loadLibraryInternal(path)
if err != nil {
return nil, err
}
magicPtr, err := internalLib.Lookup("Pg_magic_func")
if err != nil {
return nil, err
}
// We don't free the magic struct since it's a pointer to static memory
magicStructDatum, isNotNull := CallFmgrFunction(magicPtr)
if !isNotNull {
return nil, fmt.Errorf("unable to find magic function for `%s`", path)
}
magicStruct := *(FromDatum[PgMagicStruct](magicStructDatum))
lib := &Library{
Magic: magicStruct,
Funcs: make(map[string]Function),
internal: internalLib,
}
for _, funcName := range funcNames {
finfoPtr, err := internalLib.Lookup(fmt.Sprintf("pg_finfo_%s", funcName))
if err != nil {
return nil, err
}
// We don't free finfo since it's a pointer to static memory
finfoDatum, isNotNull := CallFmgrFunction(finfoPtr)
apiVersion := 0
if isNotNull {
apiVersion = int(FromDatum[PgFunctionInfo](finfoDatum).APIVersion)
}
funcPtr, err := internalLib.Lookup(funcName)
if err != nil {
return nil, err
}
lib.Funcs[funcName] = Function{
Name: funcName,
Ptr: funcPtr,
Args: nil,
APIVersion: apiVersion,
}
}
loadedLibraries[path] = lib
return lib, nil
}
@@ -0,0 +1,77 @@
// Copyright 2025 Dolthub, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//go:build darwin
package pg_extension
/*
#cgo LDFLAGS: -ldl
#include <dlfcn.h>
#include <stdlib.h>
*/
import "C"
import (
"fmt"
"unsafe"
_ "github.com/dolthub/doltgresql/core/extensions/pg_extension/library"
)
// PLATFORM specifies which platform applies to the current library loader. This will always be a three-letter string.
const PLATFORM = "MAC"
// darwinLib is the Linux-specific implementation of InternalLoadedLibrary.
type darwinLib struct {
path string
handle unsafe.Pointer
}
var _ InternalLoadedLibrary = (*darwinLib)(nil)
// loadLibraryInternal handles the loading of an extension's SO.
func loadLibraryInternal(path string) (InternalLoadedLibrary, error) {
pathC := C.CString(path)
defer C.free(unsafe.Pointer(pathC))
handle := C.dlopen(pathC, C.RTLD_LAZY|C.RTLD_GLOBAL)
if handle == nil {
return nil, fmt.Errorf("error while loading extension `%s`\n%s", path, C.GoString(C.dlerror()))
}
return &darwinLib{
path: path,
handle: handle,
}, nil
}
// Lookup implements the interface InternalLoadedLibrary.
func (u *darwinLib) Lookup(sym string) (uintptr, error) {
symC := C.CString(sym)
defer C.free(unsafe.Pointer(symC))
ptr := C.dlsym(u.handle, symC)
if ptr == nil {
return 0, fmt.Errorf("symbol %s not found", sym)
}
return uintptr(ptr), nil
}
// Close implements the interface InternalLoadedLibrary.
func (u *darwinLib) Close() error {
if C.dlclose(u.handle) != 0 {
return fmt.Errorf("error while closing extension `%s`\n%s", u.path, C.GoString(C.dlerror()))
}
return nil
}
@@ -0,0 +1,78 @@
// Copyright 2025 Dolthub, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//go:build linux
package pg_extension
/*
#cgo LDFLAGS: -ldl
#cgo LDFLAGS: -Wl,-E
#include <dlfcn.h>
#include <stdlib.h>
*/
import "C"
import (
"fmt"
"unsafe"
_ "github.com/dolthub/doltgresql/core/extensions/pg_extension/library"
)
// PLATFORM specifies which platform applies to the current library loader. This will always be a three-letter string.
const PLATFORM = "LIN"
// unixLib is the Linux-specific implementation of InternalLoadedLibrary.
type unixLib struct {
path string
handle unsafe.Pointer
}
var _ InternalLoadedLibrary = (*unixLib)(nil)
// loadLibraryInternal handles the loading of an extension's SO.
func loadLibraryInternal(path string) (InternalLoadedLibrary, error) {
pathC := C.CString(path)
defer C.free(unsafe.Pointer(pathC))
handle := C.dlopen(pathC, C.RTLD_LAZY|C.RTLD_GLOBAL)
if handle == nil {
return nil, fmt.Errorf("error while loading extension `%s`\n%s", path, C.GoString(C.dlerror()))
}
return &unixLib{
path: path,
handle: handle,
}, nil
}
// Lookup implements the interface InternalLoadedLibrary.
func (u *unixLib) Lookup(sym string) (uintptr, error) {
symC := C.CString(sym)
defer C.free(unsafe.Pointer(symC))
ptr := C.dlsym(u.handle, symC)
if ptr == nil {
return 0, fmt.Errorf("symbol %s not found", sym)
}
return uintptr(ptr), nil
}
// Close implements the interface InternalLoadedLibrary.
func (u *unixLib) Close() error {
if C.dlclose(u.handle) != 0 {
return fmt.Errorf("error while closing extension `%s`\n%s", u.path, C.GoString(C.dlerror()))
}
return nil
}
@@ -0,0 +1,136 @@
// Copyright 2025 Dolthub, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//go:build windows
package pg_extension
import (
"bytes"
"crypto/sha256"
_ "embed"
"fmt"
"io"
"os"
"path/filepath"
"runtime"
"sync"
"syscall"
"unsafe"
)
// PLATFORM specifies which platform applies to the current library loader. This will always be a three-letter string.
const PLATFORM = "WIN"
//go:embed output/postgres.exe
var libDefBytes []byte
//go:embed output/pg_extension.dll
var dllBytes []byte
// winLib is the Windows-specific implementation of InternalLoadedLibrary.
type winLib struct{ dll syscall.Handle }
var _ InternalLoadedLibrary = (*winLib)(nil)
var addPGBinDir = &sync.Once{}
// loadLibraryInternal handles the loading of an extension's DLL.
func loadLibraryInternal(path string) (InternalLoadedLibrary, error) {
addPGBinDir.Do(func() {
_, currentFileLocation, _, ok := runtime.Caller(0)
if !ok || len(currentFileLocation) == 0 {
panic("cannot find the directory where this file exists")
}
// There are three scenarios that we need to consider when attempting to load the DLL:
// 1) The DLL exists in an output folder (this will be true for development)
// 2) The DLL exists alongside the binary
// 3) The DLL does not exist alongside the binary (or is the wrong version)
// In the third situation, we write the contained DLL and definition file alongside the binary, so that we'll
// always end up in the second situation. This enables both developmental and deployment workflows without
// explicit configuration.
var dllDir string
if _, err := os.Stat(filepath.Join(filepath.Dir(currentFileLocation), "output", "postgres.exe")); err == nil {
dllDir = filepath.Join(filepath.Dir(currentFileLocation), "output")
} else {
currentBinaryLocation, err := os.Executable()
if err != nil {
panic(fmt.Errorf("cannot find where the executable was launched:\n%s", err.Error()))
}
dllDir = filepath.Dir(currentBinaryLocation)
shouldWriteFiles := false
if _, err := os.Stat(filepath.Join(dllDir, "postgres.exe")); err != nil {
shouldWriteFiles = true
} else {
func() {
// If the DLL hash doesn't match our hash, then we overwrite it
extDll, err := os.Open(filepath.Join(filepath.Dir(currentBinaryLocation), "pg_extension.dll"))
if err != nil {
shouldWriteFiles = true
return
}
defer func() {
_ = extDll.Close()
}()
dllSha := sha256.Sum256(dllBytes)
extDllSha := sha256.New()
_, _ = io.Copy(extDllSha, extDll)
shouldWriteFiles = !bytes.Equal(extDllSha.Sum(nil), dllSha[:])
}()
}
if shouldWriteFiles {
writeLocation := filepath.Dir(currentBinaryLocation)
_ = os.WriteFile(filepath.Join(writeLocation, "postgres.exe"), libDefBytes, 0755)
_ = os.WriteFile(filepath.Join(writeLocation, "pg_extension.dll"), dllBytes, 0755)
}
}
dirPtr, err := syscall.UTF16PtrFromString(dllDir)
if err != nil {
panic(err)
}
_, _, _ = syscall.MustLoadDLL("kernel32.dll").MustFindProc("SetDllDirectoryW").Call(uintptr(unsafe.Pointer(dirPtr)))
_, _ = syscall.LoadLibrary(filepath.Join(dllDir, "pg_extension.dll"))
})
d, err := syscall.LoadLibrary(path)
if err != nil {
return nil, err
}
return &winLib{dll: d}, nil
}
// Lookup implements the interface InternalLoadedLibrary.
func (w *winLib) Lookup(sym string) (uintptr, error) {
candidates := []string{
sym,
"_" + sym,
sym + "@0",
"_" + sym + "@0",
}
for bytes := 4; bytes <= 64; bytes += 4 {
candidates = append(candidates,
fmt.Sprintf("%s@%d", sym, bytes),
fmt.Sprintf("_%s@%d", sym, bytes))
}
for _, name := range candidates {
if p, err := syscall.GetProcAddress(w.dll, name); err == nil {
return p, nil
}
}
return 0, fmt.Errorf("symbol %s not found", sym)
}
// Close implements the interface InternalLoadedLibrary.
func (w *winLib) Close() error {
return syscall.FreeLibrary(w.dll)
}
+88
View File
@@ -0,0 +1,88 @@
// Copyright 2025 Dolthub, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package pg_extension
/*
#cgo CFLAGS: "-I${SRCDIR}/library"
#include "exports.h"
*/
import "C"
import "unsafe"
// FromDatum converts the given datum to the type.
func FromDatum[T any](d Datum) *T {
if d == 0 {
return nil
}
return (*T)(unsafe.Pointer(d))
}
// FromDatumGoString converts the given datum to a string.
func FromDatumGoString(d Datum) string {
if d == 0 {
return ""
}
return C.GoString((*C.char)(unsafe.Pointer(d)))
}
// FromDatumGoBytes converts the given datum to a byte array of length N.
func FromDatumGoBytes(d Datum, n uint) []byte {
if d == 0 {
return []byte{}
}
return C.GoBytes(unsafe.Pointer(d), C.int(n))
}
// ToDatum converts the given pointer to a Datum.
func ToDatum[T any](val *T) Datum {
if val == nil {
return 0
}
return Datum(unsafe.Pointer(val))
}
// ToDatumGoString converts the given string to a Datum.
func ToDatumGoString(str string) Datum {
return Datum(unsafe.Pointer(C.CString(str)))
}
// ToDatumGoBytes converts the given byte slice to a Datum.
func ToDatumGoBytes(data []byte) Datum {
return Datum(unsafe.Pointer(C.CBytes(data)))
}
// Malloc allocates the given type within the C heap. These should always be followed up with a Free at some point
// afterward.
func Malloc[T any]() *T {
var structToDetermineSize T
return (*T)(C.malloc(C.size_t(unsafe.Sizeof(structToDetermineSize))))
}
// ZeroMemory writes all zeroes to the memory location occupied by the given pointer.
func ZeroMemory[T any](val *T) {
var structToDetermineSize T
C.memset(unsafe.Pointer(val), 0, C.size_t(unsafe.Sizeof(structToDetermineSize)))
}
// Free frees the given pointer from C heap. Generally, this is paired with a pointer returned from Malloc.
func Free[T any](val *T) {
C.free(unsafe.Pointer(val))
}
// FreeDatum frees the given Datum. Care should be exercised as datums may refer to static memory, and attempting to
// free static memory will result in a crash.
func FreeDatum(val Datum) {
C.free(unsafe.Pointer(val))
}
+138
View File
@@ -0,0 +1,138 @@
// Copyright 2025 Dolthub, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package extensions
import (
"context"
"github.com/cockroachdb/errors"
"github.com/dolthub/dolt/go/libraries/doltcore/doltdb"
"github.com/dolthub/doltgresql/core/id"
"github.com/dolthub/doltgresql/core/rootobject/objinterface"
pgtypes "github.com/dolthub/doltgresql/server/types"
)
// DeserializeRootObject implements the interface objinterface.Collection.
func (pge *Collection) DeserializeRootObject(ctx context.Context, data []byte) (objinterface.RootObject, error) {
return DeserializeExtension(ctx, data)
}
// DiffRootObjects implements the interface objinterface.Collection.
func (pge *Collection) DiffRootObjects(ctx context.Context, fromHash string, ours objinterface.RootObject, theirs objinterface.RootObject, ancestor objinterface.RootObject) ([]objinterface.RootObjectDiff, objinterface.RootObject, error) {
return nil, nil, errors.Errorf("extensions should never produce conflicts")
}
// DropRootObject implements the interface objinterface.Collection.
func (pge *Collection) DropRootObject(ctx context.Context, identifier id.Id) error {
if identifier.Section() != id.Section_Extension {
return errors.Errorf(`extension "%s" does not exist`, identifier.String())
}
return pge.DropLoadedExtension(ctx, id.Extension(identifier))
}
// GetFieldType implements the interface objinterface.Collection.
func (pge *Collection) GetFieldType(ctx context.Context, fieldName string) *pgtypes.DoltgresType {
return nil
}
// GetID implements the interface objinterface.Collection.
func (pge *Collection) GetID() objinterface.RootObjectID {
return objinterface.RootObjectID_Extensions
}
// GetRootObject implements the interface objinterface.Collection.
func (pge *Collection) GetRootObject(ctx context.Context, identifier id.Id) (objinterface.RootObject, bool, error) {
if identifier.Section() != id.Section_Extension {
return nil, false, nil
}
ext, err := pge.GetLoadedExtension(ctx, id.Extension(identifier))
return ext, err == nil && ext.Namespace.IsValid(), err
}
// HasRootObject implements the interface objinterface.Collection.
func (pge *Collection) HasRootObject(ctx context.Context, identifier id.Id) (bool, error) {
if identifier.Section() != id.Section_Extension {
return false, nil
}
return pge.HasLoadedExtension(ctx, id.Extension(identifier)), nil
}
// IDToTableName implements the interface objinterface.Collection.
func (pge *Collection) IDToTableName(identifier id.Id) doltdb.TableName {
if identifier.Section() != id.Section_Extension {
return doltdb.TableName{}
}
return doltdb.TableName{Name: id.Extension(identifier).Name()}
}
// IterAll implements the interface objinterface.Collection.
func (pge *Collection) IterAll(ctx context.Context, callback func(rootObj objinterface.RootObject) (stop bool, err error)) error {
for _, extID := range pge.idCache {
stop, err := callback(pge.accessCache[extID])
if err != nil {
return err
} else if stop {
return nil
}
}
return nil
}
// IterIDs implements the interface objinterface.Collection.
func (pge *Collection) IterIDs(ctx context.Context, callback func(identifier id.Id) (stop bool, err error)) error {
for _, extID := range pge.idCache {
stop, err := callback(extID.AsId())
if err != nil {
return err
} else if stop {
return nil
}
}
return nil
}
// PutRootObject implements the interface objinterface.Collection.
func (pge *Collection) PutRootObject(ctx context.Context, rootObj objinterface.RootObject) error {
ext, ok := rootObj.(Extension)
if !ok {
return errors.Newf("invalid extension root object: %T", rootObj)
}
return pge.AddLoadedExtension(ctx, ext)
}
// RenameRootObject implements the interface objinterface.Collection.
func (pge *Collection) RenameRootObject(ctx context.Context, oldName id.Id, newName id.Id) error {
return errors.New(`extensions cannot be renamed`)
}
// ResolveName implements the interface objinterface.Collection.
func (pge *Collection) ResolveName(ctx context.Context, name doltdb.TableName) (doltdb.TableName, id.Id, error) {
extID := id.NewExtension(name.Name)
if pge.HasLoadedExtension(ctx, extID) {
return doltdb.TableName{Name: name.Name}, extID.AsId(), nil
}
return doltdb.TableName{}, id.Null, nil
}
// TableNameToID implements the interface objinterface.Collection.
func (pge *Collection) TableNameToID(name doltdb.TableName) id.Id {
return id.NewExtension(name.Name).AsId()
}
// UpdateField implements the interface objinterface.Collection.
func (pge *Collection) UpdateField(ctx context.Context, rootObject objinterface.RootObject, fieldName string, newValue any) (objinterface.RootObject, error) {
return nil, errors.New("updating through the conflicts table for this object type is not yet supported")
}
+68
View File
@@ -0,0 +1,68 @@
// Copyright 2025 Dolthub, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package extensions
import (
"context"
"github.com/cockroachdb/errors"
"github.com/dolthub/doltgresql/core/id"
"github.com/dolthub/doltgresql/utils"
)
// Serialize returns the Extension as a byte slice. If the Extension is invalid (invalid ExtName), then this returns a
// nil slice.
func (ext Extension) Serialize(ctx context.Context) ([]byte, error) {
if !ext.ExtName.IsValid() {
return nil, nil
}
// Initialize the writer
writer := utils.NewWriter(256)
writer.VariableUint(0) // Version
// Write the extension data
writer.Id(ext.ExtName.AsId())
writer.Id(ext.Namespace.AsId())
writer.Bool(ext.Relocatable)
writer.String(string(ext.LibIdentifier))
// Returns the data
return writer.Data(), nil
}
// DeserializeExtension returns the Extension that was serialized in the byte slice. Returns an empty Extension (has an
// invalid ID) if data is nil or empty.
func DeserializeExtension(ctx context.Context, data []byte) (Extension, error) {
if len(data) == 0 {
return Extension{}, nil
}
reader := utils.NewReader(data)
version := reader.VariableUint()
if version != 0 {
return Extension{}, errors.Errorf("version %d of extensions are not supported, please upgrade the server", version)
}
// Read from the reader
ext := Extension{}
ext.ExtName = id.Extension(reader.Id())
ext.Namespace = id.Namespace(reader.Id())
ext.Relocatable = reader.Bool()
ext.LibIdentifier = LibraryIdentifier(reader.String())
if !reader.IsEmpty() {
return Extension{}, errors.Errorf("extra data found while deserializing an extension")
}
// Return the deserialized object
return ext, nil
}
+427
View File
@@ -0,0 +1,427 @@
// Copyright 2024 Dolthub, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package functions
import (
"context"
"fmt"
"maps"
"slices"
"strings"
"github.com/cockroachdb/errors"
"github.com/dolthub/dolt/go/libraries/doltcore/doltdb"
"github.com/dolthub/dolt/go/store/hash"
"github.com/dolthub/dolt/go/store/prolly"
"github.com/dolthub/dolt/go/store/prolly/tree"
"github.com/dolthub/doltgresql/core/id"
"github.com/dolthub/doltgresql/core/rootobject/objinterface"
"github.com/dolthub/doltgresql/server/plpgsql"
)
// Collection contains a collection of functions.
type Collection struct {
accessCache map[id.Function]Function // This cache is used for general access when you know the exact ID
overloadCache map[id.Function][]id.Function // This cache is used to find overloads if you know the name
idCache []id.Function // This cache simply contains the name of every function
mapHash hash.Hash // This is cached so that we don't have to calculate the hash every time
underlyingMap prolly.AddressMap
ns tree.NodeStore
}
// Function represents a created function.
type Function struct {
ID id.Function
ReturnType id.Type
ParameterNames []string
ParameterTypes []id.Type
ParameterDefaults []string
Variadic bool
IsNonDeterministic bool
Strict bool
Definition string
ExtensionName string // Only used when this is an extension function
ExtensionSymbol string // Only used when this is an extension function
Operations []plpgsql.InterpreterOperation // Only used when this is a plpgsql language
SQLDefinition string // Only used when this is a sql language
SetOf bool
}
var _ objinterface.Collection = (*Collection)(nil)
var _ objinterface.RootObject = Function{}
// NewCollection returns a new Collection.
func NewCollection(ctx context.Context, underlyingMap prolly.AddressMap, ns tree.NodeStore) (*Collection, error) {
collection := &Collection{
accessCache: make(map[id.Function]Function),
overloadCache: make(map[id.Function][]id.Function),
idCache: nil,
mapHash: hash.Hash{},
underlyingMap: underlyingMap,
ns: ns,
}
return collection, collection.reloadCaches(ctx)
}
// GetFunction returns the function with the given ID. Returns a function with an invalid ID if it cannot be found
// (Function.ID.IsValid() == false).
func (pgf *Collection) GetFunction(ctx context.Context, funcID id.Function) (Function, error) {
if f, ok := pgf.accessCache[funcID]; ok {
return f, nil
}
return Function{}, nil
}
// GetFunctionOverloads returns the overloads for the function matching the schema and the function name. The parameter
// types are ignored when searching for overloads.
func (pgf *Collection) GetFunctionOverloads(ctx context.Context, funcID id.Function) ([]Function, error) {
overloads, ok := pgf.overloadCache[id.NewFunction(funcID.SchemaName(), funcID.FunctionName())]
if !ok || len(overloads) == 0 {
return nil, nil
}
funcs := make([]Function, len(overloads))
for i, overload := range overloads {
funcs[i] = pgf.accessCache[overload]
}
return funcs, nil
}
// HasFunction returns whether the function is present.
func (pgf *Collection) HasFunction(ctx context.Context, funcID id.Function) bool {
_, ok := pgf.accessCache[funcID]
return ok
}
// AddFunction adds a new function.
func (pgf *Collection) AddFunction(ctx context.Context, f Function) error {
// First we'll check to see if it exists
if _, ok := pgf.accessCache[f.ID]; ok {
return errors.Errorf(`function "%s" already exists with same argument types`, f.ID.FunctionName())
}
// Now we'll add the function to our map
data, err := f.Serialize(ctx)
if err != nil {
return err
}
h, err := pgf.ns.WriteBytes(ctx, data)
if err != nil {
return err
}
mapEditor := pgf.underlyingMap.Editor()
if err = mapEditor.Add(ctx, string(f.ID), h); err != nil {
return err
}
newMap, err := mapEditor.Flush(ctx)
if err != nil {
return err
}
pgf.underlyingMap = newMap
pgf.mapHash = pgf.underlyingMap.HashOf()
return pgf.reloadCaches(ctx)
}
// DropFunction drops an existing function.
func (pgf *Collection) DropFunction(ctx context.Context, funcIDs ...id.Function) error {
if len(funcIDs) == 0 {
return nil
}
// Check that each name exists before performing any deletions
for _, funcID := range funcIDs {
if _, ok := pgf.accessCache[funcID]; !ok {
return errors.Errorf(`function %s does not exist`, funcID.FunctionName())
}
}
// Now we'll remove the functions from the map
mapEditor := pgf.underlyingMap.Editor()
for _, funcID := range funcIDs {
err := mapEditor.Delete(ctx, string(funcID))
if err != nil {
return err
}
}
newMap, err := mapEditor.Flush(ctx)
if err != nil {
return err
}
pgf.underlyingMap = newMap
pgf.mapHash = pgf.underlyingMap.HashOf()
return pgf.reloadCaches(ctx)
}
// resolveName returns the fully resolved name of the given function. Returns an error if the name is ambiguous.
//
// The following formats are examples of a formatted name:
// name()
// name(type1, schema.type2)
// name(,,)
func (pgf *Collection) resolveName(ctx context.Context, schemaName string, formattedName string) (id.Function, error) {
if len(pgf.accessCache) == 0 || len(formattedName) == 0 {
return id.NullFunction, nil
}
// Extract the actual name from the format
leftParenIndex := strings.IndexByte(formattedName, '(')
if leftParenIndex == -1 {
return id.NullFunction, nil
}
if formattedName[len(formattedName)-1] != ')' {
return id.NullFunction, nil
}
functionName := strings.TrimSpace(formattedName[:leftParenIndex])
var typeIDs []id.Type
typePortion := strings.TrimSpace(formattedName[leftParenIndex+1 : len(formattedName)-1])
if len(typePortion) > 0 {
// If the type portion is just an empty string, then we don't want any type IDs
typeStrings := strings.Split(strings.TrimSpace(formattedName[leftParenIndex+1:len(formattedName)-1]), ",")
typeIDs = make([]id.Type, len(typeStrings))
for i, typeString := range typeStrings {
typeParts := strings.Split(typeString, ".")
switch len(typeParts) {
case 1:
typeIDs[i] = id.NewType("", strings.TrimSpace(typeParts[0]))
case 2:
typeIDs[i] = id.NewType(strings.TrimSpace(typeParts[0]), strings.TrimSpace(typeParts[1]))
default:
return id.NullFunction, nil
}
}
}
// If there's an exact match, then we return exactly that
fullID := id.NewFunction(schemaName, functionName, typeIDs...)
if _, ok := pgf.accessCache[fullID]; ok {
return fullID, nil
}
// Otherwise we'll iterate over all the names
var resolvedID id.Function
OuterLoop:
for _, funcID := range pgf.idCache {
if !strings.EqualFold(functionName, funcID.FunctionName()) {
continue
}
if len(schemaName) > 0 && !strings.EqualFold(schemaName, funcID.SchemaName()) {
continue
}
if len(typeIDs) > 0 {
if funcID.ParameterCount() != len(typeIDs) {
continue
}
for i, param := range funcID.Parameters() {
if len(typeIDs[i].TypeName()) > 0 && !strings.EqualFold(typeIDs[i].TypeName(), param.TypeName()) {
continue OuterLoop
}
if len(typeIDs[i].SchemaName()) > 0 && !strings.EqualFold(typeIDs[i].SchemaName(), param.SchemaName()) {
continue OuterLoop
}
}
}
// Everything must have matched to have made it here
if resolvedID.IsValid() {
funcTableName := FunctionIDToTableName(funcID)
resolvedTableName := FunctionIDToTableName(resolvedID)
return id.NullFunction, fmt.Errorf("`%s.%s` is ambiguous, matches `%s` and `%s`",
schemaName, formattedName, funcTableName.String(), resolvedTableName.String())
}
resolvedID = funcID
}
return resolvedID, nil
}
// iterateIDs iterates over all function IDs in the collection.
func (pgf *Collection) iterateIDs(ctx context.Context, callback func(funcID id.Function) (stop bool, err error)) error {
for _, funcID := range pgf.idCache {
stop, err := callback(funcID)
if err != nil {
return err
} else if stop {
return nil
}
}
return nil
}
// IterateFunctions iterates over all functions in the collection.
func (pgf *Collection) IterateFunctions(ctx context.Context, callback func(f Function) (stop bool, err error)) error {
for _, funcID := range pgf.idCache {
stop, err := callback(pgf.accessCache[funcID])
if err != nil {
return err
} else if stop {
return nil
}
}
return nil
}
// Clone returns a new *Collection with the same contents as the original.
func (pgf *Collection) Clone(ctx context.Context) *Collection {
return &Collection{
accessCache: maps.Clone(pgf.accessCache),
overloadCache: maps.Clone(pgf.overloadCache),
idCache: slices.Clone(pgf.idCache),
underlyingMap: pgf.underlyingMap,
mapHash: pgf.mapHash,
ns: pgf.ns,
}
}
// Map writes any cached sequences to the underlying map, and then returns the underlying map.
func (pgf *Collection) Map(ctx context.Context) (prolly.AddressMap, error) {
return pgf.underlyingMap, nil
}
// DiffersFrom returns true when the hash that is associated with the underlying map for this collection is different
// from the hash in the given root.
func (pgf *Collection) DiffersFrom(ctx context.Context, root objinterface.RootValue) bool {
hashOnGivenRoot, err := pgf.LoadCollectionHash(ctx, root)
if err != nil {
return true
}
if pgf.mapHash.Equal(hashOnGivenRoot) {
return false
}
// An empty map should match an uninitialized collection on the root
count, err := pgf.underlyingMap.Count()
if err == nil && count == 0 && hashOnGivenRoot.IsEmpty() {
return false
}
return true
}
// reloadCaches writes the underlying map's contents to the caches.
func (pgf *Collection) reloadCaches(ctx context.Context) error {
count, err := pgf.underlyingMap.Count()
if err != nil {
return err
}
clear(pgf.accessCache)
clear(pgf.overloadCache)
pgf.mapHash = pgf.underlyingMap.HashOf()
pgf.idCache = make([]id.Function, 0, count)
return pgf.underlyingMap.IterAll(ctx, func(_ string, h hash.Hash) error {
if h.IsEmpty() {
return nil
}
data, err := pgf.ns.ReadBytes(ctx, h)
if err != nil {
return err
}
f, err := DeserializeFunction(ctx, data)
if err != nil {
return err
}
pgf.accessCache[f.ID] = f
partialID := id.NewFunction(f.ID.SchemaName(), f.ID.FunctionName())
pgf.overloadCache[partialID] = append(pgf.overloadCache[partialID], f.ID)
pgf.idCache = append(pgf.idCache, f.ID)
return nil
})
}
// tableNameToID returns the ID that was encoded via the Name() call, as the returned TableName contains additional
// information (which this is able to process).
func (pgf *Collection) tableNameToID(schemaName string, formattedName string) id.Function {
leftParenIndex := strings.IndexByte(formattedName, '(')
if leftParenIndex == -1 {
return id.NullFunction
}
if formattedName[len(formattedName)-1] != ')' {
return id.NullFunction
}
functionName := strings.TrimSpace(formattedName[:leftParenIndex])
var typeIDs []id.Type
typePortion := strings.TrimSpace(formattedName[leftParenIndex+1 : len(formattedName)-1])
if len(typePortion) > 0 {
// If the type portion is just an empty string, then we don't want any type IDs
typeStrings := strings.Split(strings.TrimSpace(formattedName[leftParenIndex+1:len(formattedName)-1]), ",")
typeIDs = make([]id.Type, len(typeStrings))
for i, typeString := range typeStrings {
typeParts := strings.Split(typeString, ".")
switch len(typeParts) {
case 1:
typeIDs[i] = id.NewType("", strings.TrimSpace(typeParts[0]))
case 2:
typeIDs[i] = id.NewType(strings.TrimSpace(typeParts[0]), strings.TrimSpace(typeParts[1]))
default:
return id.NullFunction
}
}
}
return id.NewFunction(schemaName, functionName, typeIDs...)
}
// GetID implements the interface objinterface.RootObject.
func (function Function) GetID() id.Id {
return function.ID.AsId()
}
// GetInnerDefinition returns the inner definition inside the CREATE FUNCTION statement.
func (function Function) GetInnerDefinition() string {
// TODO: right now we're hardcode searching for $$, which will fail for some definition strings
start := strings.Index(function.Definition, "$$")
end := strings.LastIndex(function.Definition, "$$")
if start == -1 || end == -1 {
// Return the whole definition for now
return function.Definition
}
return strings.TrimSpace(function.Definition[start+2 : end])
}
// ReplaceDefinition returns a new definition with the inner portion replaced with the given string.
func (function Function) ReplaceDefinition(newInner string) string {
return strings.Replace(function.Definition, function.GetInnerDefinition(), newInner, 1)
}
// GetRootObjectID implements the interface objinterface.RootObject.
func (function Function) GetRootObjectID() objinterface.RootObjectID {
return objinterface.RootObjectID_Functions
}
// HashOf implements the interface objinterface.RootObject.
func (function Function) HashOf(ctx context.Context) (hash.Hash, error) {
data, err := function.Serialize(ctx)
if err != nil {
return hash.Hash{}, err
}
return hash.Of(data), nil
}
// Name implements the interface objinterface.RootObject.
func (function Function) Name() doltdb.TableName {
return FunctionIDToTableName(function.ID)
}
// FunctionIDToTableName returns the ID in a format that's better for user consumption.
func FunctionIDToTableName(funcID id.Function) doltdb.TableName {
paramTypes := funcID.Parameters()
strTypes := make([]string, len(paramTypes))
for i, paramType := range paramTypes {
if paramType.SchemaName() == "pg_catalog" || paramType.SchemaName() == funcID.SchemaName() {
strTypes[i] = paramType.TypeName()
} else {
strTypes[i] = fmt.Sprintf("%s.%s", paramType.SchemaName(), paramType.TypeName())
}
}
return doltdb.TableName{
Name: fmt.Sprintf("%s(%s)", funcID.FunctionName(), strings.Join(strTypes, ",")),
Schema: funcID.SchemaName(),
}
}
+126
View File
@@ -0,0 +1,126 @@
// Copyright 2025 Dolthub, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package functions
import (
"context"
"github.com/cockroachdb/errors"
"github.com/dolthub/dolt/go/libraries/doltcore/doltdb"
"github.com/dolthub/dolt/go/libraries/doltcore/merge"
"github.com/dolthub/dolt/go/store/hash"
"github.com/dolthub/dolt/go/store/prolly"
"github.com/dolthub/doltgresql/core/id"
pgmerge "github.com/dolthub/doltgresql/core/merge"
"github.com/dolthub/doltgresql/core/rootobject/objinterface"
"github.com/dolthub/doltgresql/flatbuffers/gen/serial"
)
// storage is used to read from and write to the root.
var storage = objinterface.RootObjectSerializer{
Bytes: (*serial.RootValue).FunctionsBytes,
RootValueAdd: serial.RootValueAddFunctions,
}
// HandleMerge implements the interface objinterface.Collection.
func (*Collection) HandleMerge(ctx context.Context, mro merge.MergeRootObject) (doltdb.RootObject, *merge.MergeStats, error) {
ourFunc := mro.OurRootObj.(Function)
theirFunc := mro.TheirRootObj.(Function)
// Ensure that they have the same identifier
if ourFunc.ID != theirFunc.ID {
return nil, nil, errors.Newf("attempted to merge different functions: `%s` and `%s`",
ourFunc.Name().String(), theirFunc.Name().String())
}
ourHash, err := ourFunc.HashOf(ctx)
if err != nil {
return nil, nil, err
}
theirHash, err := theirFunc.HashOf(ctx)
if err != nil {
return nil, nil, err
}
if ourHash.Equal(theirHash) {
return mro.OurRootObj, &merge.MergeStats{
Operation: merge.TableUnmodified,
Adds: 0,
Deletes: 0,
Modifications: 0,
DataConflicts: 0,
SchemaConflicts: 0,
RootObjectConflicts: 0,
ConstraintViolations: 0,
}, nil
}
return pgmerge.CreateConflict(ctx, mro.RightSrc, ourFunc, theirFunc, mro.AncestorRootObj)
}
// LoadCollection implements the interface objinterface.Collection.
func (*Collection) LoadCollection(ctx context.Context, root objinterface.RootValue) (objinterface.Collection, error) {
return LoadFunctions(ctx, root)
}
// LoadCollectionHash implements the interface objinterface.Collection.
func (*Collection) LoadCollectionHash(ctx context.Context, root objinterface.RootValue) (hash.Hash, error) {
m, ok, err := storage.GetProllyMap(ctx, root)
if err != nil || !ok {
return hash.Hash{}, err
}
return m.HashOf(), nil
}
// LoadFunctions loads the functions collection from the given root.
func LoadFunctions(ctx context.Context, root objinterface.RootValue) (*Collection, error) {
m, ok, err := storage.GetProllyMap(ctx, root)
if err != nil {
return nil, err
}
if !ok {
m, err = prolly.NewEmptyAddressMap(root.NodeStore())
if err != nil {
return nil, err
}
}
return NewCollection(ctx, m, root.NodeStore())
}
// ResolveNameFromObjects implements the interface objinterface.Collection.
func (*Collection) ResolveNameFromObjects(ctx context.Context, name doltdb.TableName, rootObjects []objinterface.RootObject) (doltdb.TableName, id.Id, error) {
tempCollection := Collection{
accessCache: make(map[id.Function]Function),
idCache: make([]id.Function, 0, len(rootObjects)),
}
for _, rootObject := range rootObjects {
if obj, ok := rootObject.(Function); ok {
tempCollection.accessCache[obj.ID] = obj
tempCollection.idCache = append(tempCollection.idCache, obj.ID)
}
}
return tempCollection.ResolveName(ctx, name)
}
// Serializer implements the interface objinterface.Collection.
func (*Collection) Serializer() objinterface.RootObjectSerializer {
return storage
}
// UpdateRoot implements the interface objinterface.Collection.
func (pgf *Collection) UpdateRoot(ctx context.Context, root objinterface.RootValue) (objinterface.RootValue, error) {
m, err := pgf.Map(ctx)
if err != nil {
return nil, err
}
return storage.WriteProllyMap(ctx, root, m)
}
+324
View File
@@ -0,0 +1,324 @@
// Copyright 2025 Dolthub, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package functions
import (
"context"
"strings"
"github.com/cockroachdb/errors"
"github.com/dolthub/dolt/go/libraries/doltcore/doltdb"
"github.com/dolthub/doltgresql/core/id"
pgmerge "github.com/dolthub/doltgresql/core/merge"
"github.com/dolthub/doltgresql/core/rootobject/objinterface"
"github.com/dolthub/doltgresql/server/plpgsql"
pgtypes "github.com/dolthub/doltgresql/server/types"
)
const (
FIELD_NAME_PARAMETER_NAMES = "parameter_names"
FIELD_NAME_RETURN_TYPE = "return_type"
FIELD_NAME_NON_DETERMINISTIC = "non_deterministic"
FIELD_NAME_STRICT = "strict"
FIELD_NAME_DEFINITION = "definition"
FIELD_NAME_EXTENSION_NAME = "extension_name"
FIELD_NAME_EXTENSION_SYMBOL = "extension_symbol"
FIELD_NAME_SQL_DEFINITION = "sql_definition"
FIELD_NAME_SET_OF = "set_of"
)
// DeserializeRootObject implements the interface objinterface.Collection.
func (pgf *Collection) DeserializeRootObject(ctx context.Context, data []byte) (objinterface.RootObject, error) {
return DeserializeFunction(ctx, data)
}
// DiffRootObjects implements the interface objinterface.Collection.
func (pgf *Collection) DiffRootObjects(ctx context.Context, fromHash string, o objinterface.RootObject, t objinterface.RootObject, a objinterface.RootObject) ([]objinterface.RootObjectDiff, objinterface.RootObject, error) {
// We ignore many fields when diffing, as differences in these fields would result in a different function due to overloading
// For example, "func_name(text)" and "func_name(varchar)" cannot produce a conflict as they're different functions
ours := o.(Function)
theirs := t.(Function)
ancestor, hasAncestor := a.(Function)
var diffs []objinterface.RootObjectDiff
{
ourParamNames := strings.Join(ours.ParameterNames, ",")
theirParamNames := strings.Join(theirs.ParameterNames, ",")
ancParamNames := strings.Join(ancestor.ParameterNames, ",")
diff := objinterface.RootObjectDiff{
Type: pgtypes.Text,
FromHash: fromHash,
FieldName: FIELD_NAME_PARAMETER_NAMES,
}
if pgmerge.DiffValues(&diff, ourParamNames, theirParamNames, ancParamNames, hasAncestor) {
diffs = append(diffs, diff)
} else {
ours.ParameterNames = strings.Split(diff.OurValue.(string), ",")
}
}
if ours.ReturnType != theirs.ReturnType {
diff := objinterface.RootObjectDiff{
Type: pgtypes.Text,
FromHash: fromHash,
FieldName: FIELD_NAME_RETURN_TYPE,
}
if pgmerge.DiffValues(&diff, ours.ReturnType.TypeName(), theirs.ReturnType.TypeName(), ancestor.ReturnType.TypeName(), hasAncestor) {
diffs = append(diffs, diff)
} else {
ours.ReturnType = id.NewType(ours.ReturnType.SchemaName(), diff.OurValue.(string))
}
}
if ours.IsNonDeterministic != theirs.IsNonDeterministic {
diff := objinterface.RootObjectDiff{
Type: pgtypes.Bool,
FromHash: fromHash,
FieldName: FIELD_NAME_NON_DETERMINISTIC,
}
if pgmerge.DiffValues(&diff, ours.IsNonDeterministic, theirs.IsNonDeterministic, ancestor.IsNonDeterministic, hasAncestor) {
diffs = append(diffs, diff)
} else {
ours.IsNonDeterministic = diff.OurValue.(bool)
}
}
if ours.Strict != theirs.Strict {
diff := objinterface.RootObjectDiff{
Type: pgtypes.Bool,
FromHash: fromHash,
FieldName: FIELD_NAME_STRICT,
}
if pgmerge.DiffValues(&diff, ours.Strict, theirs.Strict, ancestor.Strict, hasAncestor) {
diffs = append(diffs, diff)
} else {
ours.Strict = diff.OurValue.(bool)
}
}
if ours.Definition != theirs.Definition {
diff := objinterface.RootObjectDiff{
Type: pgtypes.Text,
FromHash: fromHash,
FieldName: FIELD_NAME_DEFINITION,
}
if pgmerge.DiffValues(&diff, ours.GetInnerDefinition(), theirs.GetInnerDefinition(), ancestor.GetInnerDefinition(), hasAncestor) {
diffs = append(diffs, diff)
} else {
ours.Definition = ours.ReplaceDefinition(diff.OurValue.(string))
}
}
if ours.ExtensionName != theirs.ExtensionName {
diff := objinterface.RootObjectDiff{
Type: pgtypes.Text,
FromHash: fromHash,
FieldName: FIELD_NAME_EXTENSION_NAME,
}
if pgmerge.DiffValues(&diff, ours.ExtensionName, theirs.ExtensionName, ancestor.ExtensionName, hasAncestor) {
diffs = append(diffs, diff)
} else {
ours.ExtensionName = diff.OurValue.(string)
}
}
if ours.ExtensionSymbol != theirs.ExtensionSymbol {
diff := objinterface.RootObjectDiff{
Type: pgtypes.Text,
FromHash: fromHash,
FieldName: FIELD_NAME_EXTENSION_SYMBOL,
}
if pgmerge.DiffValues(&diff, ours.ExtensionSymbol, theirs.ExtensionSymbol, ancestor.ExtensionSymbol, hasAncestor) {
diffs = append(diffs, diff)
} else {
ours.ExtensionSymbol = diff.OurValue.(string)
}
}
if ours.SQLDefinition != theirs.SQLDefinition {
diff := objinterface.RootObjectDiff{
Type: pgtypes.Text,
FromHash: fromHash,
FieldName: FIELD_NAME_SQL_DEFINITION,
}
if pgmerge.DiffValues(&diff, ours.SQLDefinition, theirs.SQLDefinition, ancestor.SQLDefinition, hasAncestor) {
diffs = append(diffs, diff)
} else {
ours.SQLDefinition = diff.OurValue.(string)
}
}
if ours.SetOf != theirs.SetOf {
diff := objinterface.RootObjectDiff{
Type: pgtypes.Bool,
FromHash: fromHash,
FieldName: FIELD_NAME_SET_OF,
}
if pgmerge.DiffValues(&diff, ours.SetOf, theirs.SetOf, ancestor.SetOf, hasAncestor) {
diffs = append(diffs, diff)
} else {
ours.SetOf = diff.OurValue.(bool)
}
}
return diffs, ours, nil
}
// DropRootObject implements the interface objinterface.Collection.
func (pgf *Collection) DropRootObject(ctx context.Context, identifier id.Id) error {
if identifier.Section() != id.Section_Function {
return errors.Errorf(`function %s does not exist`, identifier.String())
}
return pgf.DropFunction(ctx, id.Function(identifier))
}
// GetFieldType implements the interface objinterface.Collection.
func (pgf *Collection) GetFieldType(ctx context.Context, fieldName string) *pgtypes.DoltgresType {
switch fieldName {
case FIELD_NAME_PARAMETER_NAMES:
return pgtypes.Text
case FIELD_NAME_RETURN_TYPE:
return pgtypes.Text
case FIELD_NAME_NON_DETERMINISTIC:
return pgtypes.Bool
case FIELD_NAME_STRICT:
return pgtypes.Bool
case FIELD_NAME_DEFINITION:
return pgtypes.Text
case FIELD_NAME_EXTENSION_NAME:
return pgtypes.Text
case FIELD_NAME_EXTENSION_SYMBOL:
return pgtypes.Text
case FIELD_NAME_SQL_DEFINITION:
return pgtypes.Text
case FIELD_NAME_SET_OF:
return pgtypes.Bool
default:
return nil
}
}
// GetID implements the interface objinterface.Collection.
func (pgf *Collection) GetID() objinterface.RootObjectID {
return objinterface.RootObjectID_Functions
}
// GetRootObject implements the interface objinterface.Collection.
func (pgf *Collection) GetRootObject(ctx context.Context, identifier id.Id) (objinterface.RootObject, bool, error) {
if identifier.Section() != id.Section_Function {
return nil, false, nil
}
f, err := pgf.GetFunction(ctx, id.Function(identifier))
return f, err == nil && f.ID.IsValid(), err
}
// HasRootObject implements the interface objinterface.Collection.
func (pgf *Collection) HasRootObject(ctx context.Context, identifier id.Id) (bool, error) {
if identifier.Section() != id.Section_Function {
return false, nil
}
return pgf.HasFunction(ctx, id.Function(identifier)), nil
}
// IDToTableName implements the interface objinterface.Collection.
func (pgf *Collection) IDToTableName(identifier id.Id) doltdb.TableName {
if identifier.Section() != id.Section_Function {
return doltdb.TableName{}
}
return FunctionIDToTableName(id.Function(identifier))
}
// IterAll implements the interface objinterface.Collection.
func (pgf *Collection) IterAll(ctx context.Context, callback func(rootObj objinterface.RootObject) (stop bool, err error)) error {
return pgf.IterateFunctions(ctx, func(f Function) (stop bool, err error) {
return callback(f)
})
}
// IterIDs implements the interface objinterface.Collection.
func (pgf *Collection) IterIDs(ctx context.Context, callback func(identifier id.Id) (stop bool, err error)) error {
return pgf.iterateIDs(ctx, func(funcID id.Function) (stop bool, err error) {
return callback(funcID.AsId())
})
}
// PutRootObject implements the interface objinterface.Collection.
func (pgf *Collection) PutRootObject(ctx context.Context, rootObj objinterface.RootObject) error {
f, ok := rootObj.(Function)
if !ok {
return errors.Newf("invalid function root object: %T", rootObj)
}
return pgf.AddFunction(ctx, f)
}
// RenameRootObject implements the interface objinterface.Collection.
func (pgf *Collection) RenameRootObject(ctx context.Context, oldName id.Id, newName id.Id) error {
if !oldName.IsValid() || !newName.IsValid() || oldName.Section() != newName.Section() || oldName.Section() != id.Section_Function {
return errors.New("cannot rename function due to invalid name")
}
oldFuncName := id.Function(oldName)
newFuncName := id.Function(newName)
if oldFuncName.ParameterCount() != newFuncName.ParameterCount() {
return errors.Newf(`old function id had "%d" parameters, new function id has "%d" parameters`,
oldFuncName.ParameterCount(), newFuncName.ParameterCount())
}
f, err := pgf.GetFunction(ctx, oldFuncName)
if err != nil {
return err
}
if err = pgf.DropFunction(ctx, oldFuncName); err != nil {
return err
}
f.ID = newFuncName
return pgf.AddFunction(ctx, f)
}
// ResolveName implements the interface objinterface.Collection.
func (pgf *Collection) ResolveName(ctx context.Context, name doltdb.TableName) (doltdb.TableName, id.Id, error) {
rawID, err := pgf.resolveName(ctx, name.Schema, name.Name)
if err != nil || !rawID.IsValid() {
return doltdb.TableName{}, id.Null, err
}
return FunctionIDToTableName(rawID), rawID.AsId(), nil
}
// TableNameToID implements the interface objinterface.Collection.
func (pgf *Collection) TableNameToID(name doltdb.TableName) id.Id {
return pgf.tableNameToID(name.Schema, name.Name).AsId()
}
// UpdateField implements the interface objinterface.Collection.
func (pgf *Collection) UpdateField(ctx context.Context, rootObject objinterface.RootObject, fieldName string, newValue any) (objinterface.RootObject, error) {
function := rootObject.(Function)
switch fieldName {
case FIELD_NAME_PARAMETER_NAMES:
function.ParameterNames = strings.Split(newValue.(string), ",")
case FIELD_NAME_RETURN_TYPE:
function.ReturnType = id.NewType(function.ReturnType.SchemaName(), newValue.(string))
case FIELD_NAME_NON_DETERMINISTIC:
function.IsNonDeterministic = newValue.(bool)
case FIELD_NAME_STRICT:
function.Strict = newValue.(bool)
case FIELD_NAME_DEFINITION:
function.Definition = function.ReplaceDefinition(newValue.(string))
parsedBody, err := plpgsql.Parse(function.Definition)
if err != nil {
return nil, err
}
function.Operations = parsedBody
case FIELD_NAME_EXTENSION_NAME:
function.ExtensionName = newValue.(string)
case FIELD_NAME_EXTENSION_SYMBOL:
function.ExtensionSymbol = newValue.(string)
case FIELD_NAME_SQL_DEFINITION:
function.SQLDefinition = newValue.(string)
case FIELD_NAME_SET_OF:
function.SetOf = newValue.(bool)
default:
return nil, errors.Newf("unknown field name: `%s`", fieldName)
}
return function, nil
}
+118
View File
@@ -0,0 +1,118 @@
// Copyright 2024 Dolthub, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package functions
import (
"context"
"github.com/cockroachdb/errors"
"github.com/dolthub/doltgresql/core/id"
"github.com/dolthub/doltgresql/server/plpgsql"
"github.com/dolthub/doltgresql/utils"
)
// Serialize returns the Function as a byte slice. If the Function is invalid, then this returns a nil slice.
func (function Function) Serialize(ctx context.Context) ([]byte, error) {
if !function.ID.IsValid() {
return nil, nil
}
// Write all of the functions to the writer
writer := utils.NewWriter(256)
writer.VariableUint(3) // Version
// Write the function data
writer.Id(function.ID.AsId())
writer.Id(function.ReturnType.AsId())
writer.StringSlice(function.ParameterNames)
writer.IdTypeSlice(function.ParameterTypes)
writer.Bool(function.Variadic)
writer.Bool(function.IsNonDeterministic)
writer.Bool(function.Strict)
writer.String(function.Definition)
// Write the operations
writer.VariableUint(uint64(len(function.Operations)))
for _, op := range function.Operations {
writer.Uint16(uint16(op.OpCode))
writer.String(op.PrimaryData)
writer.StringSlice(op.SecondaryData)
writer.String(op.Target)
writer.Int32(int32(op.Index))
writer.StringMap(op.Options)
}
// Write version 1 data
writer.String(function.ExtensionName)
writer.String(function.ExtensionSymbol)
// Write version 2 data
writer.String(function.SQLDefinition)
writer.Bool(function.SetOf)
// Write version 3 data
writer.StringSlice(function.ParameterDefaults)
// Returns the data
return writer.Data(), nil
}
// DeserializeFunction returns the Function that was serialized in the byte slice. Returns an empty Function (invalid
// ID) if data is nil or empty.
func DeserializeFunction(ctx context.Context, data []byte) (Function, error) {
if len(data) == 0 {
return Function{}, nil
}
reader := utils.NewReader(data)
version := reader.VariableUint()
if version > 3 {
return Function{}, errors.Errorf("version %d of functions is not supported, please upgrade the server", version)
}
// Read from the reader
f := Function{}
f.ID = id.Function(reader.Id())
f.ReturnType = id.Type(reader.Id())
f.ParameterNames = reader.StringSlice()
f.ParameterTypes = reader.IdTypeSlice()
f.Variadic = reader.Bool()
f.IsNonDeterministic = reader.Bool()
f.Strict = reader.Bool()
f.Definition = reader.String()
// Read the operations
opCount := reader.VariableUint()
f.Operations = make([]plpgsql.InterpreterOperation, opCount)
for opIdx := uint64(0); opIdx < opCount; opIdx++ {
op := plpgsql.InterpreterOperation{}
op.OpCode = plpgsql.OpCode(reader.Uint16())
op.PrimaryData = reader.String()
op.SecondaryData = reader.StringSlice()
op.Target = reader.String()
op.Index = int(reader.Int32())
op.Options = reader.StringMap()
f.Operations[opIdx] = op
}
if version >= 1 {
f.ExtensionName = reader.String()
f.ExtensionSymbol = reader.String()
}
if version >= 2 {
f.SQLDefinition = reader.String()
f.SetOf = reader.Bool()
}
if version >= 3 {
f.ParameterDefaults = reader.StringSlice()
}
if !reader.IsEmpty() {
return Function{}, errors.Errorf("extra data found while deserializing a function")
}
// Return the deserialized object
return f, nil
}
+144
View File
@@ -0,0 +1,144 @@
// Copyright 2024 Dolthub, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package id
import (
"hash/crc32"
"sync"
)
// builtinOidLimit is the largest OID that Postgres will assign to built-in items, so we use this to mitigate conflicts
// with existing and future built-in OIDs.
const builtinOidLimit = 65535
var (
// crcTable is the table that is used for our CRC operations.
crcTable = crc32.MakeTable(crc32.Castagnoli)
// globalCache is the cache structure that is used for the server session.
globalCache = &cacheStruct{
mutex: &sync.RWMutex{},
toOID: map[Id]uint32{Null: 0},
toInternal: map[uint32]Id{0: Null},
}
)
// cacheStruct is the cache structure that holds mappings between the internal ID and external OID (used by Postgres).
// The mappings are temporary, and exist only within a server session. We must discourage users from storing converted
// OIDs, and to use the actual OID type, since the type uses internal IDs so long as it's not returned to the user.
type cacheStruct struct {
mutex *sync.RWMutex
toOID map[Id]uint32
toInternal map[uint32]Id
}
// Cache returns the global cache that is used for the server session.
func Cache() *cacheStruct {
return globalCache
}
// ToOID returns the OID associated with the given internal ID.
func (cache *cacheStruct) ToOID(id Id) uint32 {
// If the ID is in the cache, then we can just return its associated OID
cache.mutex.RLock()
if oid, ok := cache.toOID[id]; ok {
cache.mutex.RUnlock()
return oid
}
cache.mutex.RUnlock()
if id.Section() == Section_OID {
return Oid(id).OID()
}
cache.mutex.Lock()
defer cache.mutex.Unlock()
underlyingBytes := id.UnderlyingBytes()
oid := crc32.Checksum(underlyingBytes, crcTable)
// If the generated OID is valid, then we'll add it to the cache and return it
if _, ok := cache.toInternal[oid]; !ok && oid > builtinOidLimit {
cache.toOID[id] = oid
cache.toInternal[oid] = id
return oid
}
// In this case, the OID is not valid, so we'll run a small loop to generate an OID based on the actual ID.
// This retains some level of determinism for OID to ID relationships.
modifiedBytes := make([]byte, len(underlyingBytes)+1)
copy(modifiedBytes[1:], underlyingBytes)
for i := byte(0); i < 255; i++ {
modifiedBytes[0] = i
oid = crc32.Checksum(underlyingBytes, crcTable)
if _, ok := cache.toInternal[oid]; !ok && oid > builtinOidLimit {
cache.toOID[id] = oid
cache.toInternal[oid] = id
return oid
}
}
// If we're here, then we'll just search for an empty OID as a last resort
for i := uint32(4294967295); i > builtinOidLimit; i-- {
if _, ok := cache.toInternal[oid]; !ok {
cache.toOID[id] = oid
cache.toInternal[oid] = id
return oid
}
}
// We must have over 4 billion items in the database, so we'll panic since there's nothing we can do
panic("all OIDs have been taken")
}
// ToInternal returns the internal ID associated with the given OID.
func (cache *cacheStruct) ToInternal(oid uint32) Id {
cache.mutex.RLock()
defer cache.mutex.RUnlock()
if id, ok := cache.toInternal[oid]; ok {
return id
}
// The OID is not in the cache, so it's invalid
return ""
}
// Exists returns whether the given internal ID exists within the cache. This should primarily be used for the default
// functions, as it's not guaranteed that user functions will be in the cache, especially after a server restart.
func (cache *cacheStruct) Exists(id Id) bool {
cache.mutex.RLock()
defer cache.mutex.RUnlock()
_, ok := cache.toOID[id]
return ok
}
// setBuiltIn sets the given ID to the OID. This should only be used for the built-in items.
func (cache *cacheStruct) setBuiltIn(id Id, oid uint32) {
if oid > builtinOidLimit {
panic("oid is not a built-in")
}
cache.toOID[id] = oid
cache.toInternal[oid] = id
}
// update is used to change the OID mapping of an existing internal ID that has been changed (where the internal ID
// points to the same logical item).
//
//lint:ignore U1000 For future use
func (cache *cacheStruct) update(old Id, new Id) {
cache.mutex.Lock()
defer cache.mutex.Unlock()
// If the old ID doesn't exist in the cache, then we don't have anything to update
oid, ok := cache.toOID[old]
if !ok {
return
}
// We'll delete the old entry and add the new entry, keeping the OID the same for the server session
delete(cache.toOID, old)
delete(cache.toInternal, oid)
cache.toOID[new] = oid
cache.toInternal[oid] = new
}
+26
View File
@@ -0,0 +1,26 @@
// Copyright 2024 Dolthub, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package id
// This adds all of the built-in schemas to the cache.
func init() {
globalCache.setBuiltIn(NewId(Section_AccessMethod, "heap"), 2)
globalCache.setBuiltIn(NewId(Section_AccessMethod, "btree"), 403)
globalCache.setBuiltIn(NewId(Section_AccessMethod, "hash"), 405)
globalCache.setBuiltIn(NewId(Section_AccessMethod, "gist"), 783)
globalCache.setBuiltIn(NewId(Section_AccessMethod, "gin"), 2742)
globalCache.setBuiltIn(NewId(Section_AccessMethod, "brin"), 3580)
globalCache.setBuiltIn(NewId(Section_AccessMethod, "spgist"), 4000)
}
+806
View File
@@ -0,0 +1,806 @@
// Copyright 2024 Dolthub, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package id
// This adds all of the built-in schemas to the cache.
func init() {
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "default"), 100)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "C"), 950)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "POSIX"), 951)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "ucs_basic"), 12340)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "und-x-icu"), 12341)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "af-x-icu"), 12342)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "af-NA-x-icu"), 12343)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "af-ZA-x-icu"), 12344)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "agq-x-icu"), 12345)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "agq-CM-x-icu"), 12346)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "ak-x-icu"), 12347)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "ak-GH-x-icu"), 12348)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "am-x-icu"), 12349)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "am-ET-x-icu"), 12350)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "ar-x-icu"), 12351)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "ar-001-x-icu"), 12352)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "ar-AE-x-icu"), 12353)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "ar-BH-x-icu"), 12354)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "ar-DJ-x-icu"), 12355)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "ar-DZ-x-icu"), 12356)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "ar-EG-x-icu"), 12357)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "ar-EH-x-icu"), 12358)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "ar-ER-x-icu"), 12359)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "ar-IL-x-icu"), 12360)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "ar-IQ-x-icu"), 12361)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "ar-JO-x-icu"), 12362)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "ar-KM-x-icu"), 12363)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "ar-KW-x-icu"), 12364)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "ar-LB-x-icu"), 12365)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "ar-LY-x-icu"), 12366)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "ar-MA-x-icu"), 12367)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "ar-MR-x-icu"), 12368)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "ar-OM-x-icu"), 12369)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "ar-PS-x-icu"), 12370)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "ar-QA-x-icu"), 12371)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "ar-SA-x-icu"), 12372)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "ar-SD-x-icu"), 12373)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "ar-SO-x-icu"), 12374)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "ar-SS-x-icu"), 12375)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "ar-SY-x-icu"), 12376)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "ar-TD-x-icu"), 12377)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "ar-TN-x-icu"), 12378)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "ar-YE-x-icu"), 12379)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "as-x-icu"), 12380)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "as-IN-x-icu"), 12381)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "asa-x-icu"), 12382)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "asa-TZ-x-icu"), 12383)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "ast-x-icu"), 12384)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "ast-ES-x-icu"), 12385)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "az-x-icu"), 12386)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "az-Cyrl-x-icu"), 12387)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "az-Cyrl-AZ-x-icu"), 12388)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "az-Latn-x-icu"), 12389)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "az-Latn-AZ-x-icu"), 12390)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "bas-x-icu"), 12391)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "bas-CM-x-icu"), 12392)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "be-x-icu"), 12393)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "be-BY-x-icu"), 12394)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "bem-x-icu"), 12395)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "bem-ZM-x-icu"), 12396)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "bez-x-icu"), 12397)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "bez-TZ-x-icu"), 12398)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "bg-x-icu"), 12399)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "bg-BG-x-icu"), 12400)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "bm-x-icu"), 12401)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "bm-ML-x-icu"), 12402)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "bn-x-icu"), 12403)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "bn-BD-x-icu"), 12404)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "bn-IN-x-icu"), 12405)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "bo-x-icu"), 12406)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "bo-CN-x-icu"), 12407)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "bo-IN-x-icu"), 12408)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "br-x-icu"), 12409)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "br-FR-x-icu"), 12410)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "brx-x-icu"), 12411)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "brx-IN-x-icu"), 12412)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "bs-x-icu"), 12413)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "bs-Cyrl-x-icu"), 12414)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "bs-Cyrl-BA-x-icu"), 12415)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "bs-Latn-x-icu"), 12416)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "bs-Latn-BA-x-icu"), 12417)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "ca-x-icu"), 12418)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "ca-AD-x-icu"), 12419)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "ca-ES-x-icu"), 12420)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "ca-FR-x-icu"), 12421)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "ca-IT-x-icu"), 12422)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "ccp-x-icu"), 12423)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "ccp-BD-x-icu"), 12424)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "ccp-IN-x-icu"), 12425)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "ce-x-icu"), 12426)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "ce-RU-x-icu"), 12427)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "ceb-x-icu"), 12428)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "ceb-PH-x-icu"), 12429)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "cgg-x-icu"), 12430)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "cgg-UG-x-icu"), 12431)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "chr-x-icu"), 12432)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "chr-US-x-icu"), 12433)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "ckb-x-icu"), 12434)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "ckb-IQ-x-icu"), 12435)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "ckb-IR-x-icu"), 12436)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "cs-x-icu"), 12437)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "cs-CZ-x-icu"), 12438)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "cy-x-icu"), 12439)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "cy-GB-x-icu"), 12440)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "da-x-icu"), 12441)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "da-DK-x-icu"), 12442)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "da-GL-x-icu"), 12443)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "dav-x-icu"), 12444)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "dav-KE-x-icu"), 12445)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "de-x-icu"), 12446)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "de-AT-x-icu"), 12447)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "de-BE-x-icu"), 12448)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "de-CH-x-icu"), 12449)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "de-DE-x-icu"), 12450)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "de-IT-x-icu"), 12451)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "de-LI-x-icu"), 12452)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "de-LU-x-icu"), 12453)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "dje-x-icu"), 12454)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "dje-NE-x-icu"), 12455)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "dsb-x-icu"), 12456)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "dsb-DE-x-icu"), 12457)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "dua-x-icu"), 12458)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "dua-CM-x-icu"), 12459)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "dyo-x-icu"), 12460)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "dyo-SN-x-icu"), 12461)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "dz-x-icu"), 12462)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "dz-BT-x-icu"), 12463)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "ebu-x-icu"), 12464)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "ebu-KE-x-icu"), 12465)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "ee-x-icu"), 12466)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "ee-GH-x-icu"), 12467)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "ee-TG-x-icu"), 12468)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "el-x-icu"), 12469)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "el-CY-x-icu"), 12470)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "el-GR-x-icu"), 12471)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "en-x-icu"), 12472)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "en-001-x-icu"), 12473)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "en-150-x-icu"), 12474)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "en-AE-x-icu"), 12475)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "en-AG-x-icu"), 12476)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "en-AI-x-icu"), 12477)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "en-AS-x-icu"), 12478)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "en-AT-x-icu"), 12479)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "en-AU-x-icu"), 12480)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "en-BB-x-icu"), 12481)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "en-BE-x-icu"), 12482)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "en-BI-x-icu"), 12483)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "en-BM-x-icu"), 12484)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "en-BS-x-icu"), 12485)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "en-BW-x-icu"), 12486)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "en-BZ-x-icu"), 12487)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "en-CA-x-icu"), 12488)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "en-CC-x-icu"), 12489)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "en-CH-x-icu"), 12490)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "en-CK-x-icu"), 12491)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "en-CM-x-icu"), 12492)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "en-CX-x-icu"), 12493)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "en-CY-x-icu"), 12494)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "en-DE-x-icu"), 12495)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "en-DG-x-icu"), 12496)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "en-DK-x-icu"), 12497)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "en-DM-x-icu"), 12498)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "en-ER-x-icu"), 12499)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "en-FI-x-icu"), 12500)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "en-FJ-x-icu"), 12501)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "en-FK-x-icu"), 12502)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "en-FM-x-icu"), 12503)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "en-GB-x-icu"), 12504)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "en-GD-x-icu"), 12505)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "en-GG-x-icu"), 12506)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "en-GH-x-icu"), 12507)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "en-GI-x-icu"), 12508)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "en-GM-x-icu"), 12509)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "en-GU-x-icu"), 12510)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "en-GY-x-icu"), 12511)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "en-HK-x-icu"), 12512)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "en-IE-x-icu"), 12513)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "en-IL-x-icu"), 12514)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "en-IM-x-icu"), 12515)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "en-IN-x-icu"), 12516)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "en-IO-x-icu"), 12517)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "en-JE-x-icu"), 12518)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "en-JM-x-icu"), 12519)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "en-KE-x-icu"), 12520)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "en-KI-x-icu"), 12521)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "en-KN-x-icu"), 12522)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "en-KY-x-icu"), 12523)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "en-LC-x-icu"), 12524)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "en-LR-x-icu"), 12525)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "en-LS-x-icu"), 12526)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "en-MG-x-icu"), 12527)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "en-MH-x-icu"), 12528)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "en-MO-x-icu"), 12529)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "en-MP-x-icu"), 12530)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "en-MS-x-icu"), 12531)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "en-MT-x-icu"), 12532)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "en-MU-x-icu"), 12533)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "en-MW-x-icu"), 12534)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "en-MY-x-icu"), 12535)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "en-NA-x-icu"), 12536)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "en-NF-x-icu"), 12537)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "en-NG-x-icu"), 12538)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "en-NL-x-icu"), 12539)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "en-NR-x-icu"), 12540)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "en-NU-x-icu"), 12541)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "en-NZ-x-icu"), 12542)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "en-PG-x-icu"), 12543)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "en-PH-x-icu"), 12544)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "en-PK-x-icu"), 12545)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "en-PN-x-icu"), 12546)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "en-PR-x-icu"), 12547)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "en-PW-x-icu"), 12548)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "en-RW-x-icu"), 12549)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "en-SB-x-icu"), 12550)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "en-SC-x-icu"), 12551)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "en-SD-x-icu"), 12552)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "en-SE-x-icu"), 12553)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "en-SG-x-icu"), 12554)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "en-SH-x-icu"), 12555)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "en-SI-x-icu"), 12556)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "en-SL-x-icu"), 12557)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "en-SS-x-icu"), 12558)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "en-SX-x-icu"), 12559)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "en-SZ-x-icu"), 12560)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "en-TC-x-icu"), 12561)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "en-TK-x-icu"), 12562)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "en-TO-x-icu"), 12563)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "en-TT-x-icu"), 12564)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "en-TV-x-icu"), 12565)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "en-TZ-x-icu"), 12566)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "en-UG-x-icu"), 12567)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "en-UM-x-icu"), 12568)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "en-US-x-icu"), 12569)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "en-US-u-va-posix-x-icu"), 12570)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "en-VC-x-icu"), 12571)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "en-VG-x-icu"), 12572)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "en-VI-x-icu"), 12573)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "en-VU-x-icu"), 12574)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "en-WS-x-icu"), 12575)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "en-ZA-x-icu"), 12576)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "en-ZM-x-icu"), 12577)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "en-ZW-x-icu"), 12578)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "eo-x-icu"), 12579)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "eo-001-x-icu"), 12580)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "es-x-icu"), 12581)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "es-419-x-icu"), 12582)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "es-AR-x-icu"), 12583)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "es-BO-x-icu"), 12584)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "es-BR-x-icu"), 12585)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "es-BZ-x-icu"), 12586)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "es-CL-x-icu"), 12587)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "es-CO-x-icu"), 12588)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "es-CR-x-icu"), 12589)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "es-CU-x-icu"), 12590)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "es-DO-x-icu"), 12591)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "es-EA-x-icu"), 12592)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "es-EC-x-icu"), 12593)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "es-ES-x-icu"), 12594)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "es-GQ-x-icu"), 12595)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "es-GT-x-icu"), 12596)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "es-HN-x-icu"), 12597)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "es-IC-x-icu"), 12598)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "es-MX-x-icu"), 12599)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "es-NI-x-icu"), 12600)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "es-PA-x-icu"), 12601)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "es-PE-x-icu"), 12602)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "es-PH-x-icu"), 12603)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "es-PR-x-icu"), 12604)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "es-PY-x-icu"), 12605)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "es-SV-x-icu"), 12606)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "es-US-x-icu"), 12607)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "es-UY-x-icu"), 12608)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "es-VE-x-icu"), 12609)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "et-x-icu"), 12610)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "et-EE-x-icu"), 12611)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "eu-x-icu"), 12612)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "eu-ES-x-icu"), 12613)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "ewo-x-icu"), 12614)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "ewo-CM-x-icu"), 12615)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "fa-x-icu"), 12616)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "fa-AF-x-icu"), 12617)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "fa-IR-x-icu"), 12618)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "ff-x-icu"), 12619)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "ff-Adlm-x-icu"), 12620)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "ff-Adlm-BF-x-icu"), 12621)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "ff-Adlm-CM-x-icu"), 12622)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "ff-Adlm-GH-x-icu"), 12623)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "ff-Adlm-GM-x-icu"), 12624)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "ff-Adlm-GN-x-icu"), 12625)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "ff-Adlm-GW-x-icu"), 12626)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "ff-Adlm-LR-x-icu"), 12627)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "ff-Adlm-MR-x-icu"), 12628)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "ff-Adlm-NE-x-icu"), 12629)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "ff-Adlm-NG-x-icu"), 12630)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "ff-Adlm-SL-x-icu"), 12631)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "ff-Adlm-SN-x-icu"), 12632)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "ff-Latn-x-icu"), 12633)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "ff-Latn-BF-x-icu"), 12634)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "ff-Latn-CM-x-icu"), 12635)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "ff-Latn-GH-x-icu"), 12636)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "ff-Latn-GM-x-icu"), 12637)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "ff-Latn-GN-x-icu"), 12638)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "ff-Latn-GW-x-icu"), 12639)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "ff-Latn-LR-x-icu"), 12640)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "ff-Latn-MR-x-icu"), 12641)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "ff-Latn-NE-x-icu"), 12642)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "ff-Latn-NG-x-icu"), 12643)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "ff-Latn-SL-x-icu"), 12644)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "ff-Latn-SN-x-icu"), 12645)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "fi-x-icu"), 12646)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "fi-FI-x-icu"), 12647)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "fil-x-icu"), 12648)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "fil-PH-x-icu"), 12649)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "fo-x-icu"), 12650)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "fo-DK-x-icu"), 12651)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "fo-FO-x-icu"), 12652)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "fr-x-icu"), 12653)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "fr-BE-x-icu"), 12654)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "fr-BF-x-icu"), 12655)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "fr-BI-x-icu"), 12656)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "fr-BJ-x-icu"), 12657)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "fr-BL-x-icu"), 12658)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "fr-CA-x-icu"), 12659)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "fr-CD-x-icu"), 12660)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "fr-CF-x-icu"), 12661)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "fr-CG-x-icu"), 12662)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "fr-CH-x-icu"), 12663)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "fr-CI-x-icu"), 12664)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "fr-CM-x-icu"), 12665)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "fr-DJ-x-icu"), 12666)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "fr-DZ-x-icu"), 12667)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "fr-FR-x-icu"), 12668)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "fr-GA-x-icu"), 12669)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "fr-GF-x-icu"), 12670)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "fr-GN-x-icu"), 12671)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "fr-GP-x-icu"), 12672)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "fr-GQ-x-icu"), 12673)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "fr-HT-x-icu"), 12674)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "fr-KM-x-icu"), 12675)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "fr-LU-x-icu"), 12676)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "fr-MA-x-icu"), 12677)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "fr-MC-x-icu"), 12678)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "fr-MF-x-icu"), 12679)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "fr-MG-x-icu"), 12680)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "fr-ML-x-icu"), 12681)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "fr-MQ-x-icu"), 12682)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "fr-MR-x-icu"), 12683)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "fr-MU-x-icu"), 12684)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "fr-NC-x-icu"), 12685)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "fr-NE-x-icu"), 12686)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "fr-PF-x-icu"), 12687)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "fr-PM-x-icu"), 12688)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "fr-RE-x-icu"), 12689)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "fr-RW-x-icu"), 12690)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "fr-SC-x-icu"), 12691)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "fr-SN-x-icu"), 12692)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "fr-SY-x-icu"), 12693)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "fr-TD-x-icu"), 12694)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "fr-TG-x-icu"), 12695)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "fr-TN-x-icu"), 12696)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "fr-VU-x-icu"), 12697)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "fr-WF-x-icu"), 12698)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "fr-YT-x-icu"), 12699)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "fur-x-icu"), 12700)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "fur-IT-x-icu"), 12701)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "fy-x-icu"), 12702)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "fy-NL-x-icu"), 12703)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "ga-x-icu"), 12704)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "ga-GB-x-icu"), 12705)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "ga-IE-x-icu"), 12706)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "gd-x-icu"), 12707)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "gd-GB-x-icu"), 12708)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "gl-x-icu"), 12709)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "gl-ES-x-icu"), 12710)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "gsw-x-icu"), 12711)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "gsw-CH-x-icu"), 12712)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "gsw-FR-x-icu"), 12713)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "gsw-LI-x-icu"), 12714)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "gu-x-icu"), 12715)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "gu-IN-x-icu"), 12716)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "guz-x-icu"), 12717)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "guz-KE-x-icu"), 12718)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "gv-x-icu"), 12719)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "gv-IM-x-icu"), 12720)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "ha-x-icu"), 12721)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "ha-GH-x-icu"), 12722)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "ha-NE-x-icu"), 12723)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "ha-NG-x-icu"), 12724)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "haw-x-icu"), 12725)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "haw-US-x-icu"), 12726)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "he-x-icu"), 12727)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "he-IL-x-icu"), 12728)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "hi-x-icu"), 12729)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "hi-IN-x-icu"), 12730)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "hr-x-icu"), 12731)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "hr-BA-x-icu"), 12732)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "hr-HR-x-icu"), 12733)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "hsb-x-icu"), 12734)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "hsb-DE-x-icu"), 12735)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "hu-x-icu"), 12736)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "hu-HU-x-icu"), 12737)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "hy-x-icu"), 12738)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "hy-AM-x-icu"), 12739)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "ia-x-icu"), 27401)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "ia-001-x-icu"), 27411)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "id-x-icu"), 27421)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "id-ID-x-icu"), 27431)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "ig-x-icu"), 27441)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "ig-NG-x-icu"), 27451)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "ii-x-icu"), 27461)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "ii-CN-x-icu"), 27471)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "is-x-icu"), 27481)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "is-IS-x-icu"), 27491)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "it-x-icu"), 27501)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "it-CH-x-icu"), 27511)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "it-IT-x-icu"), 27521)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "it-SM-x-icu"), 27531)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "it-VA-x-icu"), 27541)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "ja-x-icu"), 27551)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "ja-JP-x-icu"), 27561)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "jgo-x-icu"), 27571)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "jgo-CM-x-icu"), 27581)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "jmc-x-icu"), 27591)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "jmc-TZ-x-icu"), 27601)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "jv-x-icu"), 27611)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "jv-ID-x-icu"), 27621)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "ka-x-icu"), 27631)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "ka-GE-x-icu"), 27641)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "kab-x-icu"), 27651)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "kab-DZ-x-icu"), 27661)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "kam-x-icu"), 27671)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "kam-KE-x-icu"), 27681)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "kde-x-icu"), 27691)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "kde-TZ-x-icu"), 27701)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "kea-x-icu"), 27711)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "kea-CV-x-icu"), 27721)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "khq-x-icu"), 27731)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "khq-ML-x-icu"), 27741)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "ki-x-icu"), 27751)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "ki-KE-x-icu"), 27761)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "kk-x-icu"), 27771)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "kk-KZ-x-icu"), 27781)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "kkj-x-icu"), 27791)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "kkj-CM-x-icu"), 27801)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "kl-x-icu"), 27811)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "kl-GL-x-icu"), 27821)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "kln-x-icu"), 27831)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "kln-KE-x-icu"), 27841)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "km-x-icu"), 27851)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "km-KH-x-icu"), 27861)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "kn-x-icu"), 27871)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "kn-IN-x-icu"), 27881)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "ko-x-icu"), 27891)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "ko-KP-x-icu"), 27901)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "ko-KR-x-icu"), 27911)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "kok-x-icu"), 27921)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "kok-IN-x-icu"), 27931)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "ks-x-icu"), 27941)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "ks-Arab-x-icu"), 27951)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "ks-Arab-IN-x-icu"), 27961)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "ksb-x-icu"), 27971)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "ksb-TZ-x-icu"), 27981)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "ksf-x-icu"), 27991)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "ksf-CM-x-icu"), 28001)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "ksh-x-icu"), 28011)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "ksh-DE-x-icu"), 28021)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "ku-x-icu"), 28031)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "ku-TR-x-icu"), 28041)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "kw-x-icu"), 28051)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "kw-GB-x-icu"), 28061)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "ky-x-icu"), 28071)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "ky-KG-x-icu"), 28081)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "lag-x-icu"), 28091)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "lag-TZ-x-icu"), 28101)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "lb-x-icu"), 28111)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "lb-LU-x-icu"), 28121)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "lg-x-icu"), 28131)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "lg-UG-x-icu"), 28141)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "lkt-x-icu"), 28151)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "lkt-US-x-icu"), 28161)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "ln-x-icu"), 28171)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "ln-AO-x-icu"), 28181)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "ln-CD-x-icu"), 28191)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "ln-CF-x-icu"), 28201)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "ln-CG-x-icu"), 28211)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "lo-x-icu"), 28221)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "lo-LA-x-icu"), 28231)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "lrc-x-icu"), 28241)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "lrc-IQ-x-icu"), 28251)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "lrc-IR-x-icu"), 28261)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "lt-x-icu"), 28271)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "lt-LT-x-icu"), 28281)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "lu-x-icu"), 28291)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "lu-CD-x-icu"), 28301)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "luo-x-icu"), 28311)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "luo-KE-x-icu"), 28321)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "luy-x-icu"), 28331)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "luy-KE-x-icu"), 28341)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "lv-x-icu"), 28351)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "lv-LV-x-icu"), 28361)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "mai-x-icu"), 28371)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "mai-IN-x-icu"), 28381)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "mas-x-icu"), 28391)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "mas-KE-x-icu"), 28401)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "mas-TZ-x-icu"), 28411)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "mer-x-icu"), 28421)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "mer-KE-x-icu"), 28431)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "mfe-x-icu"), 28441)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "mfe-MU-x-icu"), 28451)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "mg-x-icu"), 28461)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "mg-MG-x-icu"), 28471)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "mgh-x-icu"), 28481)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "mgh-MZ-x-icu"), 28491)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "mgo-x-icu"), 28501)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "mgo-CM-x-icu"), 28511)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "mi-x-icu"), 28521)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "mi-NZ-x-icu"), 28531)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "mk-x-icu"), 28541)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "mk-MK-x-icu"), 28551)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "ml-x-icu"), 28561)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "ml-IN-x-icu"), 28571)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "mn-x-icu"), 28581)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "mn-MN-x-icu"), 28591)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "mni-x-icu"), 28601)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "mni-Beng-x-icu"), 28611)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "mni-Beng-IN-x-icu"), 28621)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "mr-x-icu"), 28631)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "mr-IN-x-icu"), 28641)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "ms-x-icu"), 28651)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "ms-BN-x-icu"), 28661)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "ms-ID-x-icu"), 28671)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "ms-MY-x-icu"), 28681)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "ms-SG-x-icu"), 28691)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "mt-x-icu"), 28701)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "mt-MT-x-icu"), 28711)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "mua-x-icu"), 28721)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "mua-CM-x-icu"), 28731)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "my-x-icu"), 28741)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "my-MM-x-icu"), 28751)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "mzn-x-icu"), 28761)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "mzn-IR-x-icu"), 28771)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "naq-x-icu"), 28781)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "naq-NA-x-icu"), 28791)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "nb-x-icu"), 28801)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "nb-NO-x-icu"), 28811)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "nb-SJ-x-icu"), 28821)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "nd-x-icu"), 28831)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "nd-ZW-x-icu"), 28841)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "nds-x-icu"), 28851)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "nds-DE-x-icu"), 28861)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "nds-NL-x-icu"), 28871)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "ne-x-icu"), 28881)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "ne-IN-x-icu"), 28891)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "ne-NP-x-icu"), 28901)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "nl-x-icu"), 28911)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "nl-AW-x-icu"), 28921)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "nl-BE-x-icu"), 28931)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "nl-BQ-x-icu"), 28941)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "nl-CW-x-icu"), 28951)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "nl-NL-x-icu"), 28961)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "nl-SR-x-icu"), 28971)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "nl-SX-x-icu"), 28981)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "nmg-x-icu"), 28991)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "nmg-CM-x-icu"), 29001)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "nn-x-icu"), 29011)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "nn-NO-x-icu"), 29021)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "nnh-x-icu"), 29031)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "nnh-CM-x-icu"), 29041)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "nus-x-icu"), 29051)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "nus-SS-x-icu"), 29061)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "nyn-x-icu"), 29071)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "nyn-UG-x-icu"), 29081)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "om-x-icu"), 29091)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "om-ET-x-icu"), 29101)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "om-KE-x-icu"), 29111)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "or-x-icu"), 29121)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "or-IN-x-icu"), 29131)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "os-x-icu"), 29141)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "os-GE-x-icu"), 29151)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "os-RU-x-icu"), 29161)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "pa-x-icu"), 29171)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "pa-Arab-x-icu"), 29181)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "pa-Arab-PK-x-icu"), 29191)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "pa-Guru-x-icu"), 29201)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "pa-Guru-IN-x-icu"), 29211)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "pcm-x-icu"), 29221)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "pcm-NG-x-icu"), 29231)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "pl-x-icu"), 29241)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "pl-PL-x-icu"), 29251)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "ps-x-icu"), 29261)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "ps-AF-x-icu"), 29271)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "ps-PK-x-icu"), 29281)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "pt-x-icu"), 29291)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "pt-AO-x-icu"), 29301)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "pt-BR-x-icu"), 29311)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "pt-CH-x-icu"), 29321)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "pt-CV-x-icu"), 29331)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "pt-GQ-x-icu"), 29341)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "pt-GW-x-icu"), 29351)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "pt-LU-x-icu"), 29361)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "pt-MO-x-icu"), 29371)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "pt-MZ-x-icu"), 29381)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "pt-PT-x-icu"), 29391)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "pt-ST-x-icu"), 29401)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "pt-TL-x-icu"), 29411)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "qu-x-icu"), 29421)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "qu-BO-x-icu"), 29431)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "qu-EC-x-icu"), 29441)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "qu-PE-x-icu"), 29451)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "rm-x-icu"), 29461)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "rm-CH-x-icu"), 29471)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "rn-x-icu"), 29481)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "rn-BI-x-icu"), 29491)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "ro-x-icu"), 29501)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "ro-MD-x-icu"), 29511)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "ro-RO-x-icu"), 29521)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "rof-x-icu"), 29531)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "rof-TZ-x-icu"), 29541)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "ru-x-icu"), 29551)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "ru-BY-x-icu"), 29561)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "ru-KG-x-icu"), 29571)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "ru-KZ-x-icu"), 29581)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "ru-MD-x-icu"), 29591)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "ru-RU-x-icu"), 29601)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "ru-UA-x-icu"), 29611)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "rw-x-icu"), 29621)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "rw-RW-x-icu"), 29631)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "rwk-x-icu"), 29641)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "rwk-TZ-x-icu"), 29651)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "sah-x-icu"), 29661)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "sah-RU-x-icu"), 29671)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "saq-x-icu"), 29681)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "saq-KE-x-icu"), 29691)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "sat-x-icu"), 29701)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "sat-Olck-x-icu"), 29711)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "sat-Olck-IN-x-icu"), 29721)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "sbp-x-icu"), 29731)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "sbp-TZ-x-icu"), 29741)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "sd-x-icu"), 29751)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "sd-Arab-x-icu"), 29761)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "sd-Arab-PK-x-icu"), 29771)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "sd-Deva-x-icu"), 29781)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "sd-Deva-IN-x-icu"), 29791)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "se-x-icu"), 29801)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "se-FI-x-icu"), 29811)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "se-NO-x-icu"), 29821)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "se-SE-x-icu"), 29831)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "seh-x-icu"), 29841)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "seh-MZ-x-icu"), 29851)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "ses-x-icu"), 29861)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "ses-ML-x-icu"), 29871)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "sg-x-icu"), 29881)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "sg-CF-x-icu"), 29891)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "shi-x-icu"), 29901)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "shi-Latn-x-icu"), 29911)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "shi-Latn-MA-x-icu"), 29921)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "shi-Tfng-x-icu"), 29931)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "shi-Tfng-MA-x-icu"), 29941)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "si-x-icu"), 29951)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "si-LK-x-icu"), 29961)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "sk-x-icu"), 29971)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "sk-SK-x-icu"), 29981)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "sl-x-icu"), 29991)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "sl-SI-x-icu"), 30001)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "smn-x-icu"), 30011)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "smn-FI-x-icu"), 30021)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "sn-x-icu"), 30031)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "sn-ZW-x-icu"), 30041)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "so-x-icu"), 30051)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "so-DJ-x-icu"), 30061)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "so-ET-x-icu"), 30071)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "so-KE-x-icu"), 30081)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "so-SO-x-icu"), 30091)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "sq-x-icu"), 30101)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "sq-AL-x-icu"), 30111)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "sq-MK-x-icu"), 30121)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "sq-XK-x-icu"), 30131)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "sr-x-icu"), 30141)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "sr-Cyrl-x-icu"), 30151)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "sr-Cyrl-BA-x-icu"), 30161)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "sr-Cyrl-ME-x-icu"), 30171)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "sr-Cyrl-RS-x-icu"), 30181)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "sr-Cyrl-XK-x-icu"), 30191)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "sr-Latn-x-icu"), 30201)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "sr-Latn-BA-x-icu"), 30211)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "sr-Latn-ME-x-icu"), 30221)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "sr-Latn-RS-x-icu"), 30231)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "sr-Latn-XK-x-icu"), 30241)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "su-x-icu"), 30251)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "su-Latn-x-icu"), 30261)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "su-Latn-ID-x-icu"), 30271)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "sv-x-icu"), 30281)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "sv-AX-x-icu"), 30291)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "sv-FI-x-icu"), 30301)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "sv-SE-x-icu"), 30311)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "sw-x-icu"), 30321)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "sw-CD-x-icu"), 30331)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "sw-KE-x-icu"), 30341)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "sw-TZ-x-icu"), 30351)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "sw-UG-x-icu"), 30361)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "ta-x-icu"), 30371)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "ta-IN-x-icu"), 30381)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "ta-LK-x-icu"), 30391)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "ta-MY-x-icu"), 30401)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "ta-SG-x-icu"), 30411)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "te-x-icu"), 30421)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "te-IN-x-icu"), 30431)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "teo-x-icu"), 30441)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "teo-KE-x-icu"), 30451)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "teo-UG-x-icu"), 30461)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "tg-x-icu"), 30471)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "tg-TJ-x-icu"), 30481)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "th-x-icu"), 30491)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "th-TH-x-icu"), 30501)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "ti-x-icu"), 30511)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "ti-ER-x-icu"), 30521)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "ti-ET-x-icu"), 30531)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "tk-x-icu"), 30541)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "tk-TM-x-icu"), 30551)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "to-x-icu"), 30561)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "to-TO-x-icu"), 30571)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "tr-x-icu"), 30581)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "tr-CY-x-icu"), 30591)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "tr-TR-x-icu"), 30601)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "tt-x-icu"), 30611)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "tt-RU-x-icu"), 30621)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "twq-x-icu"), 30631)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "twq-NE-x-icu"), 30641)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "tzm-x-icu"), 30651)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "tzm-MA-x-icu"), 30661)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "ug-x-icu"), 30671)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "ug-CN-x-icu"), 30681)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "uk-x-icu"), 30691)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "uk-UA-x-icu"), 30701)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "ur-x-icu"), 30711)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "ur-IN-x-icu"), 30721)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "ur-PK-x-icu"), 30731)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "uz-x-icu"), 30741)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "uz-Arab-x-icu"), 30751)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "uz-Arab-AF-x-icu"), 30761)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "uz-Cyrl-x-icu"), 30771)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "uz-Cyrl-UZ-x-icu"), 30781)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "uz-Latn-x-icu"), 30791)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "uz-Latn-UZ-x-icu"), 30801)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "vai-x-icu"), 30811)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "vai-Latn-x-icu"), 30821)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "vai-Latn-LR-x-icu"), 30831)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "vai-Vaii-x-icu"), 30841)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "vai-Vaii-LR-x-icu"), 30851)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "vi-x-icu"), 30861)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "vi-VN-x-icu"), 30871)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "vun-x-icu"), 30881)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "vun-TZ-x-icu"), 30891)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "wae-x-icu"), 30901)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "wae-CH-x-icu"), 30911)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "wo-x-icu"), 30921)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "wo-SN-x-icu"), 30931)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "xh-x-icu"), 30941)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "xh-ZA-x-icu"), 30951)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "xog-x-icu"), 30961)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "xog-UG-x-icu"), 30971)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "yav-x-icu"), 30981)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "yav-CM-x-icu"), 30991)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "yi-x-icu"), 31001)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "yi-001-x-icu"), 31011)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "yo-x-icu"), 31021)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "yo-BJ-x-icu"), 31031)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "yo-NG-x-icu"), 31041)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "yue-x-icu"), 31051)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "yue-Hans-x-icu"), 31061)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "yue-Hans-CN-x-icu"), 31071)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "yue-Hant-x-icu"), 31081)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "yue-Hant-HK-x-icu"), 31091)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "zgh-x-icu"), 31101)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "zgh-MA-x-icu"), 31111)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "zh-x-icu"), 31121)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "zh-Hans-x-icu"), 31131)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "zh-Hans-CN-x-icu"), 31141)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "zh-Hans-HK-x-icu"), 31151)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "zh-Hans-MO-x-icu"), 31161)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "zh-Hans-SG-x-icu"), 31171)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "zh-Hant-x-icu"), 31181)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "zh-Hant-HK-x-icu"), 31191)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "zh-Hant-MO-x-icu"), 31201)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "zh-Hant-TW-x-icu"), 31211)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "zu-x-icu"), 31221)
globalCache.setBuiltIn(NewId(Section_Collation, "pg_catalog", "zu-ZA-x-icu"), 31231)
}
+22
View File
@@ -0,0 +1,22 @@
// Copyright 2024 Dolthub, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package id
// This adds all of the built-in databases to the cache.
func init() {
globalCache.setBuiltIn(NewId(Section_Database, "template1"), 1)
globalCache.setBuiltIn(NewId(Section_Database, "template0"), 4)
globalCache.setBuiltIn(NewId(Section_Database, "postgres"), 5)
}
File diff suppressed because it is too large Load Diff
+23
View File
@@ -0,0 +1,23 @@
// Copyright 2024 Dolthub, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package id
// This adds all of the built-in schemas to the cache.
func init() {
globalCache.setBuiltIn(NewId(Section_Namespace, "pg_catalog"), 11)
globalCache.setBuiltIn(NewId(Section_Namespace, "pg_toast"), 99)
globalCache.setBuiltIn(NewId(Section_Namespace, "public"), 2200)
globalCache.setBuiltIn(NewId(Section_Namespace, "information_schema"), 13183)
}
+188
View File
@@ -0,0 +1,188 @@
// Copyright 2024 Dolthub, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package id
import "github.com/lib/pq/oid"
// This adds all of the built-in type IDs to the cache.
func init() {
globalCache.setBuiltIn(NewId(Section_Type, "pg_catalog", "_abstime"), uint32(oid.T__abstime))
globalCache.setBuiltIn(NewId(Section_Type, "pg_catalog", "_aclitem"), uint32(oid.T__aclitem))
globalCache.setBuiltIn(NewId(Section_Type, "pg_catalog", "_bit"), uint32(oid.T__bit))
globalCache.setBuiltIn(NewId(Section_Type, "pg_catalog", "_bool"), uint32(oid.T__bool))
globalCache.setBuiltIn(NewId(Section_Type, "pg_catalog", "_box"), uint32(oid.T__box))
globalCache.setBuiltIn(NewId(Section_Type, "pg_catalog", "_bpchar"), uint32(oid.T__bpchar))
globalCache.setBuiltIn(NewId(Section_Type, "pg_catalog", "_bytea"), uint32(oid.T__bytea))
globalCache.setBuiltIn(NewId(Section_Type, "pg_catalog", "_char"), uint32(oid.T__char))
globalCache.setBuiltIn(NewId(Section_Type, "pg_catalog", "_cid"), uint32(oid.T__cid))
globalCache.setBuiltIn(NewId(Section_Type, "pg_catalog", "_cidr"), uint32(oid.T__cidr))
globalCache.setBuiltIn(NewId(Section_Type, "pg_catalog", "_circle"), uint32(oid.T__circle))
globalCache.setBuiltIn(NewId(Section_Type, "pg_catalog", "_cstring"), uint32(oid.T__cstring))
globalCache.setBuiltIn(NewId(Section_Type, "pg_catalog", "_date"), uint32(oid.T__date))
globalCache.setBuiltIn(NewId(Section_Type, "pg_catalog", "_daterange"), uint32(oid.T__daterange))
globalCache.setBuiltIn(NewId(Section_Type, "pg_catalog", "_float4"), uint32(oid.T__float4))
globalCache.setBuiltIn(NewId(Section_Type, "pg_catalog", "_float8"), uint32(oid.T__float8))
globalCache.setBuiltIn(NewId(Section_Type, "pg_catalog", "_gtsvector"), uint32(oid.T__gtsvector))
globalCache.setBuiltIn(NewId(Section_Type, "pg_catalog", "_inet"), uint32(oid.T__inet))
globalCache.setBuiltIn(NewId(Section_Type, "pg_catalog", "_int2"), uint32(oid.T__int2))
globalCache.setBuiltIn(NewId(Section_Type, "pg_catalog", "_int2vector"), uint32(oid.T__int2vector))
globalCache.setBuiltIn(NewId(Section_Type, "pg_catalog", "_int4"), uint32(oid.T__int4))
globalCache.setBuiltIn(NewId(Section_Type, "pg_catalog", "_int4range"), uint32(oid.T__int4range))
globalCache.setBuiltIn(NewId(Section_Type, "pg_catalog", "_int8"), uint32(oid.T__int8))
globalCache.setBuiltIn(NewId(Section_Type, "pg_catalog", "_int8range"), uint32(oid.T__int8range))
globalCache.setBuiltIn(NewId(Section_Type, "pg_catalog", "_interval"), uint32(oid.T__interval))
globalCache.setBuiltIn(NewId(Section_Type, "pg_catalog", "_json"), uint32(oid.T__json))
globalCache.setBuiltIn(NewId(Section_Type, "pg_catalog", "_jsonb"), uint32(oid.T__jsonb))
globalCache.setBuiltIn(NewId(Section_Type, "pg_catalog", "_line"), uint32(oid.T__line))
globalCache.setBuiltIn(NewId(Section_Type, "pg_catalog", "_lseg"), uint32(oid.T__lseg))
globalCache.setBuiltIn(NewId(Section_Type, "pg_catalog", "_macaddr"), uint32(oid.T__macaddr))
globalCache.setBuiltIn(NewId(Section_Type, "pg_catalog", "_money"), uint32(oid.T__money))
globalCache.setBuiltIn(NewId(Section_Type, "pg_catalog", "_name"), uint32(oid.T__name))
globalCache.setBuiltIn(NewId(Section_Type, "pg_catalog", "_numeric"), uint32(oid.T__numeric))
globalCache.setBuiltIn(NewId(Section_Type, "pg_catalog", "_numrange"), uint32(oid.T__numrange))
globalCache.setBuiltIn(NewId(Section_Type, "pg_catalog", "_oid"), uint32(oid.T__oid))
globalCache.setBuiltIn(NewId(Section_Type, "pg_catalog", "_oidvector"), uint32(oid.T__oidvector))
globalCache.setBuiltIn(NewId(Section_Type, "pg_catalog", "_path"), uint32(oid.T__path))
globalCache.setBuiltIn(NewId(Section_Type, "pg_catalog", "_pg_lsn"), uint32(oid.T__pg_lsn))
globalCache.setBuiltIn(NewId(Section_Type, "pg_catalog", "_point"), uint32(oid.T__point))
globalCache.setBuiltIn(NewId(Section_Type, "pg_catalog", "_polygon"), uint32(oid.T__polygon))
globalCache.setBuiltIn(NewId(Section_Type, "pg_catalog", "_record"), uint32(oid.T__record))
globalCache.setBuiltIn(NewId(Section_Type, "pg_catalog", "_refcursor"), uint32(oid.T__refcursor))
globalCache.setBuiltIn(NewId(Section_Type, "pg_catalog", "_regclass"), uint32(oid.T__regclass))
globalCache.setBuiltIn(NewId(Section_Type, "pg_catalog", "_regconfig"), uint32(oid.T__regconfig))
globalCache.setBuiltIn(NewId(Section_Type, "pg_catalog", "_regdictionary"), uint32(oid.T__regdictionary))
globalCache.setBuiltIn(NewId(Section_Type, "pg_catalog", "_regnamespace"), uint32(oid.T__regnamespace))
globalCache.setBuiltIn(NewId(Section_Type, "pg_catalog", "_regoper"), uint32(oid.T__regoper))
globalCache.setBuiltIn(NewId(Section_Type, "pg_catalog", "_regoperator"), uint32(oid.T__regoperator))
globalCache.setBuiltIn(NewId(Section_Type, "pg_catalog", "_regproc"), uint32(oid.T__regproc))
globalCache.setBuiltIn(NewId(Section_Type, "pg_catalog", "_regprocedure"), uint32(oid.T__regprocedure))
globalCache.setBuiltIn(NewId(Section_Type, "pg_catalog", "_regrole"), uint32(oid.T__regrole))
globalCache.setBuiltIn(NewId(Section_Type, "pg_catalog", "_regtype"), uint32(oid.T__regtype))
globalCache.setBuiltIn(NewId(Section_Type, "pg_catalog", "_reltime"), uint32(oid.T__reltime))
globalCache.setBuiltIn(NewId(Section_Type, "pg_catalog", "_text"), uint32(oid.T__text))
globalCache.setBuiltIn(NewId(Section_Type, "pg_catalog", "_tid"), uint32(oid.T__tid))
globalCache.setBuiltIn(NewId(Section_Type, "pg_catalog", "_time"), uint32(oid.T__time))
globalCache.setBuiltIn(NewId(Section_Type, "pg_catalog", "_timestamp"), uint32(oid.T__timestamp))
globalCache.setBuiltIn(NewId(Section_Type, "pg_catalog", "_timestamptz"), uint32(oid.T__timestamptz))
globalCache.setBuiltIn(NewId(Section_Type, "pg_catalog", "_timetz"), uint32(oid.T__timetz))
globalCache.setBuiltIn(NewId(Section_Type, "pg_catalog", "_tinterval"), uint32(oid.T__tinterval))
globalCache.setBuiltIn(NewId(Section_Type, "pg_catalog", "_tsquery"), uint32(oid.T__tsquery))
globalCache.setBuiltIn(NewId(Section_Type, "pg_catalog", "_tsrange"), uint32(oid.T__tsrange))
globalCache.setBuiltIn(NewId(Section_Type, "pg_catalog", "_tstzrange"), uint32(oid.T__tstzrange))
globalCache.setBuiltIn(NewId(Section_Type, "pg_catalog", "_tsvector"), uint32(oid.T__tsvector))
globalCache.setBuiltIn(NewId(Section_Type, "pg_catalog", "_txid_snapshot"), uint32(oid.T__txid_snapshot))
globalCache.setBuiltIn(NewId(Section_Type, "pg_catalog", "_uuid"), uint32(oid.T__uuid))
globalCache.setBuiltIn(NewId(Section_Type, "pg_catalog", "_varbit"), uint32(oid.T__varbit))
globalCache.setBuiltIn(NewId(Section_Type, "pg_catalog", "_varchar"), uint32(oid.T__varchar))
globalCache.setBuiltIn(NewId(Section_Type, "pg_catalog", "_xid"), uint32(oid.T__xid))
globalCache.setBuiltIn(NewId(Section_Type, "pg_catalog", "_xml"), uint32(oid.T__xml))
globalCache.setBuiltIn(NewId(Section_Type, "pg_catalog", "abstime"), uint32(oid.T_abstime))
globalCache.setBuiltIn(NewId(Section_Type, "pg_catalog", "aclitem"), uint32(oid.T_aclitem))
globalCache.setBuiltIn(NewId(Section_Type, "pg_catalog", "any"), uint32(oid.T_any))
globalCache.setBuiltIn(NewId(Section_Type, "pg_catalog", "anyarray"), uint32(oid.T_anyarray))
globalCache.setBuiltIn(NewId(Section_Type, "pg_catalog", "anyelement"), uint32(oid.T_anyelement))
globalCache.setBuiltIn(NewId(Section_Type, "pg_catalog", "anyenum"), uint32(oid.T_anyenum))
globalCache.setBuiltIn(NewId(Section_Type, "pg_catalog", "anynonarray"), uint32(oid.T_anynonarray))
globalCache.setBuiltIn(NewId(Section_Type, "pg_catalog", "anyrange"), uint32(oid.T_anyrange))
globalCache.setBuiltIn(NewId(Section_Type, "pg_catalog", "bit"), uint32(oid.T_bit))
globalCache.setBuiltIn(NewId(Section_Type, "pg_catalog", "bool"), uint32(oid.T_bool))
globalCache.setBuiltIn(NewId(Section_Type, "pg_catalog", "box"), uint32(oid.T_box))
globalCache.setBuiltIn(NewId(Section_Type, "pg_catalog", "bpchar"), uint32(oid.T_bpchar))
globalCache.setBuiltIn(NewId(Section_Type, "pg_catalog", "bytea"), uint32(oid.T_bytea))
globalCache.setBuiltIn(NewId(Section_Type, "pg_catalog", "char"), uint32(oid.T_char))
globalCache.setBuiltIn(NewId(Section_Type, "pg_catalog", "cid"), uint32(oid.T_cid))
globalCache.setBuiltIn(NewId(Section_Type, "pg_catalog", "cidr"), uint32(oid.T_cidr))
globalCache.setBuiltIn(NewId(Section_Type, "pg_catalog", "circle"), uint32(oid.T_circle))
globalCache.setBuiltIn(NewId(Section_Type, "pg_catalog", "cstring"), uint32(oid.T_cstring))
globalCache.setBuiltIn(NewId(Section_Type, "pg_catalog", "date"), uint32(oid.T_date))
globalCache.setBuiltIn(NewId(Section_Type, "pg_catalog", "daterange"), uint32(oid.T_daterange))
globalCache.setBuiltIn(NewId(Section_Type, "pg_catalog", "event_trigger"), uint32(oid.T_event_trigger))
globalCache.setBuiltIn(NewId(Section_Type, "pg_catalog", "fdw_handler"), uint32(oid.T_fdw_handler))
globalCache.setBuiltIn(NewId(Section_Type, "pg_catalog", "float4"), uint32(oid.T_float4))
globalCache.setBuiltIn(NewId(Section_Type, "pg_catalog", "float8"), uint32(oid.T_float8))
globalCache.setBuiltIn(NewId(Section_Type, "pg_catalog", "gtsvector"), uint32(oid.T_gtsvector))
globalCache.setBuiltIn(NewId(Section_Type, "pg_catalog", "index_am_handler"), uint32(oid.T_index_am_handler))
globalCache.setBuiltIn(NewId(Section_Type, "pg_catalog", "inet"), uint32(oid.T_inet))
globalCache.setBuiltIn(NewId(Section_Type, "pg_catalog", "int2"), uint32(oid.T_int2))
globalCache.setBuiltIn(NewId(Section_Type, "pg_catalog", "int2vector"), uint32(oid.T_int2vector))
globalCache.setBuiltIn(NewId(Section_Type, "pg_catalog", "int4"), uint32(oid.T_int4))
globalCache.setBuiltIn(NewId(Section_Type, "pg_catalog", "int4range"), uint32(oid.T_int4range))
globalCache.setBuiltIn(NewId(Section_Type, "pg_catalog", "int8"), uint32(oid.T_int8))
globalCache.setBuiltIn(NewId(Section_Type, "pg_catalog", "int8range"), uint32(oid.T_int8range))
globalCache.setBuiltIn(NewId(Section_Type, "pg_catalog", "internal"), uint32(oid.T_internal))
globalCache.setBuiltIn(NewId(Section_Type, "pg_catalog", "interval"), uint32(oid.T_interval))
globalCache.setBuiltIn(NewId(Section_Type, "pg_catalog", "json"), uint32(oid.T_json))
globalCache.setBuiltIn(NewId(Section_Type, "pg_catalog", "jsonb"), uint32(oid.T_jsonb))
globalCache.setBuiltIn(NewId(Section_Type, "pg_catalog", "language_handler"), uint32(oid.T_language_handler))
globalCache.setBuiltIn(NewId(Section_Type, "pg_catalog", "line"), uint32(oid.T_line))
globalCache.setBuiltIn(NewId(Section_Type, "pg_catalog", "lseg"), uint32(oid.T_lseg))
globalCache.setBuiltIn(NewId(Section_Type, "pg_catalog", "macaddr"), uint32(oid.T_macaddr))
globalCache.setBuiltIn(NewId(Section_Type, "pg_catalog", "money"), uint32(oid.T_money))
globalCache.setBuiltIn(NewId(Section_Type, "pg_catalog", "name"), uint32(oid.T_name))
globalCache.setBuiltIn(NewId(Section_Type, "pg_catalog", "numeric"), uint32(oid.T_numeric))
globalCache.setBuiltIn(NewId(Section_Type, "pg_catalog", "numrange"), uint32(oid.T_numrange))
globalCache.setBuiltIn(NewId(Section_Type, "pg_catalog", "oid"), uint32(oid.T_oid))
globalCache.setBuiltIn(NewId(Section_Type, "pg_catalog", "oidvector"), uint32(oid.T_oidvector))
globalCache.setBuiltIn(NewId(Section_Type, "pg_catalog", "opaque"), uint32(oid.T_opaque))
globalCache.setBuiltIn(NewId(Section_Type, "pg_catalog", "path"), uint32(oid.T_path))
globalCache.setBuiltIn(NewId(Section_Type, "pg_catalog", "pg_attribute"), uint32(oid.T_pg_attribute))
globalCache.setBuiltIn(NewId(Section_Type, "pg_catalog", "pg_auth_members"), uint32(oid.T_pg_auth_members))
globalCache.setBuiltIn(NewId(Section_Type, "pg_catalog", "pg_authid"), uint32(oid.T_pg_authid))
globalCache.setBuiltIn(NewId(Section_Type, "pg_catalog", "pg_class"), uint32(oid.T_pg_class))
globalCache.setBuiltIn(NewId(Section_Type, "pg_catalog", "pg_database"), uint32(oid.T_pg_database))
globalCache.setBuiltIn(NewId(Section_Type, "pg_catalog", "pg_ddl_command"), uint32(oid.T_pg_ddl_command))
globalCache.setBuiltIn(NewId(Section_Type, "pg_catalog", "pg_lsn"), uint32(oid.T_pg_lsn))
globalCache.setBuiltIn(NewId(Section_Type, "pg_catalog", "pg_node_tree"), uint32(oid.T_pg_node_tree))
globalCache.setBuiltIn(NewId(Section_Type, "pg_catalog", "pg_proc"), uint32(oid.T_pg_proc))
globalCache.setBuiltIn(NewId(Section_Type, "pg_catalog", "pg_shseclabel"), uint32(oid.T_pg_shseclabel))
globalCache.setBuiltIn(NewId(Section_Type, "pg_catalog", "pg_type"), uint32(oid.T_pg_type))
globalCache.setBuiltIn(NewId(Section_Type, "pg_catalog", "point"), uint32(oid.T_point))
globalCache.setBuiltIn(NewId(Section_Type, "pg_catalog", "polygon"), uint32(oid.T_polygon))
globalCache.setBuiltIn(NewId(Section_Type, "pg_catalog", "record"), uint32(oid.T_record))
globalCache.setBuiltIn(NewId(Section_Type, "pg_catalog", "refcursor"), uint32(oid.T_refcursor))
globalCache.setBuiltIn(NewId(Section_Type, "pg_catalog", "regclass"), uint32(oid.T_regclass))
globalCache.setBuiltIn(NewId(Section_Type, "pg_catalog", "regconfig"), uint32(oid.T_regconfig))
globalCache.setBuiltIn(NewId(Section_Type, "pg_catalog", "regdictionary"), uint32(oid.T_regdictionary))
globalCache.setBuiltIn(NewId(Section_Type, "pg_catalog", "regnamespace"), uint32(oid.T_regnamespace))
globalCache.setBuiltIn(NewId(Section_Type, "pg_catalog", "regoper"), uint32(oid.T_regoper))
globalCache.setBuiltIn(NewId(Section_Type, "pg_catalog", "regoperator"), uint32(oid.T_regoperator))
globalCache.setBuiltIn(NewId(Section_Type, "pg_catalog", "regproc"), uint32(oid.T_regproc))
globalCache.setBuiltIn(NewId(Section_Type, "pg_catalog", "regprocedure"), uint32(oid.T_regprocedure))
globalCache.setBuiltIn(NewId(Section_Type, "pg_catalog", "regrole"), uint32(oid.T_regrole))
globalCache.setBuiltIn(NewId(Section_Type, "pg_catalog", "regtype"), uint32(oid.T_regtype))
globalCache.setBuiltIn(NewId(Section_Type, "pg_catalog", "reltime"), uint32(oid.T_reltime))
globalCache.setBuiltIn(NewId(Section_Type, "pg_catalog", "smgr"), uint32(oid.T_smgr))
globalCache.setBuiltIn(NewId(Section_Type, "pg_catalog", "text"), uint32(oid.T_text))
globalCache.setBuiltIn(NewId(Section_Type, "pg_catalog", "tid"), uint32(oid.T_tid))
globalCache.setBuiltIn(NewId(Section_Type, "pg_catalog", "time"), uint32(oid.T_time))
globalCache.setBuiltIn(NewId(Section_Type, "pg_catalog", "timestamp"), uint32(oid.T_timestamp))
globalCache.setBuiltIn(NewId(Section_Type, "pg_catalog", "timestamptz"), uint32(oid.T_timestamptz))
globalCache.setBuiltIn(NewId(Section_Type, "pg_catalog", "timetz"), uint32(oid.T_timetz))
globalCache.setBuiltIn(NewId(Section_Type, "pg_catalog", "tinterval"), uint32(oid.T_tinterval))
globalCache.setBuiltIn(NewId(Section_Type, "pg_catalog", "trigger"), uint32(oid.T_trigger))
globalCache.setBuiltIn(NewId(Section_Type, "pg_catalog", "tsm_handler"), uint32(oid.T_tsm_handler))
globalCache.setBuiltIn(NewId(Section_Type, "pg_catalog", "tsquery"), uint32(oid.T_tsquery))
globalCache.setBuiltIn(NewId(Section_Type, "pg_catalog", "tsrange"), uint32(oid.T_tsrange))
globalCache.setBuiltIn(NewId(Section_Type, "pg_catalog", "tstzrange"), uint32(oid.T_tstzrange))
globalCache.setBuiltIn(NewId(Section_Type, "pg_catalog", "tsvector"), uint32(oid.T_tsvector))
globalCache.setBuiltIn(NewId(Section_Type, "pg_catalog", "txid_snapshot"), uint32(oid.T_txid_snapshot))
globalCache.setBuiltIn(NewId(Section_Type, "pg_catalog", "unknown"), uint32(oid.T_unknown))
globalCache.setBuiltIn(NewId(Section_Type, "pg_catalog", "uuid"), uint32(oid.T_uuid))
globalCache.setBuiltIn(NewId(Section_Type, "pg_catalog", "varbit"), uint32(oid.T_varbit))
globalCache.setBuiltIn(NewId(Section_Type, "pg_catalog", "varchar"), uint32(oid.T_varchar))
globalCache.setBuiltIn(NewId(Section_Type, "pg_catalog", "void"), uint32(oid.T_void))
globalCache.setBuiltIn(NewId(Section_Type, "pg_catalog", "xid"), uint32(oid.T_xid))
globalCache.setBuiltIn(NewId(Section_Type, "pg_catalog", "xml"), uint32(oid.T_xml))
}
+260
View File
@@ -0,0 +1,260 @@
// Copyright 2024 Dolthub, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package id
import (
"bytes"
"fmt"
"strings"
"unsafe"
)
// Id uses one of two formats. Which format is being used is marked by the upper Section bit being either 0 or 1.
// Often, an ID contains information that will commonly be accessed by the item, so the first format is tailored for
// efficient retrieval of specific segments. If an item is larger than the size limit (255, size is stored as an uint8),
// then we use the second format, which inserts a separator between items. This allows Id to hold any data in case
// the need arises in the future, but in practice we'll only see the first format (since data will usually be
// identifiers or smaller embedded IDs). Id IDs will be accessed far more often than they'll be created, hence the
// focus on efficient retrieval rather than simplicity of storage.
//
// First format (upper bit is 0):
// The first byte is the section
// The second byte contains the number of segments N (up to 255 segments)
// The next N bytes contain the length of each respective segment (up to 255 bytes)
// The remaining bytes are the original string data, stored contiguously
// Second format (upper bit is 1):
// The first byte is the section
// The remaining bytes are the original string data, stored with the separator between each segment
const (
// idSeparator marks the different data sections in an Id. This is the null byte since that byte is invalid in
// all identifiers, so we can guarantee that it's safe to use as a separator. This is used when an individual data
// segment is larger than 254 bytes.
idSeparator = "\x00"
// formatMask is the upper bit that determines whether we're using the first or second format.
formatMask = uint8(0x80)
// Null is an empty, invalid ID.
Null Id = ""
// NullAccessMethod is an empty, invalid ID. This is exactly equivalent to Null.
NullAccessMethod AccessMethod = ""
// NullCast is an empty, invalid ID. This is exactly equivalent to Null.
NullCast Cast = ""
// NullCheck is an empty, invalid ID. This is exactly equivalent to Null.
NullCheck Check = ""
// NullCollation is an empty, invalid ID. This is exactly equivalent to Null.
NullCollation Collation = ""
// NullColumnDefault is an empty, invalid ID. This is exactly equivalent to Null.
NullColumnDefault ColumnDefault = ""
// NullDatabase is an empty, invalid ID. This is exactly equivalent to Null.
NullDatabase Database = ""
// NullEnumLabel is an empty, invalid ID. This is exactly equivalent to Null.
NullEnumLabel EnumLabel = ""
// NullExtension is an empty, invalid ID. This is exactly equivalent to Null.
NullExtension Extension = ""
// NullForeignKey is an empty, invalid ID. This is exactly equivalent to Null.
NullForeignKey ForeignKey = ""
// NullFunction is an empty, invalid ID. This is exactly equivalent to Null.
NullFunction Function = ""
// NullIndex is an empty, invalid ID. This is exactly equivalent to Null.
NullIndex Index = ""
// NullNamespace is an empty, invalid ID. This is exactly equivalent to Null.
NullNamespace Namespace = ""
// NullProcedure is an empty, invalid ID. This is exactly equivalent to Null.
NullProcedure Procedure = ""
// NullSequence is an empty, invalid ID. This is exactly equivalent to Null.
NullSequence Sequence = ""
// NullTable is an empty, invalid ID. This is exactly equivalent to Null.
NullTable Table = ""
// NullTrigger is an empty, invalid ID. This is exactly equivalent to Null.
NullTrigger Trigger = ""
// NullType is an empty, invalid ID. This is exactly equivalent to Null.
NullType Type = ""
// NullView is an empty, invalid ID. This is exactly equivalent to Null.
NullView View = ""
)
// Id is an ID that is used within Doltgres. This ID is never exposed to clients through any normal means, and
// exists solely for internal operations to be able to identify specific items. This functions as an internal
// replacement for Postgres' OIDs.
type Id string
// NewId constructs an Id using the given section and data. In general, you should prefer to use the `NewIDTYPE` that
// matches the Section that's being created, and then convert that to an Id for returning or storage. You almost never
// want to call this function directly.
func NewId(section Section, data ...string) Id {
if section == Section_Null {
// It's easier if there's only one canonical way to represent a null ID, so we'll return our constant instead of
// creating a new string
return Null
}
if len(data) > 255 {
return newIdSecondFormat(section, data)
}
buf := bytes.Buffer{}
buf.WriteByte(uint8(section))
buf.WriteByte(uint8(len(data)))
for _, segment := range data {
segmentLength := len(segment)
if segmentLength > 255 {
return newIdSecondFormat(section, data)
}
buf.WriteByte(uint8(segmentLength))
}
for _, segment := range data {
buf.WriteString(segment)
}
return Id(buf.Bytes())
}
// newIdSecondFormat constructs an Id using the given section and data. This always returns the second format (using the
// separator).
func newIdSecondFormat(section Section, data []string) Id {
buf := bytes.Buffer{}
buf.WriteByte(uint8(section) | formatMask)
for i, segment := range data {
if i > 0 {
buf.WriteString(idSeparator)
}
buf.WriteString(segment)
}
return Id(buf.Bytes())
}
// IsValid returns whether the Id is valid.
func (id Id) IsValid() bool {
// We don't allow setting the section to Section_Null, so we can do a simple length check
return len(id) > 0
}
// Section returns the Section for this Id.
func (id Id) Section() Section {
if len(id) == 0 {
return Section_Null
}
return Section(id[0] & (^formatMask))
}
// Data returns the original data used to create this Id.
func (id Id) Data() []string {
if len(id) <= 1 {
return nil
}
if id[0]&formatMask == formatMask {
// Second format
return strings.Split(string(id[1:]), idSeparator)
} else {
// First format
segmentCount := int(id[1])
data := id[2+segmentCount:] // We skip 2 for the section and count bytes, then the number of segment counts
segments := make([]string, segmentCount)
start := 0
for i := 0; i < segmentCount; i++ {
length := int(id[2+i])
segments[i] = string(data[start : start+length])
start += length
}
return segments
}
}
// SegmentCount returns the number of segments that were in the original data.
func (id Id) SegmentCount() int {
if len(id) <= 1 {
return 0
}
if id[0]&formatMask == formatMask {
// Second format
return len(id.Data())
} else {
// First format
return int(id[1])
}
}
// Segment returns the segment from the given index. An empty string is returned for an index not contained by the ID.
func (id Id) Segment(index int) string {
if index < 0 || len(id) <= 1 {
return ""
}
if id[0]&formatMask == formatMask {
// Second format
data := id.Data()
if index >= len(data) {
return ""
}
return data[index]
} else {
// First format
segmentCount := int(id[1])
data := id[2+segmentCount:] // We skip 2 for the section and count bytes, then the number of segment counts
if index >= segmentCount {
return ""
}
start := 0
currentLength := 0
for i := 0; i <= index; i++ {
start += currentLength
currentLength = int(id[2+i])
}
return string(data[start : start+currentLength])
}
}
// String returns a display-suitable version of the ID. Although the ID is implemented as a string, it should not be
// treated as a string except for the purposes of storage and retrieval.
func (id Id) String() string {
data := id.Data()
if len(data) == 0 {
return fmt.Sprintf(`{%s:[]}`, id.Section().String())
}
return fmt.Sprintf(`{%s:["%s"]}`, id.Section().String(), strings.Join(data, `","`))
}
// CaseString returns a quoted string that may be used to represent this ID in a switch-case.
func (id Id) CaseString() string {
if len(id) == 0 {
return `""`
}
if id[0]&formatMask == formatMask {
// Second format
data := strings.ReplaceAll(string(id[1:]), "\x00", `\x00`)
data = strings.ReplaceAll(data, `"`, `\x22`)
return fmt.Sprintf(`"\x%02x%s"`, id[0], data)
} else {
// First format
sb := strings.Builder{}
sb.Grow(len(id) + 32)
sb.WriteRune('"')
count := int(id[1])
sb.WriteString(fmt.Sprintf(`\x%02x\x%02x`, id[0], count))
for i := 0; i < count; i++ {
sb.WriteString(fmt.Sprintf(`\x%02x`, id[2+i]))
}
sb.WriteString(strings.ReplaceAll(string(id[2+count:]), `"`, `\x22`))
sb.WriteRune('"')
return sb.String()
}
}
// UnderlyingBytes returns the underlying bytes for the ID. These must not be modified, as this is intended solely for
// efficient usage of operations that require byte slices.
func (id Id) UnderlyingBytes() []byte {
return unsafe.Slice(unsafe.StringData(string(id)), len(id))
}
// usesSecondFormat returns whether the separator is used, which is the second format.
func (id Id) usesSecondFormat() bool {
return len(id) > 0 && id[0]&formatMask == formatMask
}

Some files were not shown because too many files have changed in this diff Show More