wehub-resource-sync
314 lines
13 KiB
Go
314 lines
13 KiB
Go
// Copyright 2025 Dolthub, Inc.
|
|
//
|
|
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
// you may not use this file except in compliance with the License.
|
|
// You may obtain a copy of the License at
|
|
//
|
|
// http://www.apache.org/licenses/LICENSE-2.0
|
|
//
|
|
// Unless required by applicable law or agreed to in writing, software
|
|
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
// See the License for the specific language governing permissions and
|
|
// limitations under the License.
|
|
|
|
package _go
|
|
|
|
import (
|
|
"fmt"
|
|
"testing"
|
|
|
|
"github.com/dolthub/go-mysql-server/enginetest/scriptgen/setup"
|
|
"github.com/jackc/pgx/v5/pgtype"
|
|
|
|
"github.com/dolthub/go-mysql-server/sql"
|
|
)
|
|
|
|
// This is a correctness test based on the AdaptiveEncodingTest in Dolt.
|
|
// Because Doltgres serializes its results over a Postgres connection, we can't inspect the encoding
|
|
// of the result row. But we can still confirm that using adaptable encoding doesn't change any expected behavior.
|
|
|
|
func makeTestBytes(size int, firstbyte byte) []byte {
|
|
bytes := make([]byte, size)
|
|
bytes[0] = firstbyte
|
|
return bytes
|
|
}
|
|
|
|
func makeTestVarbit(sizeInBits int) pgtype.Bits {
|
|
numBytes := sizeInBits / 8
|
|
leftoverBits := sizeInBits % 8
|
|
bytes := make([]byte, numBytes)
|
|
for i := 0; i < numBytes; i++ {
|
|
bytes[i] = 0xaa
|
|
}
|
|
|
|
// this is a little annoying, but if we have leftover bits, we need to pad out the remaining bits in the last byte with 0s
|
|
endingByte := byte(0)
|
|
for i := 0; i < leftoverBits/2; i++ {
|
|
endingByte |= 0b10 << (6 - i*2)
|
|
}
|
|
if leftoverBits > 0 {
|
|
bytes = append(bytes, endingByte)
|
|
}
|
|
|
|
return pgtype.Bits{
|
|
Bytes: bytes,
|
|
Len: int32(sizeInBits),
|
|
Valid: true,
|
|
}
|
|
}
|
|
|
|
// A 4000 byte file starting with 0x01 and then consisting of all zeros.
|
|
// This is larger than default target tuple size for outlining adaptive types.
|
|
// We expect a tuple to always store this value out-of-band
|
|
var fullSizeString = string(makeTestBytes(4000, 1))
|
|
|
|
// A 2000 byte file starting with 0x02 and then consisting of all zeros.
|
|
// This is over half of the default target tuple size for outlining adaptive types.
|
|
// We expect a tuple to be able to store this value inline once, but not twice.
|
|
var halfSizeString = string(makeTestBytes(2000, 2))
|
|
|
|
// A 10 byte file starting with 0x03 and then consisting of 10 zero bytes.
|
|
// This is file is smaller than an address hash.
|
|
// We expect a tuple to never store this value out-of-band.
|
|
var tinyString = string(makeTestBytes(10, 3))
|
|
|
|
// A 4000 byte file starting with ascii byte 1 and then consisting of all zeros.
|
|
// This is larger than default target tuple size for outlining adaptive types.
|
|
// We expect a tuple to always store this value out-of-band
|
|
var fullSizeVarbit = makeTestVarbit(4000)
|
|
|
|
// A 2000 byte file starting with ascii byte 1 and then consisting of all zeros.
|
|
// This is over half of the default target tuple size for outlining adaptive types.
|
|
// We expect a tuple to be able to store this value inline once, but not twice.
|
|
var halfSizeVarbit = makeTestVarbit(2000)
|
|
|
|
// A 10 byte file starting with ascii byte 1 and then consisting of 10 zero bytes.
|
|
// This is file is smaller than an address hash.
|
|
// We expect a tuple to never store this value out-of-band.
|
|
var tinyVarbit = makeTestVarbit(10)
|
|
|
|
func TestAdaptiveEncodingText(t *testing.T) {
|
|
fullSizeOutOfLineRepr := fullSizeString
|
|
columnTypes := []string{"varchar", "text"}
|
|
for _, columnType := range columnTypes {
|
|
t.Run(columnType, func(t *testing.T) {
|
|
RunScripts(t, []ScriptTest{
|
|
{
|
|
Name: "Adaptive Encoding With One Column",
|
|
SetUpScript: setup.SetupScript{
|
|
fmt.Sprintf(`create table blobt (i char(1) primary key, b %s);`, columnType),
|
|
fmt.Sprintf(`create table blobt2 (i char(2) primary key, b1 %s, b2 %s);`, columnType, columnType),
|
|
`insert into blobt values
|
|
('F', LOAD_FILE('testdata/fullSize')),
|
|
('H', LOAD_FILE('testdata/halfSize')),
|
|
('T', LOAD_FILE('testdata/tinyFile'))`,
|
|
},
|
|
Assertions: []ScriptTestAssertion{
|
|
{
|
|
Query: "select b from blobt where i = 'F'",
|
|
Expected: []sql.Row{{fullSizeString}},
|
|
},
|
|
{
|
|
// Files that can fit within a tuple are stored inline.
|
|
Query: "select b from blobt where i = 'H'",
|
|
Expected: []sql.Row{{halfSizeString}},
|
|
},
|
|
{
|
|
// An inlined adaptive column can be used in a filter.
|
|
Query: "select i from blobt where b = LOAD_FILE('testdata/fullSize')",
|
|
Expected: []sql.Row{{"F"}},
|
|
},
|
|
{
|
|
// An out-of-line adaptive column can be used in a filter.
|
|
Query: "select i from blobt where b = LOAD_FILE('testdata/halfSize')",
|
|
Expected: []sql.Row{{"H"}},
|
|
},
|
|
},
|
|
},
|
|
{
|
|
Name: "Adaptive Encoding With Two Columns",
|
|
SetUpScript: setup.SetupScript{
|
|
fmt.Sprintf(`create table blobt2 (i char(2) primary key, b1 %s, b2 %s);`, columnType, columnType),
|
|
`insert into blobt2 values
|
|
('FF', LOAD_FILE('testdata/fullSize'), LOAD_FILE('testdata/fullSize')),
|
|
('HF', LOAD_FILE('testdata/halfSize'), LOAD_FILE('testdata/fullSize')),
|
|
('TF', LOAD_FILE('testdata/tinyFile'), LOAD_FILE('testdata/fullSize')),
|
|
('FH', LOAD_FILE('testdata/fullSize'), LOAD_FILE('testdata/halfSize')),
|
|
('HH', LOAD_FILE('testdata/halfSize'), LOAD_FILE('testdata/halfSize')),
|
|
('TH', LOAD_FILE('testdata/tinyFile'), LOAD_FILE('testdata/halfSize')),
|
|
('FT', LOAD_FILE('testdata/fullSize'), LOAD_FILE('testdata/tinyFile')),
|
|
('HT', LOAD_FILE('testdata/halfSize'), LOAD_FILE('testdata/tinyFile')),
|
|
('TT', LOAD_FILE('testdata/tinyFile'), LOAD_FILE('testdata/tinyFile'))`,
|
|
},
|
|
Assertions: []ScriptTestAssertion{
|
|
{
|
|
// When a tuple with multiple adaptive columns is too large, columns are moved out-of-band from left to right.
|
|
// However, strings smaller than the address size (20 bytes) are never stored out-of-band.
|
|
Query: "select i, b1, b2 from blobt2",
|
|
Expected: []sql.Row{
|
|
{"FF", fullSizeString, fullSizeString},
|
|
{"HF", halfSizeString, fullSizeString},
|
|
{"TF", tinyString, fullSizeString},
|
|
{"FH", fullSizeString, halfSizeString},
|
|
{"HH", halfSizeString, halfSizeString},
|
|
{"TH", tinyString, halfSizeString},
|
|
{"FT", fullSizeString, tinyString},
|
|
{"HT", halfSizeString, tinyString},
|
|
{"TT", tinyString, tinyString},
|
|
},
|
|
},
|
|
{
|
|
// An adaptive column can be used in a filter when it doesn't have the same encoding in all rows.
|
|
Query: "select i from blobt2 where b1 = LOAD_FILE('testdata/halfSize')",
|
|
Expected: []sql.Row{{"HF"}, {"HH"}, {"HT"}},
|
|
},
|
|
{
|
|
// An adaptive column can be used in a filter when it doesn't have the same encoding in all rows.
|
|
Query: "select i from blobt2 where b2 = LOAD_FILE('testdata/halfSize')",
|
|
Expected: []sql.Row{{"FH"}, {"HH"}, {"TH"}},
|
|
},
|
|
{
|
|
// Test creating an index on an adaptive encoding column, matching against out-of-band values
|
|
Query: "CREATE INDEX bidx ON blobt2 (b1)",
|
|
},
|
|
{
|
|
Query: "select i, b1 FROM blobt2 WHERE b1 LIKE '\x01%'",
|
|
Expected: []sql.Row{
|
|
{"FF", fullSizeOutOfLineRepr},
|
|
{"FH", fullSizeOutOfLineRepr},
|
|
{"FT", fullSizeOutOfLineRepr},
|
|
},
|
|
},
|
|
{
|
|
// Test creating an index on an adaptive encoding column, matching against inline values
|
|
Query: "CREATE INDEX bidx2 ON blobt2 (b2)",
|
|
},
|
|
{
|
|
Query: "select i, b2 FROM blobt2 WHERE b2 LIKE '\x02%'",
|
|
Expected: []sql.Row{
|
|
{"FH", halfSizeString},
|
|
{"HH", halfSizeString},
|
|
{"TH", halfSizeString},
|
|
},
|
|
},
|
|
{
|
|
// Tuples containing adaptive columns should be independent of how the tuple was created.
|
|
// And adaptive values are always outlined starting from the left.
|
|
// This means that in a table with two adaptive columns where both columns were previously stored out-of line,
|
|
// Decreasing the size of the second column may allow both columns to be stored inline.
|
|
Query: "UPDATE blobt2 SET b2 = LOAD_FILE('testdata/tinyFile') WHERE i = 'HH'",
|
|
},
|
|
{
|
|
Query: "select i, b1, b2 from blobt2 where i = 'HH'",
|
|
Expected: []sql.Row{{"HH", halfSizeString, tinyString}},
|
|
},
|
|
{
|
|
// Similar to the above, dropping a column can change whether the other column is inlined.
|
|
Query: "ALTER TABLE blobt2 DROP COLUMN b2",
|
|
},
|
|
{
|
|
Query: "select i, b1 from blobt2",
|
|
Expected: []sql.Row{
|
|
{"FF", fullSizeString},
|
|
{"HF", halfSizeString},
|
|
{"TF", tinyString},
|
|
{"FH", fullSizeString},
|
|
{"HH", halfSizeString},
|
|
{"TH", tinyString},
|
|
{"FT", fullSizeString},
|
|
{"HT", halfSizeString},
|
|
{"TT", tinyString},
|
|
},
|
|
},
|
|
},
|
|
},
|
|
})
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestAdaptiveEncodingVarbit(t *testing.T) {
|
|
columnType := "varbit"
|
|
RunScripts(t, []ScriptTest{
|
|
{
|
|
Name: "Adaptive Encoding With One Column",
|
|
SetUpScript: setup.SetupScript{
|
|
fmt.Sprintf(`create table blobt (i char(1) primary key, b %s);`, columnType),
|
|
fmt.Sprintf(`create table blobt2 (i char(2) primary key, b1 %s, b2 %s);`, columnType, columnType),
|
|
fmt.Sprintf(`insert into blobt values
|
|
('F', LOAD_FILE('testdata/fullSizeVarbit')::%s),
|
|
('H', LOAD_FILE('testdata/halfSizeVarbit')::%s),
|
|
('T', LOAD_FILE('testdata/tinyFileVarbit')::%s)`, columnType, columnType, columnType),
|
|
},
|
|
Assertions: []ScriptTestAssertion{
|
|
{
|
|
Query: "select b from blobt where i = 'F'",
|
|
Expected: []sql.Row{{fullSizeVarbit}},
|
|
},
|
|
{
|
|
// Files that can fit within a tuple are stored inline.
|
|
Query: "select b from blobt where i = 'H'",
|
|
Expected: []sql.Row{{halfSizeVarbit}},
|
|
},
|
|
{
|
|
// An inlined adaptive column can be used in a filter.
|
|
Query: "select i from blobt where b = LOAD_FILE('testdata/fullSizeVarbit')::varbit",
|
|
Expected: []sql.Row{{"F"}},
|
|
},
|
|
{
|
|
// An out-of-line adaptive column can be used in a filter.
|
|
Query: "select i from blobt where b = LOAD_FILE('testdata/halfSizeVarbit')::varbit",
|
|
Expected: []sql.Row{{"H"}},
|
|
},
|
|
},
|
|
},
|
|
{
|
|
Name: "Adaptive Encoding With Two Columns",
|
|
SetUpScript: setup.SetupScript{
|
|
fmt.Sprintf(`create table blobt2 (i char(2) primary key, b1 %s, b2 %s);`, columnType, columnType),
|
|
fmt.Sprintf(`insert into blobt2 values
|
|
('FF', LOAD_FILE('testdata/fullSizeVarbit')::%s, LOAD_FILE('testdata/fullSizeVarbit')::%s),
|
|
('HF', LOAD_FILE('testdata/halfSizeVarbit')::%s, LOAD_FILE('testdata/fullSizeVarbit')::%s),
|
|
('TF', LOAD_FILE('testdata/tinyFileVarbit')::%s, LOAD_FILE('testdata/fullSizeVarbit')::%s),
|
|
('FH', LOAD_FILE('testdata/fullSizeVarbit')::%s, LOAD_FILE('testdata/halfSizeVarbit')::%s),
|
|
('HH', LOAD_FILE('testdata/halfSizeVarbit')::%s, LOAD_FILE('testdata/halfSizeVarbit')::%s),
|
|
('TH', LOAD_FILE('testdata/tinyFileVarbit')::%s, LOAD_FILE('testdata/halfSizeVarbit')::%s),
|
|
('FT', LOAD_FILE('testdata/fullSizeVarbit')::%s, LOAD_FILE('testdata/tinyFileVarbit')::%s),
|
|
('HT', LOAD_FILE('testdata/halfSizeVarbit')::%s, LOAD_FILE('testdata/tinyFileVarbit')::%s),
|
|
('TT', LOAD_FILE('testdata/tinyFileVarbit')::%s, LOAD_FILE('testdata/tinyFileVarbit')::%s)`, columnType, columnType,
|
|
columnType, columnType, columnType, columnType, columnType, columnType, columnType, columnType,
|
|
columnType, columnType, columnType, columnType, columnType, columnType, columnType, columnType),
|
|
},
|
|
Assertions: []ScriptTestAssertion{
|
|
{
|
|
// When a tuple with multiple adaptive columns is too large, columns are moved out-of-band from left to right.
|
|
// However, strings smaller than the address size (20 bytes) are never stored out-of-band.
|
|
Query: "select i, b1, b2 from blobt2 order by 1",
|
|
Expected: []sql.Row{
|
|
{"FF", fullSizeVarbit, fullSizeVarbit},
|
|
{"FH", fullSizeVarbit, halfSizeVarbit},
|
|
{"FT", fullSizeVarbit, tinyVarbit},
|
|
{"HF", halfSizeVarbit, fullSizeVarbit},
|
|
{"HH", halfSizeVarbit, halfSizeVarbit},
|
|
{"HT", halfSizeVarbit, tinyVarbit},
|
|
{"TF", tinyVarbit, fullSizeVarbit},
|
|
{"TH", tinyVarbit, halfSizeVarbit},
|
|
{"TT", tinyVarbit, tinyVarbit},
|
|
},
|
|
},
|
|
{
|
|
// An adaptive column can be used in a filter when it doesn't have the same encoding in all rows.
|
|
Query: "select i from blobt2 where b1 = LOAD_FILE('testdata/halfSizeVarbit')::varbit",
|
|
Expected: []sql.Row{{"HF"}, {"HH"}, {"HT"}},
|
|
},
|
|
{
|
|
// An adaptive column can be used in a filter when it doesn't have the same encoding in all rows.
|
|
Query: "select i from blobt2 where b2 = LOAD_FILE('testdata/halfSizeVarbit')::varbit",
|
|
Expected: []sql.Row{{"FH"}, {"HH"}, {"TH"}},
|
|
},
|
|
},
|
|
},
|
|
})
|
|
}
|