chore: import upstream snapshot with attribution

testing/generation/utils/generator_stack.go

@@ -0,0 +1,299 @@
+// Copyright 2023-2024 Dolthub, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package utils
+
+import (
+	"github.com/cockroachdb/errors"
+
+	"github.com/dolthub/doltgresql/utils"
+)
+
+// StatementGeneratorStackElement represents an element within the StatementGeneratorStack.
+type StatementGeneratorStackElement struct {
+	depth      int
+	generators []StatementGenerator
+}
+
+// StatementGeneratorStack handles the creation of a StatementGenerator by contextually applying operations based on the
+// current state of the internal stack whenever calls are made.
+type StatementGeneratorStack struct {
+	stack *utils.Stack[*StatementGeneratorStackElement]
+	depth int
+}
+
+// NewStatementGeneratorStack returns a new *StatementGeneratorStack.
+func NewStatementGeneratorStack() *StatementGeneratorStack {
+	sgs := &StatementGeneratorStack{
+		stack: utils.NewStack[*StatementGeneratorStackElement](),
+		depth: 0,
+	}
+	sgs.stack.Push(NewStatementGeneratorStackElement(0))
+	return sgs
+}
+
+// NewStatementGeneratorStackElement returns a new *StatementGeneratorStackElement.
+func NewStatementGeneratorStackElement(depth int, gens ...StatementGenerator) *StatementGeneratorStackElement {
+	return &StatementGeneratorStackElement{
+		depth:      depth,
+		generators: gens,
+	}
+}
+
+// AddText creates a new TextGen at the current depth.
+func (sgs *StatementGeneratorStack) AddText(text string) {
+	sgs.stack.Peek().Append(Text(text))
+}
+
+// AddVariable creates a new VariableGen at the current depth.
+func (sgs *StatementGeneratorStack) AddVariable(name string) {
+	sgs.stack.Peek().Append(Variable(name, nil))
+}
+
+// Or will take all items from the current depth and add them to a parent OrGen. Either the previous depth is an OrGen,
+// or the stack will increment the sub depth to insert an OrGen.
+func (sgs *StatementGeneratorStack) Or() error {
+	if sgs.stack.Empty() {
+		return errors.Errorf("cannot apply Or to an empty stack")
+	}
+	parentElement := sgs.stack.PeekDepth(1)
+	current := sgs.aggregate(sgs.stack.Pop().generators)
+	if parentElement == nil {
+		// We're the root, so we put an OrGen as the root, and make the current a child of the new OrGen
+		sgs.stack.Push(NewStatementGeneratorStackElement(sgs.depth, Or(current)))
+		sgs.stack.Push(NewStatementGeneratorStackElement(sgs.depth))
+	} else if len(parentElement.generators) == 0 {
+		if parentElement.depth == sgs.depth {
+			// We're still at the same depth, so it's safe to append an OrGen
+			parentElement.Append(Or(current))
+		} else {
+			// We should retain the depth, so we need to create a new element since there's a depth boundary
+			sgs.stack.Push(NewStatementGeneratorStackElement(sgs.depth, Or(current)))
+		}
+		sgs.stack.Push(NewStatementGeneratorStackElement(sgs.depth))
+	} else {
+		if parentElement.depth == sgs.depth {
+			// We're still at the same depth, so we need to check if the parent is an OrGen or not
+			if orGen, ok := parentElement.LastGenerator().(*OrGen); ok {
+				// The parent is an OrGen, so we can simply add this as another option
+				if err := orGen.AddChildren(current); err != nil {
+					return err
+				}
+			} else {
+				// The parent is not an OrGen, so we need to add to the depth
+				sgs.stack.Push(NewStatementGeneratorStackElement(sgs.depth, Or(current)))
+			}
+		} else {
+			// We should retain the depth, so we need to create a new element since there's a depth boundary
+			sgs.stack.Push(NewStatementGeneratorStackElement(sgs.depth, Or(current)))
+		}
+		sgs.stack.Push(NewStatementGeneratorStackElement(sgs.depth))
+	}
+	return nil
+}
+
+// NewScope increases the depth.
+func (sgs *StatementGeneratorStack) NewScope() {
+	sgs.depth++
+	sgs.stack.Push(NewStatementGeneratorStackElement(sgs.depth))
+}
+
+// NewOptionalScope increases the depth, creating an OptionalGen at the depth root, and adding elements to the sub depth.
+func (sgs *StatementGeneratorStack) NewOptionalScope() {
+	sgs.depth++
+	sgs.stack.Push(NewStatementGeneratorStackElement(sgs.depth, Optional()))
+	sgs.stack.Push(NewStatementGeneratorStackElement(sgs.depth))
+}
+
+// NewParenScope increases the depth, creating a CollectionGen at the depth root, adding Text("(") to the CollectionGen,
+// and adding any new elements to the sub depth.
+func (sgs *StatementGeneratorStack) NewParenScope() {
+	sgs.depth++
+	sgs.stack.Push(NewStatementGeneratorStackElement(sgs.depth, Collection(Text("("))))
+	sgs.stack.Push(NewStatementGeneratorStackElement(sgs.depth))
+}
+
+// ExitScope decrements the depth, writing all elements from the current depth (and all sub depths) to the preceding
+// depth.
+func (sgs *StatementGeneratorStack) ExitScope() error {
+	defer func() {
+		sgs.depth--
+	}()
+	current := sgs.aggregate(sgs.stack.Pop().generators)
+	for parentElement := sgs.stack.Peek(); parentElement.depth == sgs.depth; parentElement = sgs.stack.Peek() {
+		if orGen, ok := parentElement.LastGenerator().(*OrGen); ok {
+			if err := orGen.AddChildren(current); err != nil {
+				return err
+			}
+		} else {
+			parentElement.Append(current)
+		}
+		current = sgs.aggregate(sgs.stack.Pop().generators)
+	}
+	sgs.stack.Peek().Append(current)
+	return nil
+}
+
+// ExitOptionalScope decrements the depth, writing all elements from the current depth (and all sub depths) to the
+// preceding depth. This will fail if the root of the current depth is not an OptionalGen.
+func (sgs *StatementGeneratorStack) ExitOptionalScope() error {
+	defer func() {
+		sgs.depth--
+	}()
+	current := sgs.aggregate(sgs.stack.Pop().generators)
+	for parentElement := sgs.stack.Peek(); sgs.stack.PeekDepth(1).depth == sgs.depth; parentElement = sgs.stack.Peek() {
+		if orGen, ok := parentElement.LastGenerator().(*OrGen); ok {
+			if err := orGen.AddChildren(current); err != nil {
+				return err
+			}
+		} else {
+			parentElement.Append(current)
+		}
+		current = sgs.aggregate(sgs.stack.Pop().generators)
+	}
+	optionalElement := sgs.stack.Pop()
+	if optionalElement.depth != sgs.depth {
+		return errors.Errorf("internal bookkeeping error, attempted to exit from optional scope but the depth is incorrect")
+	}
+	if optionalGen, ok := optionalElement.LastGenerator().(*OptionalGen); ok {
+		if err := optionalGen.AddChildren(current); err != nil {
+			return err
+		}
+	} else {
+		return errors.Errorf("internal bookkeeping error, attempted to exit from optional scope but missing OptionalGen")
+	}
+	sgs.stack.Peek().Append(sgs.aggregate(optionalElement.generators))
+	return nil
+}
+
+// ExitParenScope decrements the depth, writing all elements from the current depth (and all sub depths) to the
+// preceding depth, while appending an ending Text(")").
+func (sgs *StatementGeneratorStack) ExitParenScope() error {
+	defer func() {
+		sgs.depth--
+	}()
+	current := sgs.aggregate(sgs.stack.Pop().generators)
+	for parentElement := sgs.stack.Peek(); sgs.stack.PeekDepth(1).depth == sgs.depth; parentElement = sgs.stack.Peek() {
+		if orGen, ok := parentElement.LastGenerator().(*OrGen); ok {
+			if err := orGen.AddChildren(current); err != nil {
+				return err
+			}
+		} else {
+			parentElement.Append(current)
+		}
+		current = sgs.aggregate(sgs.stack.Pop().generators)
+	}
+	collectionElement := sgs.stack.Pop()
+	if collectionElement.depth != sgs.depth {
+		return errors.Errorf("internal bookkeeping error, attempted to exit from paren scope but the depth is incorrect")
+	}
+	if collectionGen, ok := collectionElement.LastGenerator().(*CollectionGen); ok {
+		if err := collectionGen.AddChildren(current, Text(")")); err != nil {
+			return err
+		}
+	} else {
+		return errors.Errorf("internal bookkeeping error, attempted to exit from paren scope but missing CollectionGen")
+	}
+	sgs.stack.Peek().Append(sgs.aggregate(collectionElement.generators))
+	return nil
+}
+
+// Repeat will add the last StatementGenerator at the current depth to an OptionalGen.
+func (sgs *StatementGeneratorStack) Repeat() error {
+	current := sgs.stack.Peek()
+	lastGen := current.LastGenerator()
+	if lastGen == nil {
+		return errors.Errorf("unable to repeat as no generators exist at the current depth")
+	}
+	current.Append(Optional(lastGen.Copy()))
+	return nil
+}
+
+// OptionalRepeat will add the last StatementGenerator at the current depth to an OptionalGen. If a prefix is given,
+// then it will be added as a TextGen before the repeated generator.
+func (sgs *StatementGeneratorStack) OptionalRepeat(prefix string) error {
+	current := sgs.stack.Peek()
+	lastGen := current.LastGenerator()
+	if lastGen == nil {
+		return errors.Errorf("unable to optionally repeat as no generators exist at the current depth")
+	}
+	if len(prefix) > 0 {
+		current.Append(Optional(Collection(Text(prefix), lastGen.Copy())))
+	} else {
+		current.Append(Optional(lastGen.Copy()))
+	}
+	return nil
+}
+
+// Finish returns a StatementGenerator containing all of the generators that have been added. This returns an error if
+// the stack is not at the root depth, as that indicates a missing scope exit, or too many scope entries.
+func (sgs *StatementGeneratorStack) Finish() (StatementGenerator, error) {
+	if sgs.depth != 0 {
+		if sgs.depth < 0 {
+			return nil, errors.Errorf("depth is invalid, too many scope exits")
+		} else {
+			return nil, errors.Errorf("depth is invalid, too many scope entries")
+		}
+	} else if !sgs.stack.Empty() && sgs.stack.Peek().depth != 0 {
+		return nil, errors.Errorf("internal bookkeeping error, stack depth does not match handle depth")
+	}
+	if sgs.stack.Len() == 1 && len(sgs.stack.Peek().generators) == 0 {
+		return nil, nil
+	}
+	var lastDepth []StatementGenerator
+	for !sgs.stack.Empty() {
+		currentDepth := sgs.stack.Pop()
+		if len(currentDepth.generators) == 0 {
+			return nil, errors.Errorf("internal bookkeeping error, stack has a depth with no generators")
+		}
+		if lastGen := currentDepth.LastGenerator(); lastGen != nil {
+			if orGen, ok := lastGen.(*OrGen); ok {
+				if err := orGen.AddChildren(sgs.aggregate(lastDepth)); err != nil {
+					return nil, err
+				}
+			} else {
+				currentDepth.Append(sgs.aggregate(lastDepth))
+			}
+		} else {
+			currentDepth.Append(sgs.aggregate(lastDepth))
+		}
+		lastDepth = currentDepth.generators
+	}
+	return sgs.aggregate(lastDepth), nil
+}
+
+// aggregate returns an aggregate of the given generators. Returns nil if the slice is empty.
+func (sgs *StatementGeneratorStack) aggregate(gens []StatementGenerator) StatementGenerator {
+	gens = removeNilGenerators(gens)
+	if len(gens) == 0 {
+		return nil
+	} else if len(gens) == 1 {
+		return gens[0]
+	} else {
+		return Collection(gens...)
+	}
+}
+
+// LastGenerator returns the last StatementGenerator contained within its slice.
+func (sgse *StatementGeneratorStackElement) LastGenerator() StatementGenerator {
+	if sgse == nil || len(sgse.generators) == 0 {
+		return nil
+	}
+	return sgse.generators[len(sgse.generators)-1]
+}
+
+// Append adds the given child to this element's collection.
+func (sgse *StatementGeneratorStackElement) Append(child StatementGenerator) {
+	sgse.generators = append(sgse.generators, child)
+}

testing/generation/utils/ints.go

@@ -0,0 +1,82 @@
+// Copyright 2023-2024 Dolthub, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package utils
+
+import (
+	"crypto/rand"
+	"math"
+	"math/big"
+	"sort"
+)
+
+var (
+	BigIntZero      = big.NewInt(0)
+	BigIntOne       = big.NewInt(1)
+	BigIntTwo       = big.NewInt(2)
+	BigIntMaxInt64  = big.NewInt(math.MaxInt64)
+	BigIntMaxUint64 = new(big.Int).Add(new(big.Int).Mul(BigIntMaxInt64, BigIntTwo), BigIntOne)
+)
+
+// GenerateRandomInts generates a slice of random integers, with each integer ranging from [0, max). The returned slice
+// will be sorted from smallest to largest. If count <= 0 or max <= 0, then they will be set to 1. If count >= max, then
+// the returned slice will contain all incrementing integers [0, max).
+func GenerateRandomInts(count int64, max *big.Int) (randInts []*big.Int, err error) {
+	if count <= 0 {
+		count = 1
+	}
+	if max.Cmp(BigIntZero) == -1 {
+		max = BigIntOne
+	}
+	// If count >= max, then we'll just shortcut and add incrementing integers up to the max (not including the max)
+	if big.NewInt(count).Cmp(max) >= 0 {
+		max64 := max.Int64()
+		randInts = make([]*big.Int, max64)
+		for i := int64(0); i < max64; i++ {
+			randInts[i] = big.NewInt(i)
+		}
+		return randInts, nil
+	}
+
+	randInts = make([]*big.Int, count)
+	randIntSet := make(map[string]struct{}, count*2)
+	for i := range randInts {
+		for {
+			randInts[i], err = rand.Int(rand.Reader, max)
+			if err != nil {
+				return nil, err
+			}
+			if _, ok := randIntSet[randInts[i].String()]; !ok {
+				randIntSet[randInts[i].String()] = struct{}{}
+				break
+			}
+		}
+	}
+	sort.Slice(randInts, func(i, j int) bool {
+		return randInts[i].Cmp(randInts[j]) == -1
+	})
+	return randInts, nil
+}
+
+// GetPercentages converts the slice of numbers to percentages. The max defines the number that would equal 100%. All
+// floats will be between [0.0, 100.0], unless the number is not between [0, max].
+func GetPercentages(numbers []*big.Int, max *big.Int) []float64 {
+	maxAsFloat, _ := max.Float64()
+	percentages := make([]float64, len(numbers))
+	for i, number := range numbers {
+		numberAsFloat, _ := number.Float64()
+		percentages[i] = (numberAsFloat / maxAsFloat) * 100.0
+	}
+	return percentages
+}

testing/generation/utils/parse_tokens.go

@@ -0,0 +1,124 @@
+// Copyright 2023-2024 Dolthub, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package utils
+
+import "github.com/cockroachdb/errors"
+
+// ParseTokens parses the given tokens into a StatementGenerator.
+func ParseTokens(tokens []Token, includeRepetition bool) (StatementGenerator, error) {
+	stack := NewStatementGeneratorStack()
+	var statements []StatementGenerator
+	variables := make(map[string]StatementGenerator)
+	currentVariable := ""
+	tokenReader := NewTokenReader(tokens)
+ForLoop:
+	for {
+		token, ok := tokenReader.Next()
+		if !ok {
+			break ForLoop
+		}
+		switch token.Type {
+		case TokenType_Text:
+			stack.AddText(token.Literal)
+		case TokenType_Variable:
+			stack.AddVariable(token.Literal)
+		case TokenType_VariableDefinition:
+			currentVariable = token.Literal
+			if token, _ = tokenReader.Next(); token.Type != TokenType_LongSpace {
+				return nil, errors.Errorf("expected a long space after a variable definition declaration")
+			}
+		case TokenType_Or:
+			if err := stack.Or(); err != nil {
+				return nil, err
+			}
+		case TokenType_Repeat:
+			if includeRepetition {
+				if err := stack.Repeat(); err != nil {
+					return nil, err
+				}
+			}
+		case TokenType_OptionalRepeat:
+			if includeRepetition {
+				if err := stack.OptionalRepeat(token.Literal); err != nil {
+					return nil, err
+				}
+			}
+		case TokenType_ShortSpace, TokenType_MediumSpace:
+			return nil, errors.Errorf("token reader should have removed all short and medium spaces")
+		case TokenType_LongSpace, TokenType_EOF:
+			newStatement, err := stack.Finish()
+			if err != nil {
+				return nil, err
+			}
+			if newStatement == nil {
+				return nil, errors.Errorf("long space encountered before writing to the stack")
+			}
+			if len(currentVariable) > 0 {
+				if _, ok = variables[currentVariable]; ok {
+					return nil, errors.Errorf("multiple definitions for the same variable: %s", currentVariable)
+				}
+				variables[currentVariable] = newStatement
+				currentVariable = ""
+			} else {
+				statements = append(statements, newStatement)
+			}
+			if token.Type == TokenType_EOF {
+				break ForLoop
+			} else {
+				stack = NewStatementGeneratorStack()
+			}
+		case TokenType_ParenOpen:
+			stack.NewParenScope()
+		case TokenType_ParenClose:
+			if err := stack.ExitParenScope(); err != nil {
+				return nil, err
+			}
+		case TokenType_OptionalOpen:
+			stack.NewOptionalScope()
+		case TokenType_OptionalClose:
+			if err := stack.ExitOptionalScope(); err != nil {
+				return nil, err
+			}
+		case TokenType_OneOfOpen:
+			stack.NewScope()
+		case TokenType_OneOfClose:
+			if err := stack.ExitScope(); err != nil {
+				return nil, err
+			}
+		default:
+			panic("unhandled token type")
+		}
+	}
+	finalStackContents, err := stack.Finish()
+	if err != nil {
+		return nil, err
+	}
+	if finalStackContents != nil {
+		return nil, errors.Errorf("encountered an early EOF, as the stack was still processing")
+	}
+	if len(statements) == 0 {
+		return nil, errors.Errorf("no statements were generated from the token stream")
+	}
+	var finalStatementGenerator StatementGenerator
+	if len(statements) == 1 {
+		finalStatementGenerator = statements[0]
+	} else {
+		finalStatementGenerator = Or(statements...)
+	}
+	if err = ApplyVariableDefinition(finalStatementGenerator, variables); err != nil {
+		return nil, err
+	}
+	return finalStatementGenerator, nil
+}

testing/generation/utils/root_folder.go

@@ -0,0 +1,80 @@
+// Copyright 2024 Dolthub, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package utils
+
+import (
+	"os"
+	"path/filepath"
+	"runtime"
+
+	"github.com/cockroachdb/errors"
+)
+
+// RootFolderLocation is the location of this project's root folder.
+type RootFolderLocation struct {
+	path string
+}
+
+// GetRootFolder returns the location of this project's root folder. This is useful to locate relative directories,
+// which will often be read from and written to for generation. It is assumed that this will always be called from
+// within an IDE.
+func GetRootFolder() (RootFolderLocation, error) {
+	_, currentFileLocation, _, ok := runtime.Caller(0)
+	if !ok {
+		return RootFolderLocation{}, errors.Errorf("failed to fetch the location of the current file")
+	}
+	return RootFolderLocation{filepath.Clean(filepath.Join(filepath.Dir(currentFileLocation), "../../.."))}, nil
+}
+
+// MoveRoot returns a new RootFolderLocation that defines the root at the new path. The parameter should be relative to
+// the current root.
+func (root RootFolderLocation) MoveRoot(relativePath string) RootFolderLocation {
+	return RootFolderLocation{filepath.Clean(filepath.Join(root.path, relativePath))}
+}
+
+// GetAbsolutePath returns the absolute path of the given path, which should be relative to the project's root
+// folder.
+func (root RootFolderLocation) GetAbsolutePath(relativePath string) string {
+	return filepath.ToSlash(filepath.Join(root.path, relativePath))
+}
+
+// ReadDir is equivalent to os.ReadDir, except that it uses the root path and the given relative path.
+func (root RootFolderLocation) ReadDir(relativePath string) ([]os.DirEntry, error) {
+	return os.ReadDir(filepath.ToSlash(filepath.Join(root.path, relativePath)))
+}
+
+// ReadFile is equivalent to os.ReadFile, except that it uses the root path and the given relative path.
+func (root RootFolderLocation) ReadFile(relativePath string) ([]byte, error) {
+	return os.ReadFile(filepath.ToSlash(filepath.Join(root.path, relativePath)))
+}
+
+// ReadFileFromDirectory is similar to ReadFile, except that it takes in a relative path for the directory containing
+// the file, along with the file name to be read. This is purely for convenience, as the same behavior can be
+// accomplished using ReadFile.
+func (root RootFolderLocation) ReadFileFromDirectory(relativeDirectoryPath string, fileName string) ([]byte, error) {
+	return os.ReadFile(filepath.ToSlash(filepath.Join(root.path, relativeDirectoryPath, fileName)))
+}
+
+// WriteFile is equivalent to os.WriteFile, except that it uses the root path and the given relative path.
+func (root RootFolderLocation) WriteFile(relativePath string, data []byte, perm os.FileMode) error {
+	return os.WriteFile(filepath.ToSlash(filepath.Join(root.path, relativePath)), data, perm)
+}
+
+// WriteFileToDirectory is similar to WriteFile, except that it takes in a relative path for the directory containing
+// the file, along with the file name to be written. This is purely for convenience, as the same behavior can be
+// accomplished using WriteFile.
+func (root RootFolderLocation) WriteFileToDirectory(relativeDirectoryPath string, fileName string, data []byte, perm os.FileMode) error {
+	return os.WriteFile(filepath.ToSlash(filepath.Join(root.path, relativeDirectoryPath, fileName)), data, perm)
+}

testing/generation/utils/section_marker.go

@@ -0,0 +1,42 @@
+// Copyright 2023-2024 Dolthub, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package utils
+
+import (
+	"fmt"
+	"strings"
+)
+
+// SectionMarker returns a marker that may be used to denote sections.
+//
+// For example, SectionMarker("abc", '-', 21) would return:
+//
+// -------- abc --------
+func SectionMarker(centeredText string, fillerCharacter rune, totalLength int) string {
+	fillerStr := string(fillerCharacter)
+	remainingLength := totalLength - (len(centeredText) + 2)
+	if remainingLength <= 0 {
+		return fmt.Sprintf(" %s ", centeredText)
+	}
+	left := remainingLength / 2
+	right := remainingLength - left // Integer division doesn't do fractions, so this will handle odd counts
+	return fmt.Sprintf("%s %s %s",
+		strings.Repeat(fillerStr, left), centeredText, strings.Repeat(fillerStr, right))
+}
+
+// PrintSectionMarker is a convenience function that prints the result of SectionMarker with a newline appended.
+func PrintSectionMarker(centeredText string, fillerCharacter rune, totalLength int) {
+	fmt.Println(SectionMarker(centeredText, fillerCharacter, totalLength))
+}

testing/generation/utils/statement_generator.go

@@ -0,0 +1,766 @@
+// Copyright 2023-2024 Dolthub, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package utils
+
+import (
+	"fmt"
+	"math"
+	"math/big"
+	"sort"
+	"strings"
+)
+
+// StatementGenerator represents a statement, and is able to produce all valid variations of the statement.
+type StatementGenerator interface {
+	// AddChildren adds the given children to the generator. Not all generators accept all children, so this may error.
+	AddChildren(child ...StatementGenerator) error
+	// Consume returns true when the generator is able to produce a unique mutation, and false if it is not. Only one
+	// generator should mutate per call, meaning a parent generator should only mutate when its children return false.
+	// If the top-level generator returns false, then all permutations have been created.
+	Consume() bool
+	// SetConsumeIterations is equivalent to calling Copy then Consume the given number of times, without allocating a
+	// new StatementGenerator. This allows you to generate a specific statement efficiently, rather than calling Consume
+	// the given number of times. If the count is <= 0, then the statement will be in its original state (the same state
+	// as a StatementGenerator copy).
+	SetConsumeIterations(count *big.Int)
+	// SetConsumeIterationsFast is the same as SetConsumeIterations, except far more efficient due to using uint64,
+	// however it only works for iteration counts <= MAX_SIZE(uint64).
+	SetConsumeIterationsFast(count uint64)
+	// String returns a string based on the current permutation.
+	String() string
+	// Copy returns a copy of the given generator (along with all of its children) in its original setting. This means
+	// that the copy is in the same state that the target would be in if it had never called Consume.
+	Copy() StatementGenerator
+	// Reset sets the StatementGenerator back to its original state, which would be as though Consume was never called.
+	// This is equivalent to calling SetConsumeIterations(0), albeit slightly more efficient.
+	Reset()
+	// SourceString returns a string that may be used to recreate the StatementGenerator in a Go source file.
+	SourceString() string
+	// Permutations returns the number of unique permutations that the generator can return.
+	Permutations() *big.Int
+	// PermutationsUint64 returns the number of unique permutations that the generator can return. Returns true if the
+	// number fits within an uint64, false if it's larger than an uint64.
+	PermutationsUint64() (uint64, bool)
+}
+
+// TextGen is a generator that returns a simple string.
+type TextGen string
+
+var _ StatementGenerator = (*TextGen)(nil)
+
+// Text creates a new StatementGenerator representing a simple string.
+func Text(str string) *TextGen {
+	gen := TextGen(str)
+	return &gen
+}
+
+// AddChildren implements the interface StatementGenerator.
+func (t *TextGen) AddChildren(children ...StatementGenerator) error {
+	return fmt.Errorf("text cannot have children")
+}
+
+// Consume implements the interface StatementGenerator.
+func (t *TextGen) Consume() bool {
+	return false
+}
+
+// SetConsumeIterations implements the interface StatementGenerator.
+func (t *TextGen) SetConsumeIterations(count *big.Int) {}
+
+// SetConsumeIterationsFast implements the interface StatementGenerator.
+func (t *TextGen) SetConsumeIterationsFast(count uint64) {}
+
+// Copy implements the interface StatementGenerator.
+func (t *TextGen) Copy() StatementGenerator {
+	if t == nil {
+		return nil
+	}
+	return Text(string(*t))
+}
+
+// String implements the interface StatementGenerator.
+func (t *TextGen) String() string {
+	return string(*t)
+}
+
+// Reset implements the interface StatementGenerator.
+func (t *TextGen) Reset() {}
+
+// SourceString implements the interface StatementGenerator.
+func (t *TextGen) SourceString() string {
+	return fmt.Sprintf(`Text("%s")`, string(*t))
+}
+
+// Permutations implements the interface StatementGenerator.
+func (t *TextGen) Permutations() *big.Int {
+	return BigIntOne
+}
+
+// PermutationsUint64 implements the interface StatementGenerator.
+func (t *TextGen) PermutationsUint64() (uint64, bool) {
+	return 1, true
+}
+
+// OrGen is a generator that contains multiple child generators, and will print only one at a time. Consuming will
+// cycle to the next child.
+type OrGen struct {
+	children []StatementGenerator
+	index    int
+	localInt *big.Int
+}
+
+var _ StatementGenerator = (*OrGen)(nil)
+
+// Or creates a new StatementGenerator representing an OrGen.
+func Or(children ...StatementGenerator) *OrGen {
+	return &OrGen{
+		children: copyGenerators(children),
+		index:    0,
+		localInt: new(big.Int),
+	}
+}
+
+// AddChildren implements the interface StatementGenerator.
+func (o *OrGen) AddChildren(children ...StatementGenerator) error {
+	o.children = append(o.children, copyGenerators(children)...)
+	return nil
+}
+
+// Consume implements the interface StatementGenerator.
+func (o *OrGen) Consume() bool {
+	if len(o.children) == 0 {
+		return false
+	}
+	if o.children[o.index].Consume() {
+		return true
+	}
+	o.index++
+	if o.index >= len(o.children) {
+		o.index = 0
+		return false
+	}
+	return true
+}
+
+// SetConsumeIterations implements the interface StatementGenerator.
+func (o *OrGen) SetConsumeIterations(count *big.Int) {
+	// If we're given zero, then we'll just call Reset
+	if count.Cmp(BigIntZero) <= 0 {
+		o.Reset()
+		return
+	}
+	count = o.localInt.Mod(count, o.Permutations())
+	for i, child := range o.children {
+		// The index is equal to whichever child we stop on
+		o.index = i
+		childPermutations := child.Permutations()
+		if childPermutations.Cmp(count) > 0 {
+			// The child has more permutations than the count, so we'll stop here
+			if count.Cmp(BigIntMaxUint64) <= 0 {
+				child.SetConsumeIterationsFast(count.Uint64())
+			} else {
+				child.SetConsumeIterations(count)
+			}
+			break
+		} else {
+			// The child's permutations are <= the count, so we'll reset it and subtract it from the total.
+			// Subtraction here is the opposite of the addition we do to determine the permutation count.
+			// Important to note that the permutations equaling the count means that the index increments to the next
+			// item, but since the count will be zero, it matches the original state of that item.
+			child.Reset()
+			count.Sub(count, childPermutations)
+		}
+	}
+	// We still need to reset any children that we never looped over
+	for i := o.index + 1; i < len(o.children); i++ {
+		o.children[i].Reset()
+	}
+}
+
+// SetConsumeIterationsFast implements the interface StatementGenerator.
+func (o *OrGen) SetConsumeIterationsFast(count uint64) {
+	// This is a copy of SetConsumeIterations, except rewritten to use uint64
+	if count <= 0 {
+		o.Reset()
+		return
+	}
+	permutations, _ := o.PermutationsUint64()
+	count = count % permutations
+	for i, child := range o.children {
+		o.index = i
+		childPermutations, _ := child.PermutationsUint64()
+		if childPermutations > count {
+			child.SetConsumeIterationsFast(count)
+			break
+		} else {
+			child.Reset()
+			count -= childPermutations
+		}
+	}
+	for i := o.index + 1; i < len(o.children); i++ {
+		o.children[i].Reset()
+	}
+}
+
+// Copy implements the interface StatementGenerator.
+func (o *OrGen) Copy() StatementGenerator {
+	if o == nil {
+		return nil
+	}
+	return Or(o.children...)
+}
+
+// String implements the interface StatementGenerator.
+func (o *OrGen) String() string {
+	return o.children[o.index].String()
+}
+
+// Reset implements the interface StatementGenerator.
+func (o *OrGen) Reset() {
+	o.index = 0
+	for _, child := range o.children {
+		child.Reset()
+	}
+}
+
+// SourceString implements the interface StatementGenerator.
+func (o *OrGen) SourceString() string {
+	return fmt.Sprintf(`Or(%s)`, sourceGenerators(o.children))
+}
+
+// Permutations implements the interface StatementGenerator.
+func (o *OrGen) Permutations() *big.Int {
+	sum := big.NewInt(0)
+	for _, child := range o.children {
+		sum.Add(sum, child.Permutations())
+	}
+	return sum
+}
+
+// PermutationsUint64 implements the interface StatementGenerator.
+func (o *OrGen) PermutationsUint64() (uint64, bool) {
+	sum := uint64(0)
+	for _, child := range o.children {
+		childCount, ok := child.PermutationsUint64()
+		if !ok || sum > (math.MaxUint64-childCount) {
+			return math.MaxUint64, false
+		}
+		sum += childCount
+	}
+	return sum, true
+}
+
+// VariableGen represents a variable in the synopsis. Its values are user-configurable if they cannot be deduced from
+// the synopsis.
+type VariableGen struct {
+	name    string
+	options StatementGenerator
+}
+
+var _ StatementGenerator = (*VariableGen)(nil)
+
+// Variable creates a new StatementGenerator representing a VariableGen.
+func Variable(name string, child StatementGenerator) *VariableGen {
+	if child != nil {
+		return &VariableGen{
+			name:    name,
+			options: child.Copy(),
+		}
+	} else {
+		return &VariableGen{
+			name:    name,
+			options: nil,
+		}
+	}
+}
+
+// AddChildren implements the interface StatementGenerator.
+func (v *VariableGen) AddChildren(children ...StatementGenerator) error {
+	children = removeNilGenerators(children)
+	if len(children) == 0 {
+		return nil
+	}
+	if len(children) > 1 {
+		return fmt.Errorf("attempting to give variable `%s` too many children", v.name)
+	}
+	if v.options != nil {
+		return fmt.Errorf("variable `%s` has already been assigned", v.name)
+	}
+	v.options = children[0].Copy()
+	return nil
+}
+
+// Consume implements the interface StatementGenerator.
+func (v *VariableGen) Consume() bool {
+	if v.options != nil {
+		return v.options.Consume()
+	}
+	return false
+}
+
+// SetConsumeIterations implements the interface StatementGenerator.
+func (v *VariableGen) SetConsumeIterations(count *big.Int) {
+	if v.options != nil {
+		v.options.SetConsumeIterations(count)
+	}
+}
+
+// SetConsumeIterationsFast implements the interface StatementGenerator.
+func (v *VariableGen) SetConsumeIterationsFast(count uint64) {
+	if v.options != nil {
+		v.options.SetConsumeIterationsFast(count)
+	}
+}
+
+// Copy implements the interface StatementGenerator.
+func (v *VariableGen) Copy() StatementGenerator {
+	if v == nil {
+		return nil
+	}
+	return Variable(v.name, v.options)
+}
+
+// String implements the interface StatementGenerator.
+func (v *VariableGen) String() string {
+	if v.options != nil {
+		return v.options.String()
+	} else {
+		return v.name
+	}
+}
+
+// Reset implements the interface StatementGenerator.
+func (v *VariableGen) Reset() {
+	if v.options != nil {
+		v.options.Reset()
+	}
+}
+
+// SourceString implements the interface StatementGenerator.
+func (v *VariableGen) SourceString() string {
+	if v.options != nil {
+		return fmt.Sprintf(`Variable("%s", %s)`, v.name, v.options.SourceString())
+	} else {
+		return fmt.Sprintf(`Variable("%s", nil)`, v.name)
+	}
+}
+
+// Permutations implements the interface StatementGenerator.
+func (v *VariableGen) Permutations() *big.Int {
+	if v.options != nil {
+		return v.options.Permutations()
+	} else {
+		return BigIntOne
+	}
+}
+
+// PermutationsUint64 implements the interface StatementGenerator.
+func (v *VariableGen) PermutationsUint64() (uint64, bool) {
+	if v.options != nil {
+		return v.options.PermutationsUint64()
+	} else {
+		return 1, true
+	}
+}
+
+// CollectionGen is a generator that contains multiple child generators, and will print all of its children.
+type CollectionGen struct {
+	children []StatementGenerator
+	localInt *big.Int
+}
+
+var _ StatementGenerator = (*CollectionGen)(nil)
+
+// Collection creates a new StatementGenerator representing a CollectionGen.
+func Collection(children ...StatementGenerator) *CollectionGen {
+	return &CollectionGen{
+		children: copyGenerators(children),
+		localInt: new(big.Int),
+	}
+}
+
+// AddChildren implements the interface StatementGenerator.
+func (c *CollectionGen) AddChildren(children ...StatementGenerator) error {
+	c.children = append(c.children, copyGenerators(children)...)
+	return nil
+}
+
+// Consume implements the interface StatementGenerator.
+func (c *CollectionGen) Consume() bool {
+	for i := range c.children {
+		if c.children[i].Consume() {
+			return true
+		}
+	}
+	return false
+}
+
+// SetConsumeIterations implements the interface StatementGenerator.
+func (c *CollectionGen) SetConsumeIterations(count *big.Int) {
+	// We handle this one as though it's a non-uniform numbering system (binary and decimal are uniform systems).
+	// In a traditional number system like binary, you can find each bit's value using the following:
+	//
+	// bit = number % 2; number = number / 2;
+	//
+	// Collections behave similarly to that system, where we increment the second generator after fully incrementing the
+	// first generator. Then we have to iterate over the first generator again before we can increment the second
+	// generator again. Do this until the second generator has exhausted its permutations, and then the third generator
+	// can increment.
+	//
+	// Going back to our binary example, we can achieve that same counting effect by replacing 2 with the permutation
+	// count. This lets us have our non-uniform numbering system, and allows us to efficiently find the exact number for
+	// each generator.
+	count = c.localInt.Mod(count, c.Permutations())
+	index := 0
+	for i, child := range c.children {
+		// The index is equal to whichever child we stop on
+		index = i
+		childPermutations := child.Permutations()
+		// We give the child the modulo of the count versus its permutation count, which will determine how many
+		// iterations it's supposed to simulate from the total.
+		childIterations := new(big.Int).Mod(count, childPermutations)
+		if childIterations.Cmp(BigIntMaxUint64) <= 0 {
+			child.SetConsumeIterationsFast(childIterations.Uint64())
+		} else {
+			child.SetConsumeIterations(childIterations)
+		}
+		// We divide the count by this child's permutation count to move to the next "base".
+		count.Div(count, childPermutations)
+		// If we're at zero now, then this child used up the remaining count, so we'll stop here
+		if count.Cmp(BigIntZero) <= 0 {
+			break
+		}
+	}
+	// We still need to reset any children that we never looped over
+	for index += 1; index < len(c.children); index++ {
+		c.children[index].Reset()
+	}
+}
+
+// SetConsumeIterationsFast implements the interface StatementGenerator.
+func (c *CollectionGen) SetConsumeIterationsFast(count uint64) {
+	// This is a copy of SetConsumeIterations, except rewritten to use uint64
+	permutations, _ := c.PermutationsUint64()
+	count = count % permutations
+	index := 0
+	for i, child := range c.children {
+		index = i
+		childPermutations, _ := child.PermutationsUint64()
+		child.SetConsumeIterationsFast(count % childPermutations)
+		count /= childPermutations
+		if count <= 0 {
+			break
+		}
+	}
+	for index += 1; index < len(c.children); index++ {
+		c.children[index].Reset()
+	}
+}
+
+// Copy implements the interface StatementGenerator.
+func (c *CollectionGen) Copy() StatementGenerator {
+	if c == nil {
+		return nil
+	}
+	return Collection(c.children...)
+}
+
+// String implements the interface StatementGenerator.
+func (c *CollectionGen) String() string {
+	var childrenStrings []string
+	for i := range c.children {
+		childString := c.children[i].String()
+		if len(childString) > 0 {
+			childrenStrings = append(childrenStrings, childString)
+		}
+	}
+	return strings.Join(childrenStrings, " ")
+}
+
+// Reset implements the interface StatementGenerator.
+func (c *CollectionGen) Reset() {
+	for _, child := range c.children {
+		child.Reset()
+	}
+}
+
+// SourceString implements the interface StatementGenerator.
+func (c *CollectionGen) SourceString() string {
+	return fmt.Sprintf(`Collection(%s)`, sourceGenerators(c.children))
+}
+
+// Permutations implements the interface StatementGenerator.
+func (c *CollectionGen) Permutations() *big.Int {
+	total := big.NewInt(1)
+	for _, child := range c.children {
+		childPermutations := child.Permutations()
+		if childPermutations.Cmp(BigIntZero) != 0 {
+			total.Mul(total, childPermutations)
+		}
+	}
+	return total
+}
+
+// PermutationsUint64 implements the interface StatementGenerator.
+func (c *CollectionGen) PermutationsUint64() (uint64, bool) {
+	total := uint64(1)
+	for _, child := range c.children {
+		childPermutations, ok := child.PermutationsUint64()
+		if !ok {
+			return math.MaxUint64, false
+		}
+		if childPermutations == 0 {
+			continue
+		}
+		if total > math.MaxUint64/childPermutations {
+			return math.MaxUint64, false
+		}
+		total *= childPermutations
+	}
+	return total, true
+}
+
+// OptionalGen is a generator that will toggle between displaying its children and not displaying its children.
+type OptionalGen struct {
+	children *CollectionGen
+	display  bool
+	localInt *big.Int
+}
+
+var _ StatementGenerator = (*OptionalGen)(nil)
+
+// Optional creates a new StatementGenerator representing an OptionalGen.
+func Optional(children ...StatementGenerator) *OptionalGen {
+	return &OptionalGen{
+		children: Collection(children...),
+		display:  false,
+		localInt: new(big.Int),
+	}
+}
+
+// AddChildren implements the interface StatementGenerator.
+func (o *OptionalGen) AddChildren(children ...StatementGenerator) error {
+	return o.children.AddChildren(children...)
+}
+
+// Consume implements the interface StatementGenerator.
+func (o *OptionalGen) Consume() bool {
+	if !o.display {
+		o.display = true
+		return true
+	} else if o.children.Consume() {
+		return true
+	} else {
+		o.display = false
+		return false
+	}
+}
+
+// SetConsumeIterations implements the interface StatementGenerator.
+func (o *OptionalGen) SetConsumeIterations(count *big.Int) {
+	// If we're given zero, then we'll just call Reset
+	if count.Cmp(BigIntZero) <= 0 {
+		o.Reset()
+		return
+	}
+	// The count is >= 1, so display will be true
+	o.display = true
+	count = o.localInt.Mod(count, o.Permutations())
+	// Setting display to true uses a single Consume, so we subtract it before passing the count to the child
+	count.Sub(count, BigIntOne)
+	// We'll pass the rest of the remaining count to the child, which will be >= 0
+	o.children.SetConsumeIterations(count)
+}
+
+// SetConsumeIterationsFast implements the interface StatementGenerator.
+func (o *OptionalGen) SetConsumeIterationsFast(count uint64) {
+	// This is a copy of SetConsumeIterations, except rewritten to use uint64
+	if count <= 0 {
+		o.Reset()
+		return
+	}
+	o.display = true
+	permutations, _ := o.PermutationsUint64()
+	count = count % permutations
+	count -= 1
+	o.children.SetConsumeIterationsFast(count)
+}
+
+// Copy implements the interface StatementGenerator.
+func (o *OptionalGen) Copy() StatementGenerator {
+	if o == nil {
+		return nil
+	}
+	return Optional(o.children.children...)
+}
+
+// String implements the interface StatementGenerator.
+func (o *OptionalGen) String() string {
+	if o.display {
+		return o.children.String()
+	} else {
+		return ""
+	}
+}
+
+// Reset implements the interface StatementGenerator.
+func (o *OptionalGen) Reset() {
+	o.display = false
+	o.children.Reset()
+}
+
+// SourceString implements the interface StatementGenerator.
+func (o *OptionalGen) SourceString() string {
+	return fmt.Sprintf(`Optional(%s)`, sourceGenerators(o.children.children))
+}
+
+// Permutations implements the interface StatementGenerator.
+func (o *OptionalGen) Permutations() *big.Int {
+	return new(big.Int).Add(BigIntOne, o.children.Permutations())
+}
+
+// PermutationsUint64 implements the interface StatementGenerator.
+func (o *OptionalGen) PermutationsUint64() (uint64, bool) {
+	childCount, ok := o.children.PermutationsUint64()
+	if !ok || childCount == math.MaxUint64 {
+		return math.MaxUint64, false
+	}
+	return 1 + childCount, true
+}
+
+// ApplyVariableDefinition applies the given map of variable definitions to the statement generator. This modifies the
+// statement generator, rather than returning a copy.
+func ApplyVariableDefinition(gen StatementGenerator, definitions map[string]StatementGenerator) error {
+	if len(definitions) == 0 {
+		return nil
+	}
+	switch gen := gen.(type) {
+	case *CollectionGen:
+		for _, child := range gen.children {
+			if err := ApplyVariableDefinition(child, definitions); err != nil {
+				return err
+			}
+		}
+	case *OptionalGen:
+		if err := ApplyVariableDefinition(gen.children, definitions); err != nil {
+			return err
+		}
+	case *OrGen:
+		for _, child := range gen.children {
+			if err := ApplyVariableDefinition(child, definitions); err != nil {
+				return err
+			}
+		}
+	case *TextGen:
+		// Nothing to do here
+	case *VariableGen:
+		if gen.options == nil {
+			if definition, ok := definitions[gen.name]; ok {
+				if err := gen.AddChildren(definition); err != nil {
+					return err
+				}
+				if err := ApplyVariableDefinition(gen.options, definitions); err != nil {
+					return err
+				}
+			}
+		} else {
+			if err := ApplyVariableDefinition(gen.options, definitions); err != nil {
+				return err
+			}
+		}
+	case nil:
+		return nil
+	default:
+		return fmt.Errorf("unknown generator encountered: %T", gen)
+	}
+	return nil
+}
+
+// UnsetVariables returns the name of all variables that do not have a definition. Sorted in ascending order.
+func UnsetVariables(gen StatementGenerator) ([]string, error) {
+	varNames := make(map[string]struct{})
+	switch gen := gen.(type) {
+	case *CollectionGen:
+		for _, child := range gen.children {
+			children, err := UnsetVariables(child)
+			if err != nil {
+				return nil, err
+			}
+			for _, childName := range children {
+				varNames[childName] = struct{}{}
+			}
+		}
+	case *OptionalGen:
+		return UnsetVariables(gen.children)
+	case *OrGen:
+		return UnsetVariables(Collection(gen.children...))
+	case *TextGen:
+		// Nothing to do here
+	case *VariableGen:
+		if gen.options == nil {
+			return []string{gen.name}, nil
+		} else {
+			return UnsetVariables(gen.options)
+		}
+	default:
+		return nil, fmt.Errorf("unknown generator encountered: %T", gen)
+	}
+	var varNamesSlice []string
+	for varName := range varNames {
+		varNamesSlice = append(varNamesSlice, varName)
+	}
+	sort.Strings(varNamesSlice)
+	return varNamesSlice, nil
+}
+
+// copyGenerators returns a full copy of the given slice of generators. Each generator will be in its original state.
+func copyGenerators(gens []StatementGenerator) []StatementGenerator {
+	gens = removeNilGenerators(gens)
+	if len(gens) == 0 {
+		return nil
+	}
+	newGens := make([]StatementGenerator, len(gens))
+	for i, gen := range gens {
+		newGens[i] = gen.Copy()
+	}
+	return newGens
+}
+
+// sourceGenerators returns a comma-separated SourceString from the given generator slice.
+func sourceGenerators(gens []StatementGenerator) string {
+	gens = removeNilGenerators(gens)
+	if len(gens) == 0 {
+		return ""
+	}
+	sourceStrs := make([]string, len(gens))
+	for i, gen := range gens {
+		sourceStrs[i] = gen.SourceString()
+	}
+	return strings.Join(sourceStrs, ", ")
+}
+
+// removeNilGenerators returns a new slice of generators with all nils removed.
+func removeNilGenerators(gens []StatementGenerator) []StatementGenerator {
+	newGens := make([]StatementGenerator, 0, len(gens))
+	for i := range gens {
+		if gens[i] != nil {
+			newGens = append(newGens, gens[i])
+		}
+	}
+	if len(newGens) == 0 {
+		return nil
+	}
+	return newGens
+}

testing/generation/utils/token.go

@@ -0,0 +1,113 @@
+// Copyright 2023-2024 Dolthub, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package utils
+
+// TokenType is the type of the token.
+type TokenType uint8
+
+const (
+	TokenType_Text TokenType = iota
+	TokenType_Variable
+	TokenType_VariableDefinition
+	TokenType_Or
+	TokenType_Repeat
+	TokenType_OptionalRepeat
+	TokenType_ShortSpace
+	TokenType_MediumSpace
+	TokenType_LongSpace
+	TokenType_ParenOpen
+	TokenType_ParenClose
+	TokenType_OptionalOpen
+	TokenType_OptionalClose
+	TokenType_OneOfOpen
+	TokenType_OneOfClose
+	TokenType_EOF
+)
+
+// Token represents a token in the synopsis. Unlike with traditional lexers, whitespace is tokenized, with multiple
+// token types for differing lengths, as the whitespace has some significance to the definition.
+type Token struct {
+	Type    TokenType
+	Literal string
+}
+
+// IsSpace returns whether the token represents one of the space types.
+func (t Token) IsSpace() bool {
+	return t.Type == TokenType_ShortSpace || t.Type == TokenType_MediumSpace || t.Type == TokenType_LongSpace
+}
+
+// IsStandardSpace returns whether the token represents a short or medium space, which will not end a statement.
+func (t Token) IsStandardSpace() bool {
+	return t.Type == TokenType_ShortSpace || t.Type == TokenType_MediumSpace
+}
+
+// IsNewStatement returns whether the token represents one a long space, which will end a statement
+func (t Token) IsNewStatement() bool {
+	return t.Type == TokenType_LongSpace
+}
+
+// CreatesNewScope returns whether the token creates a new scope.
+func (t Token) CreatesNewScope() bool {
+	return t.Type == TokenType_ParenOpen || t.Type == TokenType_OptionalOpen || t.Type == TokenType_OneOfOpen
+}
+
+// ExitsScope returns whether the token exits the current scope.
+func (t Token) ExitsScope() bool {
+	return t.Type == TokenType_ParenClose || t.Type == TokenType_OptionalClose || t.Type == TokenType_OneOfClose
+}
+
+// String returns a string representation of the token.
+func (t Token) String() string {
+	switch t.Type {
+	case TokenType_Text:
+		return t.Literal
+	case TokenType_Variable:
+		return "$" + t.Literal + "$"
+	case TokenType_VariableDefinition:
+		return "where $" + t.Literal + "$ is:"
+	case TokenType_Or:
+		return "|"
+	case TokenType_Repeat:
+		return "..."
+	case TokenType_OptionalRepeat:
+		if len(t.Literal) > 0 {
+			return "[ " + t.Literal + " ... ]"
+		} else {
+			return "[ ... ]"
+		}
+	case TokenType_ShortSpace:
+		return " "
+	case TokenType_MediumSpace:
+		return "\n    "
+	case TokenType_LongSpace:
+		return "\n\n"
+	case TokenType_ParenOpen:
+		return "("
+	case TokenType_ParenClose:
+		return ")"
+	case TokenType_OptionalOpen:
+		return "["
+	case TokenType_OptionalClose:
+		return "]"
+	case TokenType_OneOfOpen:
+		return "{"
+	case TokenType_OneOfClose:
+		return "}"
+	case TokenType_EOF:
+		return ""
+	default:
+		panic("unexpected token type")
+	}
+}

testing/generation/utils/token_reader.go

@@ -0,0 +1,80 @@
+// Copyright 2023-2024 Dolthub, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package utils
+
+// TokenReader may be used to easily read through a collection of tokens generated from a synopsis. Of note, a
+// TokenReader will skip all short and medium spaces, as clients of a reader will only care about statement boundaries.
+type TokenReader struct {
+	tokens []Token
+	index  int
+}
+
+// NewTokenReader returns a new *TokenReader created from the given tokens.
+func NewTokenReader(tokens []Token) *TokenReader {
+	sanitizedTokens := make([]Token, 0, len(tokens))
+	for _, token := range tokens {
+		if token.IsStandardSpace() {
+			continue
+		}
+		sanitizedTokens = append(sanitizedTokens, token)
+	}
+	return &TokenReader{
+		tokens: sanitizedTokens,
+		index:  -1,
+	}
+}
+
+// Next returns the next token while advancing the reader. Returns false if there are no more tokens.
+func (reader *TokenReader) Next() (Token, bool) {
+	if reader.index+1 >= len(reader.tokens) {
+		return Token{Type: TokenType_EOF}, false
+	}
+	reader.index++
+	return reader.tokens[reader.index], true
+}
+
+// Peek returns the next token. Does not advance the reader. Returns false if there are no more tokens.
+func (reader *TokenReader) Peek() (Token, bool) {
+	if reader.index+1 >= len(reader.tokens) {
+		return Token{Type: TokenType_EOF}, false
+	}
+	return reader.tokens[reader.index+1], true
+}
+
+// PeekBy returns the next token that is n positions from the current token. Does not advance the reader. Returns false
+// if we are peeking beyond the slice.
+func (reader *TokenReader) PeekBy(n int) (Token, bool) {
+	if reader.index+n >= len(reader.tokens) || reader.index+n < 0 {
+		return Token{Type: TokenType_EOF}, false
+	}
+	return reader.tokens[reader.index+n], true
+}
+
+// Advance is equivalent to calling AdvanceBy(1).
+func (reader *TokenReader) Advance() {
+	reader.AdvanceBy(1)
+}
+
+// AdvanceBy advances the reader by the given amount. If the amount is greater than the number of remaining tokens, then
+// it advances to the end. Cannot advance backwards.
+func (reader *TokenReader) AdvanceBy(n int) {
+	if n < 0 {
+		n = 0
+	}
+	if reader.index+n >= len(reader.tokens) {
+		n = len(reader.tokens) - reader.index - 1
+	}
+	reader.index += n
+}