chore: import upstream snapshot with attribution

2026-07-13 12:32:25 +08:00
commit e014feafe1
2285 changed files with 1131979 additions and 0 deletions
@@ -0,0 +1,124 @@
+// Copyright 2023-2024 Dolthub, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package utils
+
+import "github.com/cockroachdb/errors"
+
+// ParseTokens parses the given tokens into a StatementGenerator.
+func ParseTokens(tokens []Token, includeRepetition bool) (StatementGenerator, error) {
+	stack := NewStatementGeneratorStack()
+	var statements []StatementGenerator
+	variables := make(map[string]StatementGenerator)
+	currentVariable := ""
+	tokenReader := NewTokenReader(tokens)
+ForLoop:
+	for {
+		token, ok := tokenReader.Next()
+		if !ok {
+			break ForLoop
+		}
+		switch token.Type {
+		case TokenType_Text:
+			stack.AddText(token.Literal)
+		case TokenType_Variable:
+			stack.AddVariable(token.Literal)
+		case TokenType_VariableDefinition:
+			currentVariable = token.Literal
+			if token, _ = tokenReader.Next(); token.Type != TokenType_LongSpace {
+				return nil, errors.Errorf("expected a long space after a variable definition declaration")
+			}
+		case TokenType_Or:
+			if err := stack.Or(); err != nil {
+				return nil, err
+			}
+		case TokenType_Repeat:
+			if includeRepetition {
+				if err := stack.Repeat(); err != nil {
+					return nil, err
+				}
+			}
+		case TokenType_OptionalRepeat:
+			if includeRepetition {
+				if err := stack.OptionalRepeat(token.Literal); err != nil {
+					return nil, err
+				}
+			}
+		case TokenType_ShortSpace, TokenType_MediumSpace:
+			return nil, errors.Errorf("token reader should have removed all short and medium spaces")
+		case TokenType_LongSpace, TokenType_EOF:
+			newStatement, err := stack.Finish()
+			if err != nil {
+				return nil, err
+			}
+			if newStatement == nil {
+				return nil, errors.Errorf("long space encountered before writing to the stack")
+			}
+			if len(currentVariable) > 0 {
+				if _, ok = variables[currentVariable]; ok {
+					return nil, errors.Errorf("multiple definitions for the same variable: %s", currentVariable)
+				}
+				variables[currentVariable] = newStatement
+				currentVariable = ""
+			} else {
+				statements = append(statements, newStatement)
+			}
+			if token.Type == TokenType_EOF {
+				break ForLoop
+			} else {
+				stack = NewStatementGeneratorStack()
+			}
+		case TokenType_ParenOpen:
+			stack.NewParenScope()
+		case TokenType_ParenClose:
+			if err := stack.ExitParenScope(); err != nil {
+				return nil, err
+			}
+		case TokenType_OptionalOpen:
+			stack.NewOptionalScope()
+		case TokenType_OptionalClose:
+			if err := stack.ExitOptionalScope(); err != nil {
+				return nil, err
+			}
+		case TokenType_OneOfOpen:
+			stack.NewScope()
+		case TokenType_OneOfClose:
+			if err := stack.ExitScope(); err != nil {
+				return nil, err
+			}
+		default:
+			panic("unhandled token type")
+		}
+	}
+	finalStackContents, err := stack.Finish()
+	if err != nil {
+		return nil, err
+	}
+	if finalStackContents != nil {
+		return nil, errors.Errorf("encountered an early EOF, as the stack was still processing")
+	}
+	if len(statements) == 0 {
+		return nil, errors.Errorf("no statements were generated from the token stream")
+	}
+	var finalStatementGenerator StatementGenerator
+	if len(statements) == 1 {
+		finalStatementGenerator = statements[0]
+	} else {
+		finalStatementGenerator = Or(statements...)
+	}
+	if err = ApplyVariableDefinition(finalStatementGenerator, variables); err != nil {
+		return nil, err
+	}
+	return finalStatementGenerator, nil
+}