chore: import upstream snapshot with attribution

2026-07-13 12:32:25 +08:00
commit e014feafe1
2285 changed files with 1131979 additions and 0 deletions
@@ -0,0 +1,330 @@
+// Copyright 2023 Dolthub, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package main
+
+import (
+	"fmt"
+	"strings"
+
+	"github.com/dolthub/doltgresql/testing/generation/utils"
+)
+
+// Scanner allows for scanning through the synopsis to generate tokens.
+type Scanner struct {
+	source     []rune
+	tokens     []utils.Token
+	sourceIdx  int
+	isFinished bool
+	savedErr   error
+}
+
+// NewScanner returns a new *Scanner by reading through the synopsis.
+func NewScanner(synopsis string) *Scanner {
+	scanner := &Scanner{
+		source:     []rune(strings.ReplaceAll(synopsis, "\r", "")),
+		tokens:     nil,
+		sourceIdx:  -1,
+		isFinished: false,
+		savedErr:   nil,
+	}
+	_, _ = scanner.Process()
+	return scanner
+}
+
+// Next returns the next rune while advancing the scanner. Returns false if there are no more runes.
+func (scanner *Scanner) Next() (rune, bool) {
+	if scanner.sourceIdx+1 >= len(scanner.source) {
+		return 0, false
+	}
+	scanner.sourceIdx++
+	return scanner.source[scanner.sourceIdx], true
+}
+
+// Peek returns the next rune. Does not advance the scanner. Returns false if there are no more runes.
+func (scanner *Scanner) Peek() (rune, bool) {
+	if scanner.sourceIdx+1 >= len(scanner.source) {
+		return 0, false
+	}
+	return scanner.source[scanner.sourceIdx+1], true
+}
+
+// PeekBy returns the next rune that is n positions from the current rune. Does not advance the scanner. Returns false
+// if we are peeking beyond the source.
+func (scanner *Scanner) PeekBy(n int) (rune, bool) {
+	if scanner.sourceIdx+n >= len(scanner.source) || scanner.sourceIdx+n < 0 {
+		return 0, false
+	}
+	return scanner.source[scanner.sourceIdx+n], true
+}
+
+// PeekMatch returns whether the given string exactly matches runes starting after the current position. This is
+// equivalent to calling PeekMatchOffset with an offset of 1.
+func (scanner *Scanner) PeekMatch(str string) bool {
+	return scanner.PeekMatchOffset(str, 1)
+}
+
+// PeekMatchOffset returns whether the given string exactly matches runes starting from the offset applied to the
+// current position. An offset of 0 means that we are including the rune at the current position.
+func (scanner *Scanner) PeekMatchOffset(str string, offset int) bool {
+	for i, r := range []rune(str) {
+		sr, ok := scanner.PeekBy(i + offset)
+		if !ok || sr != r {
+			return false
+		}
+	}
+	return true
+}
+
+// Advance is equivalent to calling AdvanceBy(1).
+func (scanner *Scanner) Advance() {
+	scanner.AdvanceBy(1)
+}
+
+// AdvanceBy advances the scanner by the given amount. If the amount is greater than the number of remaining runes, then
+// it advances to the end. Cannot advance backwards.
+func (scanner *Scanner) AdvanceBy(n int) {
+	if n < 0 {
+		n = 0
+	}
+	if scanner.sourceIdx+n >= len(scanner.source) {
+		n = len(scanner.source) - scanner.sourceIdx - 1
+	}
+	scanner.sourceIdx += n
+}
+
+// Process processes the synopsis and returns the generated tokens.
+func (scanner *Scanner) Process() ([]utils.Token, error) {
+	if scanner.isFinished || scanner.savedErr != nil {
+		return scanner.tokens, scanner.savedErr
+	}
+ScannerLoop:
+	for {
+		r, ok := scanner.Next()
+		if !ok {
+			break ScannerLoop
+		}
+		switch r {
+		case '$':
+			var name []rune
+			for r, ok = scanner.Peek(); ok && r != '$'; r, ok = scanner.Peek() {
+				scanner.Advance()
+				name = append(name, r)
+			}
+			if !ok {
+				scanner.savedErr = fmt.Errorf("unexpected EOF when reading variable name")
+				break ScannerLoop
+			}
+			scanner.Advance()
+			scanner.tokens = append(scanner.tokens, utils.Token{
+				Type:    utils.TokenType_Variable,
+				Literal: string(name),
+			})
+		case '\t':
+			scanner.savedErr = fmt.Errorf("tab found, remove all tabs from the synopsis")
+			break ScannerLoop
+		case ' ', '\n':
+			spaceCount := 0
+			lineCount := 0
+			if r == ' ' {
+				spaceCount += 1
+			}
+			if r == '\n' {
+				lineCount++
+			}
+			commentMode := false
+		WhitespaceLoop:
+			for r, ok = scanner.Peek(); ok; r, ok = scanner.Peek() {
+				if commentMode {
+					scanner.Advance()
+					if r == '\n' {
+						commentMode = false
+					}
+				} else {
+					switch r {
+					case ' ':
+						scanner.Advance()
+						spaceCount += 1
+					case '\n':
+						scanner.Advance()
+						lineCount++
+					case '/':
+						if next, _ := scanner.PeekBy(2); next == '/' {
+							if last, _ := scanner.PeekBy(0); last == '\n' {
+								scanner.Advance()
+								commentMode = true
+								continue WhitespaceLoop
+							}
+						}
+						break WhitespaceLoop
+					default:
+						break WhitespaceLoop
+					}
+				}
+			}
+			// EOF, no need to add the last bit of whitespace
+			if !ok {
+				break ScannerLoop
+			}
+			if spaceCount == 1 && lineCount == 0 {
+				scanner.tokens = append(scanner.tokens, utils.Token{Type: utils.TokenType_ShortSpace})
+			} else if spaceCount > 1 && lineCount <= 1 {
+				scanner.tokens = append(scanner.tokens, utils.Token{Type: utils.TokenType_MediumSpace})
+			} else {
+				// This will match the case where spaceCount == 0 and lineCount == 1.
+				// This may seem counter-intuitive, however it's consistent with how new statements are defined.
+				scanner.tokens = append(scanner.tokens, utils.Token{Type: utils.TokenType_LongSpace})
+			}
+		case '.':
+			dotCount := 1
+			// All dot repetition blocks look like `...`
+		DotRepetitionLoop:
+			for n := 1; true; n++ {
+				peek, _ := scanner.PeekBy(n)
+				switch peek {
+				case '.':
+					dotCount++
+				default:
+					scanner.AdvanceBy(n - 1)
+					// If the dot count is different from 3, then we'll treat it as a string
+					if dotCount == 3 {
+						scanner.tokens = append(scanner.tokens, utils.Token{Type: utils.TokenType_Repeat})
+					} else {
+						scanner.tokens = append(scanner.tokens, utils.Token{
+							Type:    utils.TokenType_Text,
+							Literal: strings.Repeat(".", dotCount),
+						})
+					}
+					break DotRepetitionLoop
+				}
+			}
+		case ',':
+			scanner.tokens = append(scanner.tokens, utils.Token{
+				Type:    utils.TokenType_Text,
+				Literal: ",",
+			})
+		case '[':
+			if scanner.PeekMatchOffset("[ ... ]", 0) {
+				scanner.AdvanceBy(6)
+				scanner.tokens = append(scanner.tokens, utils.Token{
+					Type:    utils.TokenType_OptionalRepeat,
+					Literal: "",
+				})
+			} else if scanner.PeekMatchOffset("[ , ... ]", 0) {
+				scanner.AdvanceBy(8)
+				scanner.tokens = append(scanner.tokens, utils.Token{
+					Type:    utils.TokenType_OptionalRepeat,
+					Literal: ",",
+				})
+			} else if scanner.PeekMatchOffset("[ AND ... ]", 0) {
+				scanner.AdvanceBy(10)
+				scanner.tokens = append(scanner.tokens, utils.Token{
+					Type:    utils.TokenType_OptionalRepeat,
+					Literal: "AND",
+				})
+			} else if scanner.PeekMatchOffset("[ OR ... ]", 0) {
+				scanner.AdvanceBy(9)
+				scanner.tokens = append(scanner.tokens, utils.Token{
+					Type:    utils.TokenType_OptionalRepeat,
+					Literal: "OR",
+				})
+			} else {
+				scanner.tokens = append(scanner.tokens, utils.Token{Type: utils.TokenType_OptionalOpen})
+			}
+		case ']':
+			scanner.tokens = append(scanner.tokens, utils.Token{Type: utils.TokenType_OptionalClose})
+		case '{':
+			scanner.tokens = append(scanner.tokens, utils.Token{Type: utils.TokenType_OneOfOpen})
+		case '}':
+			scanner.tokens = append(scanner.tokens, utils.Token{Type: utils.TokenType_OneOfClose})
+		case '(':
+			scanner.tokens = append(scanner.tokens, utils.Token{Type: utils.TokenType_ParenOpen})
+		case ')':
+			scanner.tokens = append(scanner.tokens, utils.Token{Type: utils.TokenType_ParenClose})
+		case '|':
+			scanner.tokens = append(scanner.tokens, utils.Token{Type: utils.TokenType_Or})
+		default:
+			if scanner.PeekMatchOffset("where $", 0) {
+				// Skip past what we peeked at
+				scanner.AdvanceBy(6)
+				var varName []rune
+			VariableDescriptionLoop:
+				for r, ok = scanner.Peek(); ok; r, ok = scanner.Peek() {
+					switch r {
+					case '$':
+						scanner.tokens = append(scanner.tokens, utils.Token{
+							Type:    utils.TokenType_VariableDefinition,
+							Literal: string(varName),
+						})
+						if !scanner.PeekMatch("$ is:") {
+							scanner.savedErr = fmt.Errorf("invalid variable definition format")
+							break ScannerLoop
+						}
+						// Skip past what we peeked at
+						scanner.AdvanceBy(5)
+						break VariableDescriptionLoop
+					default:
+						scanner.Advance()
+						varName = append(varName, r)
+					}
+				}
+			} else {
+				text := []rune{r}
+			TextLoop:
+				for r, ok = scanner.Peek(); ok; r, ok = scanner.Peek() {
+					switch r {
+					case '$', ' ', '\t', '\n', '.', ',', '[', ']', '{', '}', '(', ')', '|':
+						scanner.tokens = append(scanner.tokens, utils.Token{
+							Type:    utils.TokenType_Text,
+							Literal: string(text),
+						})
+						break TextLoop
+					default:
+						scanner.Advance()
+						text = append(text, r)
+					}
+				}
+				// If we hit EOF, then we haven't added the word yet, so we'll do it here
+				if !ok {
+					scanner.tokens = append(scanner.tokens, utils.Token{
+						Type:    utils.TokenType_Text,
+						Literal: string(text),
+					})
+				}
+			}
+		}
+	}
+	// Remove any ending spaces
+	if len(scanner.tokens) > 0 && scanner.tokens[len(scanner.tokens)-1].IsSpace() {
+		scanner.tokens = scanner.tokens[:len(scanner.tokens)-1]
+	}
+	// Add an EOF
+	scanner.tokens = append(scanner.tokens, utils.Token{Type: utils.TokenType_EOF})
+	// Set that we're finished now (we'll also finish on errors)
+	scanner.isFinished = true
+	return scanner.tokens, scanner.savedErr
+}
+
+// String returns the processed contents of the scanner as a string. This means that it will not be the original string,
+// as the processed string may differ in format.
+func (scanner *Scanner) String() string {
+	sb := strings.Builder{}
+	for _, token := range scanner.tokens {
+		str := token.String()
+		if len(str) > 0 {
+			sb.WriteString(str)
+		}
+	}
+	return sb.String()
+}