dolthub--doltgresql/testing/generation/command_docs/scanner.go

// Copyright 2023 Dolthub, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"fmt"
	"strings"

	"github.com/dolthub/doltgresql/testing/generation/utils"
)

// Scanner allows for scanning through the synopsis to generate tokens.
type Scanner struct {
	source     []rune
	tokens     []utils.Token
	sourceIdx  int
	isFinished bool
	savedErr   error
}

// NewScanner returns a new *Scanner by reading through the synopsis.
func NewScanner(synopsis string) *Scanner {
	scanner := &Scanner{
		source:     []rune(strings.ReplaceAll(synopsis, "\r", "")),
		tokens:     nil,
		sourceIdx:  -1,
		isFinished: false,
		savedErr:   nil,
	}
	_, _ = scanner.Process()
	return scanner
}

// Next returns the next rune while advancing the scanner. Returns false if there are no more runes.
func (scanner *Scanner) Next() (rune, bool) {
	if scanner.sourceIdx+1 >= len(scanner.source) {
		return 0, false
	}
	scanner.sourceIdx++
	return scanner.source[scanner.sourceIdx], true
}

// Peek returns the next rune. Does not advance the scanner. Returns false if there are no more runes.
func (scanner *Scanner) Peek() (rune, bool) {
	if scanner.sourceIdx+1 >= len(scanner.source) {
		return 0, false
	}
	return scanner.source[scanner.sourceIdx+1], true
}

// PeekBy returns the next rune that is n positions from the current rune. Does not advance the scanner. Returns false
// if we are peeking beyond the source.
func (scanner *Scanner) PeekBy(n int) (rune, bool) {
	if scanner.sourceIdx+n >= len(scanner.source) || scanner.sourceIdx+n < 0 {
		return 0, false
	}
	return scanner.source[scanner.sourceIdx+n], true
}

// PeekMatch returns whether the given string exactly matches runes starting after the current position. This is
// equivalent to calling PeekMatchOffset with an offset of 1.
func (scanner *Scanner) PeekMatch(str string) bool {
	return scanner.PeekMatchOffset(str, 1)
}

// PeekMatchOffset returns whether the given string exactly matches runes starting from the offset applied to the
// current position. An offset of 0 means that we are including the rune at the current position.
func (scanner *Scanner) PeekMatchOffset(str string, offset int) bool {
	for i, r := range []rune(str) {
		sr, ok := scanner.PeekBy(i + offset)
		if !ok || sr != r {
			return false
		}
	}
	return true
}

// Advance is equivalent to calling AdvanceBy(1).
func (scanner *Scanner) Advance() {
	scanner.AdvanceBy(1)
}

// AdvanceBy advances the scanner by the given amount. If the amount is greater than the number of remaining runes, then
// it advances to the end. Cannot advance backwards.
func (scanner *Scanner) AdvanceBy(n int) {
	if n < 0 {
		n = 0
	}
	if scanner.sourceIdx+n >= len(scanner.source) {
		n = len(scanner.source) - scanner.sourceIdx - 1
	}
	scanner.sourceIdx += n
}

// Process processes the synopsis and returns the generated tokens.
func (scanner *Scanner) Process() ([]utils.Token, error) {
	if scanner.isFinished || scanner.savedErr != nil {
		return scanner.tokens, scanner.savedErr
	}
ScannerLoop:
	for {
		r, ok := scanner.Next()
		if !ok {
			break ScannerLoop
		}
		switch r {
		case '$':
			var name []rune
			for r, ok = scanner.Peek(); ok && r != '$'; r, ok = scanner.Peek() {
				scanner.Advance()
				name = append(name, r)
			}
			if !ok {
				scanner.savedErr = fmt.Errorf("unexpected EOF when reading variable name")
				break ScannerLoop
			}
			scanner.Advance()
			scanner.tokens = append(scanner.tokens, utils.Token{
				Type:    utils.TokenType_Variable,
				Literal: string(name),
			})
		case '\t':
			scanner.savedErr = fmt.Errorf("tab found, remove all tabs from the synopsis")
			break ScannerLoop
		case ' ', '\n':
			spaceCount := 0
			lineCount := 0
			if r == ' ' {
				spaceCount += 1
			}
			if r == '\n' {
				lineCount++
			}
			commentMode := false
		WhitespaceLoop:
			for r, ok = scanner.Peek(); ok; r, ok = scanner.Peek() {
				if commentMode {
					scanner.Advance()
					if r == '\n' {
						commentMode = false
					}
				} else {
					switch r {
					case ' ':
						scanner.Advance()
						spaceCount += 1
					case '\n':
						scanner.Advance()
						lineCount++
					case '/':
						if next, _ := scanner.PeekBy(2); next == '/' {
							if last, _ := scanner.PeekBy(0); last == '\n' {
								scanner.Advance()
								commentMode = true
								continue WhitespaceLoop
							}
						}
						break WhitespaceLoop
					default:
						break WhitespaceLoop
					}
				}
			}
			// EOF, no need to add the last bit of whitespace
			if !ok {
				break ScannerLoop
			}
			if spaceCount == 1 && lineCount == 0 {
				scanner.tokens = append(scanner.tokens, utils.Token{Type: utils.TokenType_ShortSpace})
			} else if spaceCount > 1 && lineCount <= 1 {
				scanner.tokens = append(scanner.tokens, utils.Token{Type: utils.TokenType_MediumSpace})
			} else {
				// This will match the case where spaceCount == 0 and lineCount == 1.
				// This may seem counter-intuitive, however it's consistent with how new statements are defined.
				scanner.tokens = append(scanner.tokens, utils.Token{Type: utils.TokenType_LongSpace})
			}
		case '.':
			dotCount := 1
			// All dot repetition blocks look like `...`
		DotRepetitionLoop:
			for n := 1; true; n++ {
				peek, _ := scanner.PeekBy(n)
				switch peek {
				case '.':
					dotCount++
				default:
					scanner.AdvanceBy(n - 1)
					// If the dot count is different from 3, then we'll treat it as a string
					if dotCount == 3 {
						scanner.tokens = append(scanner.tokens, utils.Token{Type: utils.TokenType_Repeat})
					} else {
						scanner.tokens = append(scanner.tokens, utils.Token{
							Type:    utils.TokenType_Text,
							Literal: strings.Repeat(".", dotCount),
						})
					}
					break DotRepetitionLoop
				}
			}
		case ',':
			scanner.tokens = append(scanner.tokens, utils.Token{
				Type:    utils.TokenType_Text,
				Literal: ",",
			})
		case '[':
			if scanner.PeekMatchOffset("[ ... ]", 0) {
				scanner.AdvanceBy(6)
				scanner.tokens = append(scanner.tokens, utils.Token{
					Type:    utils.TokenType_OptionalRepeat,
					Literal: "",
				})
			} else if scanner.PeekMatchOffset("[ , ... ]", 0) {
				scanner.AdvanceBy(8)
				scanner.tokens = append(scanner.tokens, utils.Token{
					Type:    utils.TokenType_OptionalRepeat,
					Literal: ",",
				})
			} else if scanner.PeekMatchOffset("[ AND ... ]", 0) {
				scanner.AdvanceBy(10)
				scanner.tokens = append(scanner.tokens, utils.Token{
					Type:    utils.TokenType_OptionalRepeat,
					Literal: "AND",
				})
			} else if scanner.PeekMatchOffset("[ OR ... ]", 0) {
				scanner.AdvanceBy(9)
				scanner.tokens = append(scanner.tokens, utils.Token{
					Type:    utils.TokenType_OptionalRepeat,
					Literal: "OR",
				})
			} else {
				scanner.tokens = append(scanner.tokens, utils.Token{Type: utils.TokenType_OptionalOpen})
			}
		case ']':
			scanner.tokens = append(scanner.tokens, utils.Token{Type: utils.TokenType_OptionalClose})
		case '{':
			scanner.tokens = append(scanner.tokens, utils.Token{Type: utils.TokenType_OneOfOpen})
		case '}':
			scanner.tokens = append(scanner.tokens, utils.Token{Type: utils.TokenType_OneOfClose})
		case '(':
			scanner.tokens = append(scanner.tokens, utils.Token{Type: utils.TokenType_ParenOpen})
		case ')':
			scanner.tokens = append(scanner.tokens, utils.Token{Type: utils.TokenType_ParenClose})
		case '|':
			scanner.tokens = append(scanner.tokens, utils.Token{Type: utils.TokenType_Or})
		default:
			if scanner.PeekMatchOffset("where $", 0) {
				// Skip past what we peeked at
				scanner.AdvanceBy(6)
				var varName []rune
			VariableDescriptionLoop:
				for r, ok = scanner.Peek(); ok; r, ok = scanner.Peek() {
					switch r {
					case '$':
						scanner.tokens = append(scanner.tokens, utils.Token{
							Type:    utils.TokenType_VariableDefinition,
							Literal: string(varName),
						})
						if !scanner.PeekMatch("$ is:") {
							scanner.savedErr = fmt.Errorf("invalid variable definition format")
							break ScannerLoop
						}
						// Skip past what we peeked at
						scanner.AdvanceBy(5)
						break VariableDescriptionLoop
					default:
						scanner.Advance()
						varName = append(varName, r)
					}
				}
			} else {
				text := []rune{r}
			TextLoop:
				for r, ok = scanner.Peek(); ok; r, ok = scanner.Peek() {
					switch r {
					case '$', ' ', '\t', '\n', '.', ',', '[', ']', '{', '}', '(', ')', '|':
						scanner.tokens = append(scanner.tokens, utils.Token{
							Type:    utils.TokenType_Text,
							Literal: string(text),
						})
						break TextLoop
					default:
						scanner.Advance()
						text = append(text, r)
					}
				}
				// If we hit EOF, then we haven't added the word yet, so we'll do it here
				if !ok {
					scanner.tokens = append(scanner.tokens, utils.Token{
						Type:    utils.TokenType_Text,
						Literal: string(text),
					})
				}
			}
		}
	}
	// Remove any ending spaces
	if len(scanner.tokens) > 0 && scanner.tokens[len(scanner.tokens)-1].IsSpace() {
		scanner.tokens = scanner.tokens[:len(scanner.tokens)-1]
	}
	// Add an EOF
	scanner.tokens = append(scanner.tokens, utils.Token{Type: utils.TokenType_EOF})
	// Set that we're finished now (we'll also finish on errors)
	scanner.isFinished = true
	return scanner.tokens, scanner.savedErr
}

// String returns the processed contents of the scanner as a string. This means that it will not be the original string,
// as the processed string may differ in format.
func (scanner *Scanner) String() string {
	sb := strings.Builder{}
	for _, token := range scanner.tokens {
		str := token.String()
		if len(str) > 0 {
			sb.WriteString(str)
		}
	}
	return sb.String()
}