chore: import upstream snapshot with attribution

testing/generation/utils/token_reader.go

@@ -0,0 +1,80 @@
+// Copyright 2023-2024 Dolthub, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package utils
+
+// TokenReader may be used to easily read through a collection of tokens generated from a synopsis. Of note, a
+// TokenReader will skip all short and medium spaces, as clients of a reader will only care about statement boundaries.
+type TokenReader struct {
+	tokens []Token
+	index  int
+}
+
+// NewTokenReader returns a new *TokenReader created from the given tokens.
+func NewTokenReader(tokens []Token) *TokenReader {
+	sanitizedTokens := make([]Token, 0, len(tokens))
+	for _, token := range tokens {
+		if token.IsStandardSpace() {
+			continue
+		}
+		sanitizedTokens = append(sanitizedTokens, token)
+	}
+	return &TokenReader{
+		tokens: sanitizedTokens,
+		index:  -1,
+	}
+}
+
+// Next returns the next token while advancing the reader. Returns false if there are no more tokens.
+func (reader *TokenReader) Next() (Token, bool) {
+	if reader.index+1 >= len(reader.tokens) {
+		return Token{Type: TokenType_EOF}, false
+	}
+	reader.index++
+	return reader.tokens[reader.index], true
+}
+
+// Peek returns the next token. Does not advance the reader. Returns false if there are no more tokens.
+func (reader *TokenReader) Peek() (Token, bool) {
+	if reader.index+1 >= len(reader.tokens) {
+		return Token{Type: TokenType_EOF}, false
+	}
+	return reader.tokens[reader.index+1], true
+}
+
+// PeekBy returns the next token that is n positions from the current token. Does not advance the reader. Returns false
+// if we are peeking beyond the slice.
+func (reader *TokenReader) PeekBy(n int) (Token, bool) {
+	if reader.index+n >= len(reader.tokens) || reader.index+n < 0 {
+		return Token{Type: TokenType_EOF}, false
+	}
+	return reader.tokens[reader.index+n], true
+}
+
+// Advance is equivalent to calling AdvanceBy(1).
+func (reader *TokenReader) Advance() {
+	reader.AdvanceBy(1)
+}
+
+// AdvanceBy advances the reader by the given amount. If the amount is greater than the number of remaining tokens, then
+// it advances to the end. Cannot advance backwards.
+func (reader *TokenReader) AdvanceBy(n int) {
+	if n < 0 {
+		n = 0
+	}
+	if reader.index+n >= len(reader.tokens) {
+		n = len(reader.tokens) - reader.index - 1
+	}
+	reader.index += n
+}