chore: import upstream snapshot with attribution

testing/generation/utils/token.go

@@ -0,0 +1,113 @@
+// Copyright 2023-2024 Dolthub, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package utils
+
+// TokenType is the type of the token.
+type TokenType uint8
+
+const (
+	TokenType_Text TokenType = iota
+	TokenType_Variable
+	TokenType_VariableDefinition
+	TokenType_Or
+	TokenType_Repeat
+	TokenType_OptionalRepeat
+	TokenType_ShortSpace
+	TokenType_MediumSpace
+	TokenType_LongSpace
+	TokenType_ParenOpen
+	TokenType_ParenClose
+	TokenType_OptionalOpen
+	TokenType_OptionalClose
+	TokenType_OneOfOpen
+	TokenType_OneOfClose
+	TokenType_EOF
+)
+
+// Token represents a token in the synopsis. Unlike with traditional lexers, whitespace is tokenized, with multiple
+// token types for differing lengths, as the whitespace has some significance to the definition.
+type Token struct {
+	Type    TokenType
+	Literal string
+}
+
+// IsSpace returns whether the token represents one of the space types.
+func (t Token) IsSpace() bool {
+	return t.Type == TokenType_ShortSpace || t.Type == TokenType_MediumSpace || t.Type == TokenType_LongSpace
+}
+
+// IsStandardSpace returns whether the token represents a short or medium space, which will not end a statement.
+func (t Token) IsStandardSpace() bool {
+	return t.Type == TokenType_ShortSpace || t.Type == TokenType_MediumSpace
+}
+
+// IsNewStatement returns whether the token represents one a long space, which will end a statement
+func (t Token) IsNewStatement() bool {
+	return t.Type == TokenType_LongSpace
+}
+
+// CreatesNewScope returns whether the token creates a new scope.
+func (t Token) CreatesNewScope() bool {
+	return t.Type == TokenType_ParenOpen || t.Type == TokenType_OptionalOpen || t.Type == TokenType_OneOfOpen
+}
+
+// ExitsScope returns whether the token exits the current scope.
+func (t Token) ExitsScope() bool {
+	return t.Type == TokenType_ParenClose || t.Type == TokenType_OptionalClose || t.Type == TokenType_OneOfClose
+}
+
+// String returns a string representation of the token.
+func (t Token) String() string {
+	switch t.Type {
+	case TokenType_Text:
+		return t.Literal
+	case TokenType_Variable:
+		return "$" + t.Literal + "$"
+	case TokenType_VariableDefinition:
+		return "where $" + t.Literal + "$ is:"
+	case TokenType_Or:
+		return "|"
+	case TokenType_Repeat:
+		return "..."
+	case TokenType_OptionalRepeat:
+		if len(t.Literal) > 0 {
+			return "[ " + t.Literal + " ... ]"
+		} else {
+			return "[ ... ]"
+		}
+	case TokenType_ShortSpace:
+		return " "
+	case TokenType_MediumSpace:
+		return "\n    "
+	case TokenType_LongSpace:
+		return "\n\n"
+	case TokenType_ParenOpen:
+		return "("
+	case TokenType_ParenClose:
+		return ")"
+	case TokenType_OptionalOpen:
+		return "["
+	case TokenType_OptionalClose:
+		return "]"
+	case TokenType_OneOfOpen:
+		return "{"
+	case TokenType_OneOfClose:
+		return "}"
+	case TokenType_EOF:
+		return ""
+	default:
+		panic("unexpected token type")
+	}
+}