wehub-resource-sync
128 lines
1.8 KiB
Go
128 lines
1.8 KiB
Go
package tokenizer
|
|
|
|
import "fmt"
|
|
|
|
// TokenType represents a Token's type.
|
|
type TokenType int8
|
|
|
|
// All TokenType constants.
|
|
const (
|
|
Unknown TokenType = iota
|
|
|
|
Identifier
|
|
Subquery
|
|
|
|
Select
|
|
From
|
|
Where
|
|
|
|
As
|
|
Or
|
|
And
|
|
Not
|
|
|
|
In
|
|
Is
|
|
Like
|
|
RLike
|
|
|
|
Equals
|
|
NotEquals
|
|
GreaterThanEquals
|
|
GreaterThan
|
|
LessThanEquals
|
|
LessThan
|
|
|
|
Comma
|
|
Hyphen
|
|
ExclamationMark
|
|
OpenParen
|
|
CloseParen
|
|
OpenBracket
|
|
CloseBracket
|
|
)
|
|
|
|
func (t TokenType) String() string {
|
|
switch t {
|
|
case Identifier:
|
|
return "identifier"
|
|
case Subquery:
|
|
return "subquery"
|
|
case Select:
|
|
return "select"
|
|
case From:
|
|
return "from"
|
|
case As:
|
|
return "as"
|
|
case Where:
|
|
return "where"
|
|
case Or:
|
|
return "or"
|
|
case And:
|
|
return "and"
|
|
case Not:
|
|
return "not"
|
|
case In:
|
|
return "in"
|
|
case Is:
|
|
return "is"
|
|
case Like:
|
|
return "like"
|
|
case RLike:
|
|
return "RLike"
|
|
case Equals:
|
|
return "equal"
|
|
case NotEquals:
|
|
return "not-equal"
|
|
case GreaterThanEquals:
|
|
return "greater-than-or-equal"
|
|
case GreaterThan:
|
|
return "greater-than"
|
|
case LessThanEquals:
|
|
return "less-than-or-equal"
|
|
case LessThan:
|
|
return "less-than"
|
|
case Comma:
|
|
return "comma"
|
|
case Hyphen:
|
|
return "hyphen"
|
|
case ExclamationMark:
|
|
return "exclamation-mark"
|
|
case OpenParen:
|
|
return "open-parentheses"
|
|
case CloseParen:
|
|
return "close-parentheses"
|
|
case OpenBracket:
|
|
return "open-bracket"
|
|
case CloseBracket:
|
|
return "close-bracket"
|
|
default:
|
|
return "unknown"
|
|
}
|
|
}
|
|
|
|
// Token represents a single token.
|
|
type Token struct {
|
|
Type TokenType
|
|
Raw string
|
|
}
|
|
|
|
func (t *Token) String() string {
|
|
return fmt.Sprintf("{type: %s, raw: \"%s\"}",
|
|
t.Type.String(), t.Raw)
|
|
}
|
|
|
|
// Tokenizer represents a token worker.
|
|
type Tokenizer struct {
|
|
input []rune
|
|
tokens []*Token
|
|
}
|
|
|
|
// NewTokenizer initializes a new Tokenizer.
|
|
func NewTokenizer(input string) *Tokenizer {
|
|
return &Tokenizer{
|
|
input: []rune(input),
|
|
tokens: make([]*Token, 0),
|
|
}
|
|
}
|