chore: import upstream snapshot with attribution

This commit is contained in:
wehub-resource-sync
2026-07-13 12:32:14 +08:00
commit c581336356
61 changed files with 5678 additions and 0 deletions
+31
View File
@@ -0,0 +1,31 @@
name: Go
on:
push:
branches: [ master ]
pull_request:
branches: [ master ]
jobs:
build:
runs-on: ubuntu-latest
strategy:
matrix:
go-version: [ '1.19', '1.20', '1.21' ]
steps:
- uses: actions/checkout@v3
- name: Set up Go ${{ matrix.go-version }}
uses: actions/setup-go@v3
with:
go-version: ${{ matrix.go-version }}
- name: Install dependencies
run: go get .
- name: Build
run: make
- name: Test
run: make test
+45
View File
@@ -0,0 +1,45 @@
# Compiled Object files, Static and Dynamic libs (Shared Objects)
*.o
*.a
*.so
# Folders
_obj
_test
# Architecture specific extensions/prefixes
*.[568vq]
[568vq].out
*.cgo1.go
*.cgo2.c
_cgo_defun.c
_cgo_gotypes.go
_cgo_export.*
_testmain.go
*.exe
*.test
*.prof
# Output of the go coverage tool, specifically when used with LiteIDE
*.out
# Project-local glide cache, RE: https://github.com/Masterminds/glide/issues/736
.glide/
# Compiled binaries
fsql
debug
main
dist/
# Allow cmd/fsql
!cmd/fsql
# Editor workspace directories
.vscode
# Coverage output
coverage.txt
+21
View File
@@ -0,0 +1,21 @@
The MIT License (MIT)
Copyright (c) 2017 Kashav Madan
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
+105
View File
@@ -0,0 +1,105 @@
PREFIX ?= $(shell pwd)
NAME = fsql
PKG = github.com/kashav/$(NAME)
MAIN = $(PKG)/cmd/$(NAME)
DIST_DIR := ${PREFIX}/dist
DIST_DIRS := find . -type d | sed "s|^\./||" | grep -v \\. | tr '\n' '\0' | xargs -0 -I '{}'
SRCS := $(shell find . -type f -name '*.go')
PKGS := $(shell go list ./...)
VERSION := $(shell cat VERSION)
GITCOMMIT := $(shell git rev-parse --short HEAD)
ifneq ($(shell git status --porcelain --untracked-files=no),)
GITCOMMIT := $(GITCOMMIT)-dirty
endif
LDFLAGS := ${LDFLAGS} \
-X $(PKG)/meta.GITCOMMIT=${GITCOMMIT} \
-X $(PKG)/meta.VERSION=${VERSION}
.PHONY: all
all: $(NAME)
$(NAME): $(SRCS) VERSION
@echo "+ $@"
@go build -ldflags "${LDFLAGS}" -o $(NAME) -v $(MAIN)
.PHONY: install
install:
@echo "+ $@"
@go install $(PKGS)
.PHONY: get-tools
get-tools:
@echo "+ $@"
@go get -u -v golang.org/x/lint/golint
.PHONY: clean
clean:
@echo "+ $@"
$(RM) $(NAME)
$(RM) -r $(DIST_DIR)
.PHONY: fmt
fmt:
@echo "+ $@"
@test -z "$$(gofmt -s -l . 2>&1 | tee /dev/stderr)" || \
(echo >&2 "+ please format Go code with 'gofmt -s', or use 'make fmt-save'" && false)
.PHONY: fmt-save
fmt-save:
@echo "+ $@"
@gofmt -s -l . 2>&1 | xargs gofmt -s -l -w
.PHONY: vet
vet:
@echo "+ $@"
@go vet $(PKGS)
.PHONY: lint
lint:
@echo "+ $@"
$(if $(shell which golint || echo ''),, \
$(error Please install golint: `make get-tools`))
@test -z "$$(golint ./... 2>&1 | grep -v mock/ | tee /dev/stderr)"
.PHONY: test
test:
@echo "+ $@"
@go test -race $(PKGS)
.PHONY: coverage
coverage:
@echo "+ $@"
@for pkg in $(PKGS); do \
go test -test.short -race -coverprofile="../../../$$pkg/coverage.txt" $${pkg} || exit 1; \
done
.PHONY: bootstrap-dist
bootstrap-dist:
@echo "+ $@"
@go get -u -v github.com/franciscocpg/gox
.PHONY: build-all
build-all: $(SRCS) VERSION
@echo "+ $@"
@gox -verbose \
-ldflags "${LDFLAGS}" \
-os="darwin freebsd netbsd openbsd linux solaris windows" \
-arch="386 amd64 arm arm64" \
-osarch="!darwin/386 !darwin/arm" \
-output="$(DIST_DIR)/$(NAME)-{{.OS}}-{{.Arch}}/{{.Dir}}" $(MAIN)
.PHONY: dist
dist: clean build-all
@echo "+ $@"
@cd $(DIST_DIR) && \
$(DIST_DIRS) cp ../LICENSE {} && \
$(DIST_DIRS) cp ../README.md {} && \
$(DIST_DIRS) tar -zcf fsql-${VERSION}-{}.tar.gz {} && \
$(DIST_DIRS) zip -r -q fsql-${VERSION}-{}.zip {} && \
$(DIST_DIRS) rm -rf {} && \
cd ..
+341
View File
@@ -0,0 +1,341 @@
# fsql [![Go](https://github.com/kashav/fsql/actions/workflows/go.yml/badge.svg)](https://github.com/kashav/fsql/actions/workflows/go.yml)
>Search through your filesystem with SQL-esque queries.
## Contents
- [Demo](#demo)
- [Installation](#installation)
- [Usage](#usage)
- [Query Syntax](#query-syntax)
- [Examples](#usage-examples)
- [Contribute](#contribute)
- [License](#license)
## Demo
[![fsql.gif](./media/fsql.gif)](https://asciinema.org/a/120534)
## Installation
#### Binaries
[View latest release](https://github.com/kashav/fsql/releases/latest).
#### Via Go
```sh
$ go get -u -v github.com/kashav/fsql/...
$ which fsql
$GOPATH/bin/fsql
```
#### Via Homebrew
```sh
$ brew install fsql
$ which fsql
/usr/local/bin/fsql
```
#### Build manually
```sh
$ git clone https://github.com/kashav/fsql.git $GOPATH/src/github.com/kashav/fsql
$ cd $_ # $GOPATH/src/github.com/kashav/fsql
$ make
$ ./fsql
```
## Usage
fsql expects a single query via stdin. You may also choose to use fsql in interactive mode.
View the usage dialogue with the `-help` flag.
```sh
$ fsql -help
usage: fsql [options] [query]
-v print version and exit (shorthand)
-version
print version and exit
```
## Query syntax
In general, each query requires a `SELECT` clause (to specify which attributes will be shown), a `FROM` clause (to specify which directories to search), and a `WHERE` clause (to specify conditions to test against).
```console
>>> SELECT attribute, ... FROM source, ... WHERE condition;
```
You may choose to omit the `SELECT` and `WHERE` clause.
If you're providing your query via stdin, quotes are **not** required, however you'll have to escape _reserved_ characters (e.g. `*`, `<`, `>`, etc).
### Attribute
Currently supported attributes include `name`, `size`, `time`, `hash`, `mode`.
Use `all` or `*` to choose all; if no attribute is provided, this is chosen by default.
**Examples**:
Each group features a set of equivalent clauses.
```console
>>> SELECT name, size, time ...
>>> name, size, time ...
```
```console
>>> SELECT all FROM ...
>>> all FROM ...
>>> FROM ...
```
### Source
Each source should be a relative or absolute path to a directory on your machine.
Source paths may include environment variables (e.g. `$GOPATH`) or tildes (`~`). Use a hyphen (`-`) to exclude a directory. Source paths also support usage of [glob patterns](https://en.wikipedia.org/wiki/Glob_(programming)).
In the case that a directory begins with a hyphen (e.g. `-foo`), use the following to include it as a source:
```console
>>> ... FROM ./-foo ...
```
**Examples**:
```console
>>> ... FROM . ...
```
```console
>>> ... FROM ~/Desktop, ./*/**.go ...
```
```console
>>> ... FROM $GOPATH, -.git/ ...
```
### Condition
#### Condition syntax
A single condition is made up of 3 parts: an attribute, an operator, and a value.
- **Attribute**:
A valid attribute is any of the following: `name`, `size`, `mode`, `time`.
- **Operator**:
Each attribute has a set of associated operators.
- `name`:
| Operator | Description |
| :---: | --- |
| `=` | String equality |
| `<>` / `!=` | Synonymous to using `"NOT ... = ..."` |
| `IN` | Basic list inclusion |
| `LIKE` | Simple pattern matching. Use `%` to match zero, one, or multiple characters. Check that a string begins with a value: `<value>%`, ends with a value: `%<value>`, or contains a value: `%<value>%`. |
| `RLIKE` | Pattern matching with regular expressions. |
- `size` / `time`:
- All basic algebraic operators: `>`, `>=`, `<`, `<=`, `=`, and `<>` / `!=`.
- `hash`:
- `=` or `<>` / `!=`
- `mode`:
- `IS`
- **Value**:
If the value contains spaces, wrap the value in quotes (either single or double) or backticks.
The default unit for `size` is bytes.
The default format for `time` is `MMM DD YYYY HH MM` (e.g. `"Jan 02 2006 15 04"`).
Use `mode` to test if a file is regular (`IS REG`) or if it's a directory (`IS DIR`).
Use `hash` to compute and/or compare the hash value of a file. The default algorithm is `SHA1`
#### Conjunction / Disjunction
Use `AND` / `OR` to join conditions. Note that precedence is assigned based on order of appearance.
This means `WHERE a AND b OR c` is **not** the same as `WHERE c OR b AND a`. Use parentheses to get around this behaviour, i.e. `WHERE a AND b OR c` **is** the same as `WHERE c OR (b AND a)`.
**Examples**:
```console
>>> ... WHERE name = main.go OR size = 5 ...
```
```console
>>> ... WHERE name = main.go AND size > 20 ...
```
#### Negation
Use `NOT` to negate a condition. This keyword **must** precede the condition (e.g. `... WHERE NOT a ...`).
Note that negating parenthesized conditions is currently not supported. However, this can easily be resolved by applying [De Morgan's laws](https://en.wikipedia.org/wiki/De_Morgan%27s_laws) to your query. For example, `... WHERE NOT (a AND b) ...` is _logically equivalent_ to `... WHERE NOT a OR NOT b ...` (the latter is actually more optimal, due to [lazy evaluation](https://en.wikipedia.org/wiki/Lazy_evaluation)).
**Examples**:
```console
>>> ... WHERE NOT name = main.go ...
```
### Attribute Modifiers
Attribute modifiers are used to specify how input and output values should be processed. These functions are applied directly to attributes in the `SELECT` and `WHERE` clauses.
The table below lists currently-supported modifiers. Note that the first parameter to `FORMAT` is always the attribute name.
| Attribute | Modifier | Supported in `SELECT` | Supported in `WHERE` |
| :---: | --- | :---: | :---: |
| `hash` | `SHA1(, n)` | ✔️ | ✔️ |
| `name` | `UPPER` (synonymous to `FORMAT(, UPPER)`) | ✔️ | ✔️ |
| | `LOWER` (synonymous to `FORMAT(, LOWER)`) | ✔️ | ✔️ |
| | `FULLPATH` | ✔️ | |
| | `SHORTPATH` | ✔️ | |
| `size` | `FORMAT(, unit)` | ✔️ | ✔️ |
| `time` | `FORMAT(, layout)` | ✔️ | ✔️ |
- **`n`**:
Specify the length of the hash value. Use a negative integer or `ALL` to display all digits.
- **`unit`**:
Specify the size unit. One of: `B` (byte), `KB` (kilobyte), `MB` (megabyte), or `GB` (gigabyte).
- **`layout`**:
Specify the time layout. One of: [`ISO`](https://en.wikipedia.org/wiki/ISO_8601), [`UNIX`](https://en.wikipedia.org/wiki/Unix_time), or [custom](https://golang.org/pkg/time/#Time.Format). Custom layouts must be provided in reference to the following date: `Mon Jan 2 15:04:05 -0700 MST 2006`.
**Examples**:
```console
>>> SELECT SHA1(hash, 20) ...
```
```console
>>> ... WHERE UPPER(name) ...
```
```console
>>> SELECT FORMAT(size, MB) ...
```
```console
>>> ... WHERE FORMAT(time, "Mon Jan 2 2006 15:04:05") ...
```
### Subqueries
Subqueries allow for more complex condition statements. These queries are recursively evaluated while parsing. SELECTing multiple attributes in a subquery is not currently supported; if more than one attribute (or `all`) is provided, only the first attribute is used.
Support for referencing superqueries is not yet implemented, see [#4](https://github.com/kashav/fsql/issues/4) if you'd like to help with this.
**Examples**:
```console
>>> ... WHERE name IN (SELECT name FROM ../foo) ...
```
## Usage Examples
List all attributes of each directory in your home directory (note the escaped `*`):
```console
$ fsql SELECT \* FROM ~ WHERE mode IS DIR
```
List the names of all files in the Desktop and Downloads directory that contain `csc` in the name:
```console
$ fsql "SELECT name FROM ~/Desktop, ~/Downloads WHERE name LIKE %csc%"
```
List all files in the current directory that are also present in some other directory:
```console
$ fsql
>>> SELECT all FROM . WHERE name IN (
... SELECT name FROM ~/Desktop/files.bak/
... );
```
Passing queries via stdin without quotes is a bit of a pain, hopefully the next examples highlight that, my suggestion is to use interactive mode or wrap the query in quotes if you're doing anything with subqueries or attribute modifiers.
List all files named `main.go` in `$GOPATH` which are larger than 10.5 kilobytes or smaller than 100 bytes:
```console
$ fsql SELECT all FROM $GOPATH WHERE name = main.go AND \(FORMAT\(size, KB\) \>= 10.5 OR size \< 100\)
$ fsql "SELECT all FROM $GOPATH WHERE name = main.go AND (FORMAT(size, KB) >= 10.5 OR size < 100)"
$ fsql
>>> SELECT
... all
... FROM
... $GOPATH
... WHERE
... name = main.go
... AND (
... FORMAT(size, KB) >= 10.5
... OR size < 100
... )
... ;
```
List the name, size, and modification time of JavaScript files in the current directory that were modified after April 1st 2017:
```console
$ fsql SELECT UPPER\(name\), FORMAT\(size, KB\), FORMAT\(time, ISO\) FROM . WHERE name LIKE %.js AND time \> \'Apr 01 2017 00 00\'
$ fsql "SELECT UPPER(name), FORMAT(size, KB), FORMAT(time, ISO) FROM . WHERE name LIKE %.js AND time > 'Apr 01 2017 00 00'"
$ fsql
>>> SELECT
... UPPER(name),
... FORMAT(size, KB),
... FORMAT(time, ISO)
... FROM
... .
... WHERE
... name LIKE %.js
... AND time > 'Apr 01 2017 00 00'
... ;
```
## Contribute
This project is completely open source, feel free to [open an issue](https://github.com/kashav/fsql/issues) or [submit a pull request](https://github.com/kashav/fsql/pulls).
Before submitting code, please ensure that tests are passing and the linter is happy. The following commands may be of use, refer to the [Makefile](./Makefile) to see what they do.
```sh
$ make install \
get-tools
$ make fmt \
vet \
lint
$ make test \
coverage
$ make bootstrap-dist \
dist
```
## License
fsql source code is available under the [MIT license](./LICENSE).
+7
View File
@@ -0,0 +1,7 @@
# WeHub 来源说明
- 原始项目:`kashav/fsql`
- 原始仓库:https://github.com/kashav/fsql
- 导入方式:上游默认分支的最新快照
- 原作者、版权和许可证信息以原始仓库及本仓库 LICENSE 为准
- 本文件仅用于记录来源,不代表 WeHub 是原项目作者
+1
View File
@@ -0,0 +1 @@
0.5.2
+53
View File
@@ -0,0 +1,53 @@
package main
import (
"flag"
"fmt"
"log"
"os"
"strings"
"github.com/kashav/fsql"
"github.com/kashav/fsql/meta"
"github.com/kashav/fsql/terminal"
)
var options struct {
version bool
}
func readInput() string {
if len(flag.Args()) > 1 {
return strings.Join(flag.Args(), " ")
}
return flag.Args()[0]
}
func main() {
flag.Usage = func() {
fmt.Printf("usage: %s [options] [query]\n", os.Args[0])
flag.PrintDefaults()
}
flag.BoolVar(&options.version, "version", false, "print version and exit")
flag.BoolVar(&options.version, "v", false,
"print version and exit (shorthand)")
flag.Parse()
if options.version {
fmt.Printf("%s\n", meta.Meta())
os.Exit(0)
}
if len(flag.Args()) == 0 {
if err := terminal.Start(); err != nil {
log.Fatal(err.Error())
}
os.Exit(0)
}
if err := fsql.Run(readInput()); err != nil {
log.Fatal(err.Error())
}
}
+150
View File
@@ -0,0 +1,150 @@
package evaluate
import (
"fmt"
"regexp"
"strings"
"time"
"github.com/kashav/fsql/tokenizer"
"github.com/kashav/fsql/transform"
)
// cmpAlpha performs alphabetic comparison on a and b.
func cmpAlpha(o *Opts, a, b interface{}) (result bool, err error) {
switch o.Operator {
case tokenizer.Equals:
result = a.(string) == b.(string)
case tokenizer.NotEquals:
result = a.(string) != b.(string)
case tokenizer.Like:
aStr, bStr := a.(string), b.(string)
if strings.HasPrefix(bStr, "%") && strings.HasSuffix(bStr, "%") {
result = strings.Contains(aStr, bStr[1:len(bStr)-1])
} else if strings.HasPrefix(bStr, "%") {
result = strings.HasSuffix(aStr, bStr[1:])
} else if strings.HasSuffix(bStr, "%") {
result = strings.HasPrefix(aStr, bStr[:len(bStr)-1])
} else {
result = strings.Contains(aStr, bStr)
}
case tokenizer.RLike:
result = regexp.MustCompile(b.(string)).MatchString(a.(string))
case tokenizer.In:
switch t := b.(type) {
case map[interface{}]bool:
if _, ok := t[a.(string)]; ok {
result = true
}
case []string:
for _, el := range t {
if a.(string) == el {
result = true
}
}
case string:
for _, el := range strings.Split(t, ",") {
if a.(string) == el {
result = true
}
}
}
default:
err = &ErrUnsupportedOperator{o.Attribute, o.Operator}
}
return result, err
}
// cmpNumeric performs numeric comparison on a and b.
func cmpNumeric(o *Opts, a, b interface{}) (result bool, err error) {
switch o.Operator {
case tokenizer.Equals:
result = a.(int64) == b.(int64)
case tokenizer.NotEquals:
result = a.(int64) != b.(int64)
case tokenizer.GreaterThanEquals:
result = a.(int64) >= b.(int64)
case tokenizer.GreaterThan:
result = a.(int64) > b.(int64)
case tokenizer.LessThanEquals:
result = a.(int64) <= b.(int64)
case tokenizer.LessThan:
result = a.(int64) < b.(int64)
case tokenizer.In:
if _, ok := b.(map[interface{}]bool)[a.(int64)]; ok {
result = true
}
default:
err = &ErrUnsupportedOperator{o.Attribute, o.Operator}
}
return result, err
}
// cmpTime performs time comparison on a and b.
func cmpTime(o *Opts, a, b interface{}) (result bool, err error) {
switch o.Operator {
case tokenizer.Equals:
result = a.(time.Time).Equal(b.(time.Time))
case tokenizer.NotEquals:
result = !a.(time.Time).Equal(b.(time.Time))
case tokenizer.GreaterThanEquals:
result = a.(time.Time).After(b.(time.Time)) || a.(time.Time).Equal(b.(time.Time))
case tokenizer.GreaterThan:
result = a.(time.Time).After(b.(time.Time))
case tokenizer.LessThanEquals:
result = a.(time.Time).Before(b.(time.Time)) || a.(time.Time).Equal(b.(time.Time))
case tokenizer.LessThan:
result = a.(time.Time).Before(b.(time.Time))
case tokenizer.In:
if _, ok := b.(map[interface{}]bool)[a.(time.Time)]; ok {
result = true
}
default:
err = &ErrUnsupportedOperator{o.Attribute, o.Operator}
}
return result, err
}
// cmpMode performs mode comparison with info and typ.
func cmpMode(o *Opts) (result bool, err error) {
if o.Operator != tokenizer.Is {
return false, &ErrUnsupportedOperator{o.Attribute, o.Operator}
}
switch strings.ToUpper(o.Value.(string)) {
case "DIR":
result = o.File.Mode().IsDir()
case "REG":
result = o.File.Mode().IsRegular()
default:
result = false
}
return result, err
}
// cmpHash computes the hash of the current file and compares it with the
// provided value.
func cmpHash(o *Opts) (result bool, err error) {
hashType := "SHA1"
if len(o.Modifiers) > 0 {
hashType = o.Modifiers[0].Name
}
hashFunc := transform.FindHash(hashType)
if hashFunc == nil {
return false, fmt.Errorf("unexpected hash algorithm %s", hashType)
}
h, err := transform.ComputeHash(o.File, o.Path, hashFunc())
if err != nil {
return false, err
}
switch o.Operator {
case tokenizer.Equals:
result = h == o.Value
case tokenizer.NotEquals:
result = h != o.Value
default:
err = &ErrUnsupportedOperator{o.Attribute, o.Operator}
}
return result, err
}
+423
View File
@@ -0,0 +1,423 @@
package evaluate
import (
"os"
"reflect"
"testing"
"time"
"github.com/kashav/fsql/tokenizer"
)
func TestCmpAlpha(t *testing.T) {
type Input struct {
o Opts
a, b interface{}
}
type Expected struct {
result bool
err error
}
type Case struct {
input Input
expected Expected
}
// TODO: Test for errors.
cases := []Case{
{
input: Input{o: Opts{Operator: tokenizer.Equals}, a: "a", b: "a"},
expected: Expected{result: true, err: nil},
},
{
input: Input{o: Opts{Operator: tokenizer.Equals}, a: "a", b: "b"},
expected: Expected{result: false, err: nil},
},
{
input: Input{o: Opts{Operator: tokenizer.Equals}, a: "a", b: "A"},
expected: Expected{result: false, err: nil},
},
{
input: Input{o: Opts{Operator: tokenizer.NotEquals}, a: "a", b: "a"},
expected: Expected{result: false, err: nil},
},
{
input: Input{o: Opts{Operator: tokenizer.NotEquals}, a: "a", b: "b"},
expected: Expected{result: true, err: nil},
},
{
input: Input{o: Opts{Operator: tokenizer.NotEquals}, a: "a", b: "A"},
expected: Expected{result: true, err: nil},
},
{
input: Input{o: Opts{Operator: tokenizer.Like}, a: "abc", b: "%a%"},
expected: Expected{result: true, err: nil},
},
{
input: Input{o: Opts{Operator: tokenizer.Like}, a: "aaa", b: "%b%"},
expected: Expected{result: false, err: nil},
},
{
input: Input{o: Opts{Operator: tokenizer.Like}, a: "aaa", b: "%a"},
expected: Expected{result: true, err: nil},
},
{
input: Input{o: Opts{Operator: tokenizer.Like}, a: "abc", b: "%a"},
expected: Expected{result: false, err: nil},
},
{
input: Input{o: Opts{Operator: tokenizer.Like}, a: "abc", b: "a%"},
expected: Expected{result: true, err: nil},
},
{
input: Input{o: Opts{Operator: tokenizer.Like}, a: "cba", b: "a%"},
expected: Expected{result: false, err: nil},
},
{
input: Input{o: Opts{Operator: tokenizer.Like}, a: "a", b: "a"},
expected: Expected{result: true, err: nil},
},
{
input: Input{o: Opts{Operator: tokenizer.Like}, a: "a", b: "b"},
expected: Expected{result: false, err: nil},
},
{
input: Input{o: Opts{Operator: tokenizer.RLike}, a: "a", b: ".*a.*"},
expected: Expected{result: true, err: nil},
},
{
input: Input{o: Opts{Operator: tokenizer.RLike}, a: "a", b: "^$"},
expected: Expected{result: false, err: nil},
},
{
input: Input{o: Opts{Operator: tokenizer.RLike}, a: "", b: "^$"},
expected: Expected{result: true, err: nil},
},
{
input: Input{o: Opts{Operator: tokenizer.RLike}, a: "...", b: "[\\.]{3}"},
expected: Expected{result: true, err: nil},
},
{
input: Input{o: Opts{Operator: tokenizer.RLike}, a: "aaa", b: "\\s+"},
expected: Expected{result: false, err: nil},
},
{
input: Input{o: Opts{Operator: tokenizer.In}, a: "a", b: map[interface{}]bool{"a": true}},
expected: Expected{result: true, err: nil},
},
{
input: Input{o: Opts{Operator: tokenizer.In}, a: "a", b: map[interface{}]bool{"a": false}},
expected: Expected{result: true, err: nil},
},
{
input: Input{o: Opts{Operator: tokenizer.In}, a: "a", b: map[interface{}]bool{"b": true}},
expected: Expected{result: false, err: nil},
},
{
input: Input{o: Opts{Operator: tokenizer.In}, a: "a", b: map[interface{}]bool{}},
expected: Expected{result: false, err: nil},
},
}
for _, c := range cases {
actual, err := cmpAlpha(&c.input.o, c.input.a, c.input.b)
if c.expected.err == nil {
if err != nil {
t.Fatalf("\nExpected no error\n Got %v", err)
}
if !reflect.DeepEqual(c.expected.result, actual) {
t.Fatalf("%v, %v, %v\nExpected: %v\n Got: %v",
c.input.o.Operator, c.input.a, c.input.b, c.expected.result, actual)
}
} else if !reflect.DeepEqual(c.expected.err, err) {
t.Fatalf("\nExpected %v\n Got %v", c.expected.err, err)
}
}
}
func TestCmpNumeric(t *testing.T) {
type Input struct {
o Opts
a, b interface{}
}
type Expected struct {
result bool
err error
}
type Case struct {
input Input
expected Expected
}
// TODO: Test for errors.
cases := []Case{
{
input: Input{o: Opts{Operator: tokenizer.Equals}, a: int64(1), b: int64(1)},
expected: Expected{result: true, err: nil},
},
{
input: Input{o: Opts{Operator: tokenizer.Equals}, a: int64(1), b: int64(2)},
expected: Expected{result: false, err: nil},
},
{
input: Input{o: Opts{Operator: tokenizer.NotEquals}, a: int64(1), b: int64(1)},
expected: Expected{result: false, err: nil},
},
{
input: Input{o: Opts{Operator: tokenizer.NotEquals}, a: int64(1), b: int64(2)},
expected: Expected{result: true, err: nil},
},
{
input: Input{o: Opts{Operator: tokenizer.GreaterThanEquals}, a: int64(1), b: int64(1)},
expected: Expected{result: true, err: nil},
},
{
input: Input{o: Opts{Operator: tokenizer.GreaterThanEquals}, a: int64(2), b: int64(1)},
expected: Expected{result: true, err: nil},
},
{
input: Input{o: Opts{Operator: tokenizer.GreaterThanEquals}, a: int64(1), b: int64(2)},
expected: Expected{result: false, err: nil},
},
{
input: Input{o: Opts{Operator: tokenizer.GreaterThan}, a: int64(1), b: int64(1)},
expected: Expected{result: false, err: nil},
},
{
input: Input{o: Opts{Operator: tokenizer.GreaterThan}, a: int64(2), b: int64(1)},
expected: Expected{result: true, err: nil},
},
{
input: Input{o: Opts{Operator: tokenizer.GreaterThan}, a: int64(1), b: int64(2)},
expected: Expected{result: false, err: nil},
},
{
input: Input{o: Opts{Operator: tokenizer.LessThanEquals}, a: int64(1), b: int64(1)},
expected: Expected{result: true, err: nil},
},
{
input: Input{o: Opts{Operator: tokenizer.LessThanEquals}, a: int64(2), b: int64(1)},
expected: Expected{result: false, err: nil},
},
{
input: Input{o: Opts{Operator: tokenizer.LessThanEquals}, a: int64(1), b: int64(2)},
expected: Expected{result: true, err: nil},
},
{
input: Input{o: Opts{Operator: tokenizer.LessThan}, a: int64(1), b: int64(1)},
expected: Expected{result: false, err: nil},
},
{
input: Input{o: Opts{Operator: tokenizer.LessThan}, a: int64(2), b: int64(1)},
expected: Expected{result: false, err: nil},
},
{
input: Input{o: Opts{Operator: tokenizer.LessThan}, a: int64(1), b: int64(2)},
expected: Expected{result: true, err: nil},
},
{
input: Input{o: Opts{Operator: tokenizer.In}, a: int64(1), b: map[interface{}]bool{int64(1): true}},
expected: Expected{result: true, err: nil},
},
{
input: Input{o: Opts{Operator: tokenizer.In}, a: int64(1), b: map[interface{}]bool{int64(1): false}},
expected: Expected{result: true, err: nil},
},
{
input: Input{o: Opts{Operator: tokenizer.In}, a: int64(1), b: map[interface{}]bool{int64(2): true}},
expected: Expected{result: false, err: nil},
},
{
input: Input{o: Opts{Operator: tokenizer.In}, a: int64(1), b: map[interface{}]bool{}},
expected: Expected{result: false, err: nil},
},
}
for _, c := range cases {
actual, err := cmpNumeric(&c.input.o, c.input.a, c.input.b)
if c.expected.err == nil {
if err != nil {
t.Fatalf("\nExpected no error\n Got %v", err)
}
if !reflect.DeepEqual(c.expected.result, actual) {
t.Fatalf("%v, %v, %v\nExpected: %v\n Got: %v",
c.input.o.Operator, c.input.a, c.input.b, c.expected.result, actual)
}
} else if !reflect.DeepEqual(c.expected.err, err) {
t.Fatalf("\nExpected %v\n Got %v", c.expected.err, err)
}
}
}
func TestCmpTime(t *testing.T) {
type Input struct {
o Opts
a, b interface{}
}
type Expected struct {
result bool
err error
}
type Case struct {
input Input
expected Expected
}
// TODO: Test for errors.
cases := []Case{
{
input: Input{
o: Opts{Operator: tokenizer.Equals},
a: time.Now().Round(time.Minute),
b: time.Now().Round(time.Minute),
},
expected: Expected{result: true, err: nil},
},
{
input: Input{
o: Opts{Operator: tokenizer.Equals},
a: time.Now().Add(time.Hour),
b: time.Now().Add(-1 * time.Hour),
},
expected: Expected{result: false, err: nil},
},
{
input: Input{
o: Opts{Operator: tokenizer.NotEquals},
a: time.Now().Round(time.Minute),
b: time.Now().Round(time.Minute),
},
expected: Expected{result: false, err: nil},
},
{
input: Input{
o: Opts{Operator: tokenizer.NotEquals},
a: time.Now().Add(time.Hour),
b: time.Now().Add(-1 * time.Hour),
},
expected: Expected{result: true, err: nil},
},
{
input: Input{
o: Opts{Operator: tokenizer.In},
a: time.Now().Round(time.Minute),
b: map[interface{}]bool{time.Now().Round(time.Minute): true},
},
expected: Expected{result: true, err: nil},
},
{
input: Input{
o: Opts{Operator: tokenizer.In},
a: time.Now().Round(time.Minute),
b: map[interface{}]bool{time.Now().Add(time.Hour): true},
},
expected: Expected{result: false, err: nil},
},
}
for _, c := range cases {
actual, err := cmpTime(&c.input.o, c.input.a, c.input.b)
if c.expected.err == nil {
if err != nil {
t.Fatalf("\nExpected no error\n Got %v", err)
}
if !reflect.DeepEqual(c.expected.result, actual) {
t.Fatalf("%v, %v, %v\nExpected: %v\n Got: %v",
c.input.o.Operator, c.input.a, c.input.b, c.expected.result, actual)
}
} else if !reflect.DeepEqual(c.expected.err, err) {
t.Fatalf("\nExpected %v\n Got %v", c.expected.err, err)
}
}
}
func TestCmpMode(t *testing.T) {
type Input struct {
o Opts
file os.FileInfo
typ interface{}
}
type Expected struct {
result bool
err error
}
type Case struct {
input Input
expected Expected
}
// TODO: Complete these cases.
cases := []Case{}
for _, c := range cases {
actual, err := cmpMode(&c.input.o)
if c.expected.err == nil {
if err != nil {
t.Fatalf("\nExpected no error\n Got %v", err)
}
if !reflect.DeepEqual(c.expected.result, actual) {
t.Fatalf("%v, %v, %v\nExpected: %v\n Got: %v",
c.input.o.Operator, c.input.file, c.input.typ, c.expected.result, actual)
}
} else if !reflect.DeepEqual(c.expected.err, err) {
t.Fatalf("\nExpected %v\n Got %v", c.expected.err, err)
}
}
}
func TestCmpHash(t *testing.T) {
type Input struct {
o Opts
file os.FileInfo
typ interface{}
}
type Expected struct {
result bool
err error
}
type Case struct {
input Input
expected Expected
}
// TODO: Complete these cases.
cases := []Case{}
for _, c := range cases {
actual, err := cmpHash(&c.input.o)
if c.expected.err == nil {
if err != nil {
t.Fatalf("\nExpected no error\n Got %v", err)
}
if !reflect.DeepEqual(c.expected.result, actual) {
t.Fatalf("%v, %v, %v\nExpected: %v\n Got: %v",
c.input.o.Operator, c.input.file, c.input.typ, c.expected.result, actual)
}
} else if !reflect.DeepEqual(c.expected.err, err) {
t.Fatalf("\nExpected %v\n Got %v", c.expected.err, err)
}
}
}
+38
View File
@@ -0,0 +1,38 @@
package evaluate
import (
"fmt"
"github.com/kashav/fsql/tokenizer"
)
// ErrUnsupportedAttribute represents an unsupported attribute error.
type ErrUnsupportedAttribute struct {
Attribute string
}
func (e *ErrUnsupportedAttribute) Error() string {
return fmt.Sprintf("unsupported attribute %s", e.Attribute)
}
// ErrUnsupportedType represents an unsupported type error.
type ErrUnsupportedType struct {
Attribute string
Value interface{}
}
func (e *ErrUnsupportedType) Error() string {
return fmt.Sprintf("unsupported type %T for for attribute %s", e.Value,
e.Attribute)
}
// ErrUnsupportedOperator represents an unsupported operator error.
type ErrUnsupportedOperator struct {
Attribute string
Operator tokenizer.TokenType
}
func (e *ErrUnsupportedOperator) Error() string {
return fmt.Sprintf("unsupported operator %s for attribute %s",
e.Operator.String(), e.Attribute)
}
+3
View File
@@ -0,0 +1,3 @@
package evaluate
// TODO
+104
View File
@@ -0,0 +1,104 @@
package evaluate
import (
"os"
"strconv"
"time"
"github.com/kashav/fsql/tokenizer"
)
// Opts represents a set of options used in the evaluate functions.
type Opts struct {
Path string
File os.FileInfo
Attribute string
Modifiers []Modifier
Operator tokenizer.TokenType
Value interface{}
}
// Modifier represents an attribute modifier.
type Modifier struct {
Name string
Arguments []string
}
// Evaluate runs the respective evaluate function for the provided options.
func Evaluate(o *Opts) (bool, error) {
switch o.Attribute {
case "name":
return evaluateName(o)
case "size":
return evaluateSize(o)
case "time":
return evaluateTime(o)
case "mode":
return evaluateMode(o)
case "hash":
return evaluateHash(o)
}
return false, &ErrUnsupportedAttribute{o.Attribute}
}
// evaluateName evaluates a Condition with attribute `name`.
func evaluateName(o *Opts) (bool, error) {
var a, b interface{}
switch o.Value.(type) {
case string, []string, map[interface{}]bool:
a = o.File.Name()
b = o.Value
default:
return false, &ErrUnsupportedType{o.Attribute, o.Value}
}
return cmpAlpha(o, a, b)
}
// evaluateSize evaluates a Condition with attribute `size`.
func evaluateSize(o *Opts) (bool, error) {
var a, b interface{}
switch o.Value.(type) {
case float64:
a = o.File.Size()
b = int64(o.Value.(float64))
case map[interface{}]bool:
a = o.File.Size()
b = o.Value
case string:
size, err := strconv.ParseFloat(o.Value.(string), 10)
if err != nil {
return false, err
}
a = o.File.Size()
b = int64(size)
default:
return false, &ErrUnsupportedType{o.Attribute, o.Value}
}
return cmpNumeric(o, a, b)
}
// evaluateTime evaluates a Condition with attribute `time`.
func evaluateTime(o *Opts) (bool, error) {
var a, b interface{}
switch o.Value.(type) {
case string:
t, err := time.Parse("Jan 02 2006 15 04", o.Value.(string))
if err != nil {
return false, err
}
a = o.File.ModTime()
b = t
case map[interface{}]bool, time.Time:
a = o.File.ModTime()
b = o.Value
default:
return false, &ErrUnsupportedType{o.Attribute, o.Value}
}
return cmpTime(o, a, b)
}
// evaluateMode evaluates a Condition with attribute `mode`.
func evaluateMode(o *Opts) (bool, error) { return cmpMode(o) }
// evaluateHash evaluates a Condition with attribute `hash`.
func evaluateHash(o *Opts) (bool, error) { return cmpHash(o) }
+3
View File
@@ -0,0 +1,3 @@
package evaluate
// TODO
+55
View File
@@ -0,0 +1,55 @@
package fsql
import (
"bytes"
"fmt"
"os"
"github.com/kashav/fsql/parser"
)
// Run parses the input and executes the resultant query.
func Run(input string) error {
q, err := parser.Run(input)
if err != nil {
return err
}
// Find length of the longest name to normalize name output.
var max = 0
var results = make([]map[string]interface{}, 0)
err = q.Execute(
func(path string, info os.FileInfo, result map[string]interface{}) {
results = append(results, result)
if !q.HasAttribute("name") {
return
}
if s, ok := result["name"].(string); ok && len(s) > max {
max = len(s)
}
},
)
if err != nil {
return err
}
for _, result := range results {
var buf bytes.Buffer
for j, attribute := range q.Attributes {
// If the current attribute is "name", pad the output string by `max`
// spaces.
format := "%v"
if attribute == "name" {
format = fmt.Sprintf("%%-%ds", max)
}
buf.WriteString(fmt.Sprintf(format, result[attribute]))
if j != len(q.Attributes)-1 {
buf.WriteString("\t")
}
}
fmt.Printf("%s\n", buf.String())
}
return nil
}
+421
View File
@@ -0,0 +1,421 @@
package fsql
import (
"bytes"
"crypto/sha1"
"encoding/hex"
"fmt"
"io"
"os"
"path/filepath"
"reflect"
"strings"
"testing"
"time"
)
var files = map[string]*os.FileInfo{}
func TestRun_All(t *testing.T) {
type Case struct {
query string
expected string
}
cases := []Case{
{
query: "SELECT all FROM ./testdata WHERE name = foo",
expected: fmt.Sprintf("%s\t-------\tfoo\n",
strings.Join(GetAttrs("foo", "mode", "size", "time"), "\t")),
},
{
query: "SELECT all FROM ./testdata WHERE name LIKE waldo",
expected: fmt.Sprintf("%s\twaldo\n",
strings.Join(GetAttrs("foo/quuz/waldo", "mode", "size", "time", "hash"), "\t")),
},
{
query: "SELECT all FROM ./testdata WHERE FORMAT(time, 'Jan 02 2006 15:04') > 'Jan 01 2999 00:00'",
expected: "",
},
{
query: "SELECT all FROM ./testdata WHERE mode IS DIR",
expected: fmt.Sprintf(
strings.Repeat("%s\n", 8),
fmt.Sprintf(
"%s\t-------\t%-8s",
strings.Join(GetAttrs(".", "mode", "size", "time"), "\t"),
"testdata",
),
fmt.Sprintf(
"%s\t-------\t%-8s",
strings.Join(GetAttrs("bar", "mode", "size", "time"), "\t"),
"bar",
),
fmt.Sprintf(
"%s\t-------\t%-8s",
strings.Join(GetAttrs("bar/garply", "mode", "size", "time"), "\t"),
"garply",
),
fmt.Sprintf(
"%s\t-------\t%-8s",
strings.Join(GetAttrs("bar/garply/xyzzy", "mode", "size", "time"), "\t"),
"xyzzy",
),
fmt.Sprintf(
"%s\t-------\t%-8s",
strings.Join(GetAttrs("bar/garply/xyzzy/thud", "mode", "size", "time"), "\t"),
"thud",
),
fmt.Sprintf(
"%s\t-------\t%-8s",
strings.Join(GetAttrs("foo", "mode", "size", "time"), "\t"),
"foo",
),
fmt.Sprintf(
"%s\t-------\t%-8s",
strings.Join(GetAttrs("foo/quuz", "mode", "size", "time"), "\t"),
"quuz",
),
fmt.Sprintf(
"%s\t-------\t%-8s",
strings.Join(GetAttrs("foo/quuz/fred", "mode", "size", "time"), "\t"),
"fred",
),
),
},
}
for _, c := range cases {
actual := DoRun(c.query)
if !reflect.DeepEqual(c.expected, actual) {
t.Fatalf("%s\nExpected:\n%v\nGot:\n%v", c.query, c.expected, actual)
}
}
}
func TestRun_Multiple(t *testing.T) {
type Case struct {
query string
expected string
}
cases := []Case{
{
query: "SELECT name, size FROM ./testdata WHERE name = foo",
expected: fmt.Sprintf("foo\t%s\n", GetAttrs("foo", "size")[0]),
},
{
query: "SELECT size, name FROM ./testdata WHERE name = foo",
expected: fmt.Sprintf("%s\tfoo\n", GetAttrs("foo", "size")[0]),
},
{
query: "SELECT FULLPATH(name), size, time FROM ./testdata/foo",
expected: fmt.Sprintf(
strings.Repeat("%s\n", 7),
fmt.Sprintf(
"%-31s\t%s",
"testdata/foo",
strings.Join(GetAttrs("foo", "size", "time"), "\t"),
),
fmt.Sprintf(
"%-31s\t%s",
"testdata/foo/quux",
strings.Join(GetAttrs("foo/quux", "size", "time"), "\t"),
),
fmt.Sprintf(
"%-31s\t%s",
"testdata/foo/quuz",
strings.Join(GetAttrs("foo/quuz", "size", "time"), "\t"),
),
fmt.Sprintf(
"%-31s\t%s",
"testdata/foo/quuz/fred",
strings.Join(GetAttrs("foo/quuz/fred", "size", "time"), "\t"),
),
fmt.Sprintf(
"%-31s\t%s",
"testdata/foo/quuz/fred/.gitkeep",
strings.Join(GetAttrs("foo/quuz/fred/.gitkeep", "size", "time"), "\t"),
),
fmt.Sprintf(
"%-31s\t%s",
"testdata/foo/quuz/waldo",
strings.Join(GetAttrs("foo/quuz/waldo", "size", "time"), "\t"),
),
fmt.Sprintf(
"%-31s\t%s",
"testdata/foo/qux",
strings.Join(GetAttrs("foo/qux", "size", "time"), "\t"),
),
),
},
}
for _, c := range cases {
actual := DoRun(c.query)
if !reflect.DeepEqual(c.expected, actual) {
t.Fatalf("%s\nExpected:\n%v\nGot:\n%v", c.query, c.expected, actual)
}
}
}
func TestRun_Name(t *testing.T) {
type Case struct {
query string
expected string
}
cases := []Case{
{
query: "SELECT name FROM ./testdata WHERE name REGEXP ^g.*",
expected: "garply\ngrault\n",
},
{
query: "SELECT FULLPATH(name) FROM ./testdata WHERE name REGEXP ^b.*",
expected: "testdata/bar\ntestdata/baz\n",
},
{
query: "SELECT UPPER(FULLPATH(name)) FROM ./testdata WHERE mode IS DIR",
expected: fmt.Sprintf(
strings.Repeat("%-30s\n", 8),
"TESTDATA",
"TESTDATA/BAR",
"TESTDATA/BAR/GARPLY",
"TESTDATA/BAR/GARPLY/XYZZY",
"TESTDATA/BAR/GARPLY/XYZZY/THUD",
"TESTDATA/FOO",
"TESTDATA/FOO/QUUZ",
"TESTDATA/FOO/QUUZ/FRED",
),
},
}
for _, c := range cases {
actual := DoRun(c.query)
if !reflect.DeepEqual(c.expected, actual) {
t.Fatalf("%s\nExpected:\n%v\nGot:\n%v", c.query, c.expected, actual)
}
}
}
func TestRun_Size(t *testing.T) {
type Case struct {
query string
expected string
}
cases := []Case{
{
query: "SELECT size FROM ./testdata WHERE name = foo",
expected: fmt.Sprintf("%s\n", GetAttrs("foo", "size")[0]),
},
{
query: "SELECT FORMAT(size, KB) FROM ./testdata WHERE name = foo",
expected: fmt.Sprintf("%s\n", GetAttrs("foo", "size:kb")[0]),
},
{
query: "SELECT FORMAT(size, MB) FROM ./testdata WHERE name = foo",
expected: fmt.Sprintf("%s\n", GetAttrs("foo", "size:mb")[0]),
},
{
query: "SELECT FORMAT(size, GB) FROM ./testdata WHERE name = foo",
expected: fmt.Sprintf("%s\n", GetAttrs("foo", "size:gb")[0]),
},
{
query: "SELECT size FROM ./testdata WHERE name LIKE qu",
expected: fmt.Sprintf(
strings.Repeat("%s\n", 3),
GetAttrs("foo/quux", "size")[0],
GetAttrs("foo/quuz", "size")[0],
GetAttrs("foo/qux", "size")[0],
),
},
}
for _, c := range cases {
actual := DoRun(c.query)
if !reflect.DeepEqual(c.expected, actual) {
t.Fatalf("%s\nExpected:\n%v\nGot:\n%v", c.query, c.expected, actual)
}
}
}
func TestRun_Time(t *testing.T) {
type Case struct {
query string
expected string
}
cases := []Case{
{
query: "SELECT time FROM ./testdata WHERE name = baz",
expected: fmt.Sprintf("%s\n", GetAttrs("baz", "time")[0]),
},
{
query: "SELECT FORMAT(time, ISO) FROM ./testdata WHERE name = foo",
expected: fmt.Sprintf("%s\n", GetAttrs("foo", "time:iso")[0]),
},
{
query: "SELECT FORMAT(time, 2006) FROM ./testdata WHERE NOT name LIKE .%",
expected: strings.Repeat(fmt.Sprintf("%s\n", GetAttrs(".", "time:year")[0]), 14),
},
{
query: "SELECT time FROM ./testdata/foo/quuz",
expected: fmt.Sprintf(
strings.Repeat("%s\n", 4),
GetAttrs("foo/quuz", "time")[0],
GetAttrs("foo/quuz/fred", "time")[0],
GetAttrs("foo/quuz/fred/.gitkeep", "time")[0],
GetAttrs("foo/quuz/waldo", "time")[0],
),
},
}
for _, c := range cases {
actual := DoRun(c.query)
if !reflect.DeepEqual(c.expected, actual) {
t.Fatalf("%s\nExpected:\n%v\nGot:\n%v", c.query, c.expected, actual)
}
}
}
func TestRun_Mode(t *testing.T) {
type Case struct {
query string
expected string
}
cases := []Case{
{
query: "SELECT mode FROM ./testdata WHERE name = foo",
expected: fmt.Sprintf("%s\n", GetAttrs("foo", "mode")[0]),
},
{
query: "SELECT mode FROM ./testdata WHERE name = baz",
expected: fmt.Sprintf("%s\n", GetAttrs("baz", "mode")[0]),
},
{
query: "SELECT mode FROM ./testdata WHERE mode IS DIR",
expected: fmt.Sprintf(strings.Repeat("%s\n", 8),
GetAttrs("", "mode")[0],
GetAttrs("foo", "mode")[0],
GetAttrs("foo/quuz", "mode")[0],
GetAttrs("foo/quuz/fred", "mode")[0],
GetAttrs("bar", "mode")[0],
GetAttrs("bar/garply", "mode")[0],
GetAttrs("bar/garply/xyzzy", "mode")[0],
GetAttrs("bar/garply/xyzzy/thud", "mode")[0],
),
},
}
for _, c := range cases {
actual := DoRun(c.query)
if !reflect.DeepEqual(c.expected, actual) {
t.Fatalf("%s\nExpected:\n\"%v\"\nGot:\n\"%v\"", c.query, c.expected, actual)
}
}
}
func TestRun_Hash(t *testing.T) {
type Case struct {
query string
expected string
}
// TODO
cases := []Case{}
for _, c := range cases {
actual := DoRun(c.query)
if !reflect.DeepEqual(c.expected, actual) {
t.Fatalf("%s\nExpected:\n%v\nGot:\n%v", c.query, c.expected, actual)
}
}
}
func GetAttrs(path string, attrs ...string) []string {
// If the files map is empty, walk ./testdata and populate it.
if len(files) == 0 {
if err := filepath.Walk(
"./testdata",
func(path string, info os.FileInfo, err error) error {
files[filepath.Clean(path)] = &info
return nil
},
); err != nil {
return []string{}
}
}
path = filepath.Clean(fmt.Sprintf("testdata/%s", path))
file, ok := files[path]
if !ok {
return []string{}
}
result := make([]string, len(attrs))
for i, attr := range attrs {
// Hard-coding modifiers works for the time being, but we might need a more
// elegant solution when we introduce new modifiers in the future.
switch attr {
case "hash":
b, err := os.ReadFile(path)
if err != nil {
return []string{}
}
h := sha1.New()
if _, err := h.Write(b); err != nil {
return []string{}
}
result[i] = hex.EncodeToString(h.Sum(nil))[:7]
case "mode":
result[i] = (*file).Mode().String()
case "size":
result[i] = fmt.Sprintf("%d", (*file).Size())
case "size:kb", "size:mb", "size:gb":
size := (*file).Size()
switch attr[len(attr)-2:] {
case "kb":
result[i] = fmt.Sprintf("%fkb", float64(size)/(1<<10))
case "mb":
result[i] = fmt.Sprintf("%fmb", float64(size)/(1<<20))
case "gb":
result[i] = fmt.Sprintf("%fgb", float64(size)/(1<<30))
}
case "time":
result[i] = (*file).ModTime().Format(time.Stamp)
case "time:iso":
result[i] = (*file).ModTime().Format(time.RFC3339)
case "time:year":
result[i] = (*file).ModTime().Format("2006")
}
}
return result
}
// DoRun executes fsql.Run and returns the output.
func DoRun(query string) string {
stdout := os.Stdout
ch := make(chan string)
r, w, err := os.Pipe()
if err != nil {
return ""
}
os.Stdout = w
if err := Run(query); err != nil {
return ""
}
go func() {
var buf bytes.Buffer
io.Copy(&buf, r)
ch <- buf.String()
}()
w.Close()
os.Stdout = stdout
return <-ch
}
+13
View File
@@ -0,0 +1,13 @@
module github.com/kashav/fsql
go 1.21
require (
github.com/oleiade/lane v1.0.1
golang.org/x/crypto v0.17.0
)
require (
golang.org/x/sys v0.15.0 // indirect
golang.org/x/term v0.15.0 // indirect
)
+8
View File
@@ -0,0 +1,8 @@
github.com/oleiade/lane v1.0.1 h1:hXofkn7GEOubzTwNpeL9MaNy8WxolCYb9cInAIeqShU=
github.com/oleiade/lane v1.0.1/go.mod h1:IyTkraa4maLfjq/GmHR+Dxb4kCMtEGeb+qmhlrQ5Mk4=
golang.org/x/crypto v0.17.0 h1:r8bRNjWL3GshPW3gkd+RpvzWrZAwPS49OmTGZ/uhM4k=
golang.org/x/crypto v0.17.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4=
golang.org/x/sys v0.15.0 h1:h48lPFYpsTvQJZF4EKyI4aLHaev3CxivZmv7yZig9pc=
golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/term v0.15.0 h1:y/Oo/a/q3IXu26lQgl04j/gjuBDOBlx7X6Om1j2CPW4=
golang.org/x/term v0.15.0/go.mod h1:BDl952bC7+uMoWR75FIrCDx79TPU9oHkTZ9yRbYOrX0=
BIN
View File
Binary file not shown.

After

Width:  |  Height:  |  Size: 1.9 MiB

+22
View File
@@ -0,0 +1,22 @@
package meta
import "fmt"
// GITCOMMIT indicates which git hash the binary was built off of.
var GITCOMMIT string
// VERSION indicates which version of the binary is running.
var VERSION string
// Release holds the current release number, should match the value
// in $GOPATH/src/github.com/kashav/fsql/VERSION.
const Release = "0.5.2"
// Meta returns the version/commit string.
func Meta() string {
version, commit := VERSION, GITCOMMIT
if commit == "" || version == "" {
version, commit = Release, "master"
}
return fmt.Sprintf("fsql version %v, built off %v", version, commit)
}
+102
View File
@@ -0,0 +1,102 @@
package parser
import (
"strings"
"github.com/kashav/fsql/query"
"github.com/kashav/fsql/tokenizer"
)
var allAttributes = []string{"mode", "size", "time", "hash", "name"}
func isValidAttribute(attribute string) error {
for _, valid := range allAttributes {
if attribute == valid {
return nil
}
}
return &ErrUnknownToken{attribute}
}
// parseAttrs parses the list of attributes passed to the SELECT clause.
func (p *parser) parseAttrs(attributes *[]string, modifiers *map[string][]query.Modifier) error {
for {
ident := p.expect(tokenizer.Identifier)
if ident == nil {
return p.currentError()
}
if ident.Raw == "*" || ident.Raw == "all" {
*attributes = allAttributes
} else {
p.current = ident
attrModifiers := make([]query.Modifier, 0)
attribute, err := p.parseAttr(&attrModifiers)
if err != nil {
return err
}
*attributes = append(*attributes, attribute.Raw)
(*modifiers)[attribute.Raw] = attrModifiers
}
if p.expect(tokenizer.Comma) == nil {
break
}
}
return nil
}
// parseAttr recursively parses an attribute's modifiers and returns the
// associated attribute.
func (p *parser) parseAttr(modifiers *[]query.Modifier) (*tokenizer.Token, error) {
ident := p.expect(tokenizer.Identifier)
if ident == nil {
return nil, p.currentError()
}
// ident is a modifier name (e.g. `FORMAT`) iff the next token is an open
// paren, otherwise an attribute (e.g. `name`).
if token := p.expect(tokenizer.OpenParen); token == nil {
if err := isValidAttribute(ident.Raw); err != nil {
return nil, err
}
return ident, nil
}
// In the case of chained modifiers, we want to recurse and parse each
// inner modifier first. parseAttribute returns the associated attribute that
// we're looking for.
attribute, err := p.parseAttr(modifiers)
if err != nil {
return nil, err
}
if attribute == nil {
return nil, p.currentError()
}
if err := isValidAttribute(attribute.Raw); err != nil {
return nil, err
}
modifier := query.Modifier{
Name: strings.ToUpper(ident.Raw),
Arguments: make([]string, 0),
}
// Parse the modifier arguments.
for {
if token := p.expect(tokenizer.Identifier); token != nil {
modifier.Arguments = append(modifier.Arguments, token.Raw)
continue
}
if token := p.expect(tokenizer.Comma); token != nil {
continue
}
if token := p.expect(tokenizer.CloseParen); token != nil {
*modifiers = append(*modifiers, modifier)
return attribute, nil
}
}
}
+256
View File
@@ -0,0 +1,256 @@
package parser
import (
"io"
"reflect"
"testing"
"github.com/kashav/fsql/query"
"github.com/kashav/fsql/tokenizer"
)
func TestAttributeParser_ExpectCorrectAttributes(t *testing.T) {
type Expected struct {
attributes []string
err error
}
type Case struct {
input string
expected Expected
}
cases := []Case{
{
input: "name",
expected: Expected{attributes: []string{"name"}, err: nil},
},
{
input: "name, size",
expected: Expected{
attributes: []string{"name", "size"},
err: nil,
},
},
{
input: "*",
expected: Expected{attributes: allAttributes, err: nil},
},
{
input: "all",
expected: Expected{attributes: allAttributes, err: nil},
},
{
input: "format(time, iso)",
expected: Expected{attributes: []string{"time"}, err: nil},
},
{
input: "",
expected: Expected{err: io.ErrUnexpectedEOF},
},
{
input: "name,",
expected: Expected{err: io.ErrUnexpectedEOF},
},
{
input: "identifier",
expected: Expected{err: &ErrUnknownToken{"identifier"}},
},
}
for _, c := range cases {
attributes := make([]string, 0)
modifiers := make(map[string][]query.Modifier)
p := &parser{tokenizer: tokenizer.NewTokenizer(c.input)}
err := p.parseAttrs(&attributes, &modifiers)
if c.expected.err == nil {
if err != nil {
t.Fatalf("\nExpected no error\n Got %v", err)
}
if !reflect.DeepEqual(c.expected.attributes, attributes) {
t.Fatalf("\nExpected %v\n Got %v", c.expected.attributes, attributes)
}
} else if !reflect.DeepEqual(c.expected.err, err) {
t.Fatalf("\nExpected %v\n Got %v", c.expected.err, err)
}
}
}
func TestAttributeParser_ExpectCorrectModifiers(t *testing.T) {
type Expected struct {
modifiers map[string][]query.Modifier
err error
}
type Case struct {
input string
expected Expected
}
cases := []Case{
{
input: "name",
expected: Expected{
modifiers: map[string][]query.Modifier{"name": {}},
err: nil,
},
},
{
input: "upper(name)",
expected: Expected{
modifiers: map[string][]query.Modifier{
"name": {
{
Name: "UPPER",
Arguments: []string{},
},
},
},
err: nil,
},
},
{
input: "format(time, iso)",
expected: Expected{
modifiers: map[string][]query.Modifier{
"time": {
{
Name: "FORMAT",
Arguments: []string{"iso"},
},
},
},
err: nil,
},
},
{
input: "format(time, \"iso\")",
expected: Expected{
modifiers: map[string][]query.Modifier{
"time": {
{
Name: "FORMAT",
Arguments: []string{"iso"},
},
},
},
err: nil,
},
},
{
input: "lower(name), format(size, mb)",
expected: Expected{
modifiers: map[string][]query.Modifier{
"name": {
{
Name: "LOWER",
Arguments: []string{},
},
},
"size": {
{
Name: "FORMAT",
Arguments: []string{"mb"},
},
},
},
err: nil,
},
},
{
input: "format(fullpath(name), lower)",
expected: Expected{
modifiers: map[string][]query.Modifier{
"name": {
{
Name: "FULLPATH",
Arguments: []string{},
},
{
Name: "FORMAT",
Arguments: []string{"lower"},
},
},
},
err: nil,
},
},
// No function/parameter validation yet!
{
input: "foo(name)",
expected: Expected{
modifiers: map[string][]query.Modifier{
"name": {{
Name: "FOO",
Arguments: []string{},
},
},
},
err: nil,
},
},
{
input: "format(size, tb)",
expected: Expected{
modifiers: map[string][]query.Modifier{
"size": {
{
Name: "FORMAT",
Arguments: []string{"tb"},
},
},
},
err: nil,
},
},
{
input: "format(size, kb, mb)",
expected: Expected{
modifiers: map[string][]query.Modifier{
"size": {
{
Name: "FORMAT",
Arguments: []string{"kb", "mb"},
},
},
},
err: nil,
},
},
{
input: "",
expected: Expected{err: io.ErrUnexpectedEOF},
},
{
input: "lower(name),",
expected: Expected{err: io.ErrUnexpectedEOF},
},
{
input: "identifier",
expected: Expected{err: &ErrUnknownToken{"identifier"}},
},
}
for _, c := range cases {
attributes := make([]string, 0)
modifiers := make(map[string][]query.Modifier)
p := &parser{tokenizer: tokenizer.NewTokenizer(c.input)}
err := p.parseAttrs(&attributes, &modifiers)
if c.expected.err == nil {
if err != nil {
t.Fatalf("\nExpected no error\n Got %v", err)
}
if !reflect.DeepEqual(c.expected.modifiers, modifiers) {
t.Fatalf("\nExpected %v\n Got %v", c.expected.modifiers, modifiers)
}
} else if !reflect.DeepEqual(c.expected.err, err) {
t.Fatalf("\nExpected %v\n Got %v", c.expected.err, err)
}
}
}
+210
View File
@@ -0,0 +1,210 @@
package parser
import (
"errors"
"os"
"github.com/oleiade/lane"
"github.com/kashav/fsql/query"
"github.com/kashav/fsql/tokenizer"
)
// parseConditionTree parses the condition tree passed to the WHERE clause.
func (p *parser) parseConditionTree() (*query.ConditionNode, error) {
stack := lane.NewStack()
errFailedToParse := errors.New("failed to parse conditions")
for {
if p.current = p.tokenizer.Next(); p.current == nil {
break
}
switch p.current.Type {
case tokenizer.Not:
// TODO: Handle NOT (...), for the time being we proceed with the other
// tokens and handle the negation when parsing the condition.
fallthrough
case tokenizer.Identifier:
condition, err := p.parseCondition()
if err != nil {
return nil, err
}
if condition == nil {
return nil, p.currentError()
}
if condition.IsSubquery {
if err := p.parseSubquery(condition); err != nil {
return nil, err
}
}
leafNode := &query.ConditionNode{Condition: condition}
if prevNode, ok := stack.Pop().(*query.ConditionNode); !ok {
stack.Push(leafNode)
} else if prevNode.Condition == nil {
prevNode.Right = leafNode
stack.Push(prevNode)
} else {
return nil, errFailedToParse
}
case tokenizer.And, tokenizer.Or:
leftNode, ok := stack.Pop().(*query.ConditionNode)
if !ok {
return nil, errFailedToParse
}
node := query.ConditionNode{
Type: &p.current.Type,
Left: leftNode,
}
stack.Push(&node)
case tokenizer.OpenParen:
stack.Push(nil)
case tokenizer.CloseParen:
rightNode, ok := stack.Pop().(*query.ConditionNode)
if !ok {
return nil, errFailedToParse
}
if rootNode, ok := stack.Pop().(*query.ConditionNode); !ok {
stack.Push(rightNode)
} else {
rootNode.Right = rightNode
stack.Push(rootNode)
}
}
}
if stack.Size() == 0 {
return nil, p.currentError()
}
if stack.Size() > 1 {
return nil, errFailedToParse
}
node, ok := stack.Pop().(*query.ConditionNode)
if !ok {
return nil, errFailedToParse
}
return node, nil
}
// parseCondition parses and returns the next condition.
func (p *parser) parseCondition() (*query.Condition, error) {
cond := &query.Condition{}
// If we find a NOT, negate the condition.
if p.expect(tokenizer.Not) != nil {
cond.Negate = true
}
ident := p.expect(tokenizer.Identifier)
if ident == nil {
return nil, p.currentError()
}
p.current = ident
var modifiers []query.Modifier
attr, err := p.parseAttr(&modifiers)
if err != nil {
return nil, err
}
cond.Attribute = attr.Raw
cond.AttributeModifiers = modifiers
// If this condition has modifiers, then p.current was unset while parsing
// the modifier, se we set the current token manually.
if len(modifiers) > 0 {
p.current = p.tokenizer.Next()
}
if p.current == nil {
return nil, p.currentError()
}
cond.Operator = p.current.Type
p.current = nil
// Parse subquery of format `(...)`.
if p.expect(tokenizer.OpenParen) != nil {
token := p.expect(tokenizer.Subquery)
if token == nil {
return nil, p.currentError()
}
cond.IsSubquery = true
cond.Value = token.Raw
if p.expect(tokenizer.CloseParen) == nil {
return nil, p.currentError()
}
return cond, nil
}
// Parse list of values of format `[...]`.
if p.expect(tokenizer.OpenBracket) != nil {
values := make([]string, 0)
for {
if token := p.expect(tokenizer.Identifier); token != nil {
values = append(values, token.Raw)
}
if p.expect(tokenizer.Comma) != nil {
continue
}
if p.expect(tokenizer.CloseBracket) != nil {
break
}
}
cond.Value = values
return cond, nil
}
// Not a list nor a subquery -> plain identifier!
token := p.expect(tokenizer.Identifier)
if token == nil {
return nil, p.currentError()
}
cond.Value = token.Raw
return cond, nil
}
// parseSubquery parses a subquery by recursively evaluating it's condition(s).
// If the subquery contains references to aliases from the superquery, it's
// Subquery attribute is set. Otherwise, we evaluate it's Subquery and set
// it's Value to the result.
func (p *parser) parseSubquery(condition *query.Condition) error {
q, err := Run(condition.Value.(string))
if err != nil {
return err
}
// If the subquery has aliases, we'll have to parse the subquery against
// each file, so we don't do anything here.
if len(q.SourceAliases) > 0 {
condition.Subquery = q
return nil
}
value := make(map[interface{}]bool, 0)
workFunc := func(path string, info os.FileInfo, res map[string]interface{}) {
for _, attr := range [...]string{"name", "size", "time", "mode"} {
if q.HasAttribute(attr) {
value[res[attr]] = true
return
}
}
}
if err = q.Execute(workFunc); err != nil {
return err
}
condition.Value = value
condition.IsSubquery = false
return nil
}
+408
View File
@@ -0,0 +1,408 @@
package parser
import (
"errors"
"io"
"reflect"
"testing"
"github.com/kashav/fsql/query"
"github.com/kashav/fsql/tokenizer"
)
func TestConditionParser_ExpectCorrectCondition(t *testing.T) {
type Expected struct {
condition *query.Condition
err error
}
type Case struct {
input string
expected Expected
}
cases := []Case{
{
input: "name LIKE foo%",
expected: Expected{
condition: &query.Condition{
Attribute: "name",
Operator: tokenizer.Like,
Value: "foo%",
},
err: nil,
},
},
{
input: "size = 10",
expected: Expected{
condition: &query.Condition{
Attribute: "size",
Operator: tokenizer.Equals,
Value: "10",
},
err: nil,
},
},
{
input: "mode IS dir",
expected: Expected{
condition: &query.Condition{
Attribute: "mode",
Operator: tokenizer.Is,
Value: "dir",
},
err: nil,
},
},
{
input: "format(time, iso) >= 2017-05-28T16:37:18Z",
expected: Expected{
condition: &query.Condition{
Attribute: "time",
AttributeModifiers: []query.Modifier{
{
Name: "FORMAT",
Arguments: []string{"iso"},
},
},
Operator: tokenizer.GreaterThanEquals,
Value: "2017-05-28T16:37:18Z",
},
err: nil,
},
},
{
input: "upper(name) != FOO",
expected: Expected{
condition: &query.Condition{
Attribute: "name",
AttributeModifiers: []query.Modifier{
{
Name: "UPPER",
Arguments: []string{},
},
},
Operator: tokenizer.NotEquals,
Value: "FOO",
},
err: nil,
},
},
{
input: "NOT name IN [foo,bar,baz]",
expected: Expected{
condition: &query.Condition{
Attribute: "name",
Operator: tokenizer.In,
Value: []string{"foo", "bar", "baz"},
Negate: true,
},
err: nil,
},
},
// No attribute-operator validation yet (these 3 should /eventually/ throw
// some error)!
{
input: "time RLIKE '.*'",
expected: Expected{
condition: &query.Condition{
Attribute: "time",
Operator: tokenizer.RLike,
Value: ".*",
},
err: nil,
},
},
{
input: "size LIKE foo",
expected: Expected{
condition: &query.Condition{
Attribute: "size",
Operator: tokenizer.Like,
Value: "foo",
},
err: nil,
},
},
{
input: "time <> now",
expected: Expected{
condition: &query.Condition{
Attribute: "time",
Operator: tokenizer.NotEquals,
Value: "now",
},
err: nil,
},
},
{
input: "name =",
expected: Expected{err: io.ErrUnexpectedEOF},
},
{
input: "file IS dir",
expected: Expected{err: &ErrUnknownToken{"file"}},
},
}
for _, c := range cases {
p := &parser{tokenizer: tokenizer.NewTokenizer(c.input)}
actual, err := p.parseCondition()
if c.expected.err == nil {
if err != nil {
t.Fatalf("\nExpected no error\n Got %v", err)
}
if !reflect.DeepEqual(c.expected.condition, actual) {
t.Fatalf("\nExpected %v\n Got %v", c.expected.condition, actual)
}
} else if !reflect.DeepEqual(c.expected.err, err) {
t.Fatalf("\nExpected %v\n Got %v", c.expected.err, err)
}
}
}
func TestConditionParser_ExpectCorrectConditionTree(t *testing.T) {
type Expected struct {
node *query.ConditionNode
err error
}
type Case struct {
input string
expected Expected
}
// Not sure why, but compiler throws when attempting to take the memory
// address of any tokenizer.TokenType.
var (
tmpAnd = tokenizer.And
tmpOr = tokenizer.Or
)
cases := []Case{
{
input: "name LIKE foo%",
expected: Expected{
node: &query.ConditionNode{
Condition: &query.Condition{
Attribute: "name",
Operator: tokenizer.Like,
Value: "foo%",
},
},
err: nil,
},
},
{
input: "upper(name) = MAIN",
expected: Expected{
node: &query.ConditionNode{
Condition: &query.Condition{
Attribute: "name",
AttributeModifiers: []query.Modifier{
{
Name: "UPPER",
Arguments: []string{},
},
},
Operator: tokenizer.Equals,
Value: "MAIN",
},
},
err: nil,
},
},
{
input: "name LIKE %foo AND name <> bar.foo",
expected: Expected{
node: &query.ConditionNode{
Type: &tmpAnd,
Left: &query.ConditionNode{
Condition: &query.Condition{
Attribute: "name",
Operator: tokenizer.Like,
Value: "%foo",
},
},
Right: &query.ConditionNode{
Condition: &query.Condition{
Attribute: "name",
Operator: tokenizer.NotEquals,
Value: "bar.foo",
},
},
},
err: nil,
},
},
{
input: "size <= 10 OR NOT mode IS dir",
expected: Expected{
node: &query.ConditionNode{
Type: &tmpOr,
Left: &query.ConditionNode{
Condition: &query.Condition{
Attribute: "size",
Operator: tokenizer.LessThanEquals,
Value: "10",
},
},
Right: &query.ConditionNode{
Condition: &query.Condition{
Attribute: "mode",
Operator: tokenizer.Is,
Value: "dir",
Negate: true,
},
},
},
err: nil,
},
},
{
input: "size = 5 AND name = foo",
expected: Expected{
node: &query.ConditionNode{
Type: &tmpAnd,
Left: &query.ConditionNode{
Condition: &query.Condition{
Attribute: "size",
Operator: tokenizer.Equals,
Value: "5",
},
},
Right: &query.ConditionNode{
Condition: &query.Condition{
Attribute: "name",
Operator: tokenizer.Equals,
Value: "foo",
},
},
},
err: nil,
},
},
{
input: "format(size, mb) <= 2 AND (name = foo OR name = bar)",
expected: Expected{
node: &query.ConditionNode{
Type: &tmpAnd,
Left: &query.ConditionNode{
Condition: &query.Condition{
Attribute: "size",
AttributeModifiers: []query.Modifier{
{
Name: "FORMAT",
Arguments: []string{"mb"},
},
},
Operator: tokenizer.LessThanEquals,
Value: "2",
},
},
Right: &query.ConditionNode{
Type: &tmpOr,
Left: &query.ConditionNode{
Condition: &query.Condition{
Attribute: "name",
Operator: tokenizer.Equals,
Value: "foo",
},
},
Right: &query.ConditionNode{
Condition: &query.Condition{
Attribute: "name",
Operator: tokenizer.Equals,
Value: "bar",
},
},
},
},
err: nil,
},
},
{
input: "name = foo AND NOT (name = bar OR name = baz)",
expected: Expected{
node: &query.ConditionNode{},
err: &ErrUnexpectedToken{
Expected: tokenizer.Identifier,
Actual: tokenizer.OpenParen,
},
},
},
{
input: "size = 5 AND ()",
expected: Expected{err: errors.New("failed to parse conditions")},
},
// FIXME: The following case /should/ throw EOF (it doesn't right now).
// Case{input: "name = foo AND", expected: Expected{err: io.ErrUnexpectedEOF}},
}
for _, c := range cases {
p := &parser{tokenizer: tokenizer.NewTokenizer(c.input)}
actual, err := p.parseConditionTree()
if c.expected.err == nil {
if err != nil {
t.Fatalf("\nExpected no error\n Got %v", err)
}
if !reflect.DeepEqual(c.expected.node, actual) {
t.Fatalf("\nExpected %v\n Got %v", c.expected.node, actual)
}
} else if !reflect.DeepEqual(c.expected.err, err) {
t.Fatalf("\nExpected %v\n Got %v", c.expected.err, err)
}
}
}
func TestConditionParser_ExpectCorrectSubquery(t *testing.T) {
type Expected struct {
condition *query.Condition
err error
}
type Case struct {
input *query.Condition
expected Expected
}
// TODO: Complete these cases. This test relies on testdata fixtures.
cases := []Case{}
for _, c := range cases {
p := &parser{tokenizer: tokenizer.NewTokenizer("")}
err := p.parseSubquery(c.input)
if c.expected.err == nil {
if err != nil {
t.Fatalf("\nExpected no error\n Got %v", err)
}
if !reflect.DeepEqual(c.expected.condition, c.input) {
t.Fatalf("\nExpected %v\n Got %v", c.expected.condition, c.input)
}
} else if !reflect.DeepEqual(c.expected.err, err) {
t.Fatalf("\nExpected %v\n Got %v", c.expected.err, err)
}
}
}
+42
View File
@@ -0,0 +1,42 @@
package parser
import (
"fmt"
"io"
"github.com/kashav/fsql/tokenizer"
)
// ErrUnexpectedToken represents an unexpected token error.
type ErrUnexpectedToken struct {
Actual tokenizer.TokenType
Expected tokenizer.TokenType
}
func (e *ErrUnexpectedToken) Error() string {
return fmt.Sprintf("expected %s; got %s", e.Expected.String(),
e.Actual.String())
}
// ErrUnknownToken represents an unknown token error.
type ErrUnknownToken struct {
Raw string
}
func (e *ErrUnknownToken) Error() string {
return fmt.Sprintf("unknown token: %s", e.Raw)
}
// currentError returns the current error, based on the parser's current Token
// and the previously expected TokenType (set in parser.expect).
func (p *parser) currentError() error {
if p.current == nil {
return io.ErrUnexpectedEOF
}
if p.current.Type == tokenizer.Unknown {
return &ErrUnknownToken{Raw: p.current.Raw}
}
return &ErrUnexpectedToken{Actual: p.current.Type, Expected: p.expected}
}
+28
View File
@@ -0,0 +1,28 @@
package parser
import (
"testing"
"github.com/kashav/fsql/tokenizer"
)
func TestParser_ErrUnexpectedToken(t *testing.T) {
err := &ErrUnexpectedToken{
Actual: tokenizer.Select,
Expected: tokenizer.Where,
}
expected := "expected where; got select"
actual := err.Error()
if expected != actual {
t.Fatalf("\nExpected: %s\n Got: %s", expected, actual)
}
}
func TestParser_ErrUnknownTokent(t *testing.T) {
err := &ErrUnknownToken{"r"}
expected := "unknown token: r"
actual := err.Error()
if expected != actual {
t.Fatalf("\nExpected: %s\n Got: %s", expected, actual)
}
}
+140
View File
@@ -0,0 +1,140 @@
package parser
import (
"os/user"
"path/filepath"
"strings"
"github.com/kashav/fsql/query"
"github.com/kashav/fsql/tokenizer"
)
// Run parses the input string and returns the parsed AST (query).
func Run(input string) (*query.Query, error) {
return (&parser{}).parse(input)
}
type parser struct {
tokenizer *tokenizer.Tokenizer
current *tokenizer.Token
expected tokenizer.TokenType
}
// parse runs the respective parser function on each clause of the query.
func (p *parser) parse(input string) (*query.Query, error) {
q := query.NewQuery()
p.tokenizer = tokenizer.NewTokenizer(input)
if err := p.parseSelectClause(q); err != nil {
return nil, err
}
if err := p.parseFromClause(q); err != nil {
return nil, err
}
if err := p.parseWhereClause(q); err != nil {
return nil, err
}
return q, nil
}
// parseSelectClause parses the SELECT clause of the query.
func (p *parser) parseSelectClause(q *query.Query) error {
// Determine if we should show all attributes. This is only true when
// no attributes are provided (regardless of if the SELECT keyword is
// provided or not).
var showAll = true
if p.expect(tokenizer.Select) == nil {
if p.current == nil || p.current.Type == tokenizer.Identifier {
showAll = false
} else if p.current.Type == tokenizer.From || p.current.Type == tokenizer.Where {
// No SELECT and next token is FROM/WHERE, show all!
showAll = true
} else {
// No SELECT and next token is not Identifier nor FROM/WHERE -> malformed
// input.
return p.currentError()
}
} else if current := p.expect(tokenizer.Identifier); current != nil {
p.current = current
showAll = false
}
if showAll {
q.Attributes = allAttributes
} else if err := p.parseAttrs(&q.Attributes, &q.Modifiers); err != nil {
return err
}
return nil
}
// parseFromClause parses the FROM clause of the query.
func (p *parser) parseFromClause(q *query.Query) error {
if p.expect(tokenizer.From) == nil {
err := p.currentError()
if p.expect(tokenizer.Identifier) != nil {
// No FROM, but an identifier -> malformed query.
return err
}
// No specified directory, so we default to the CWD.
q.Sources["include"] = append(q.Sources["include"], ".")
return nil
}
if err := p.parseSourceList(&q.Sources, &q.SourceAliases); err != nil {
return err
}
// Replace the tilde with the home directory in each source directory. This
// is only required when the query is wrapped in quotes, since the shell
// will automatically expand tildes otherwise.
u, err := user.Current()
if err != nil {
return err
}
for _, sourceType := range []string{"include", "exclude"} {
for i, src := range q.Sources[sourceType] {
if strings.Contains(src, "~") {
q.Sources[sourceType][i] = filepath.Join(u.HomeDir, src[1:])
}
}
}
return nil
}
// parseWhereClause parses the WHERE clause of the query.
func (p *parser) parseWhereClause(q *query.Query) error {
if p.expect(tokenizer.Where) == nil {
err := p.currentError()
if p.expect(tokenizer.Identifier) == nil {
return nil
}
return err
}
root, err := p.parseConditionTree()
if err != nil {
return err
}
q.ConditionTree = root
return nil
}
// expect returns the next token if it matches the expectation t, and
// nil otherwise.
func (p *parser) expect(t tokenizer.TokenType) *tokenizer.Token {
p.expected = t
if p.current == nil {
p.current = p.tokenizer.Next()
}
if p.current != nil && p.current.Type == t {
tok := p.current
p.current = nil
return tok
}
return nil
}
+437
View File
@@ -0,0 +1,437 @@
package parser
import (
"io"
"os/user"
"reflect"
"testing"
"github.com/kashav/fsql/query"
"github.com/kashav/fsql/tokenizer"
)
func TestParser_ParseSelect(t *testing.T) {
type Expected struct {
attributes []string
modifiers map[string][]query.Modifier
err error
}
type Case struct {
input string
expected Expected
}
cases := []Case{
{
input: "all",
expected: Expected{
attributes: allAttributes,
modifiers: map[string][]query.Modifier{},
err: nil,
},
},
{
input: "SELECT",
expected: Expected{
attributes: allAttributes,
modifiers: map[string][]query.Modifier{},
err: nil,
},
},
{
input: "FROM",
expected: Expected{
attributes: allAttributes,
modifiers: map[string][]query.Modifier{},
err: nil,
},
},
{
input: "SELECT name",
expected: Expected{
attributes: []string{"name"},
modifiers: map[string][]query.Modifier{"name": {}},
err: nil,
},
},
{
input: "SELECT format(size, kb)",
expected: Expected{
attributes: []string{"size"},
modifiers: map[string][]query.Modifier{
"size": {
{
Name: "FORMAT",
Arguments: []string{"kb"},
},
},
},
err: nil,
},
},
{
input: "",
expected: Expected{err: io.ErrUnexpectedEOF},
},
}
for _, c := range cases {
q := query.NewQuery()
err := (&parser{tokenizer: tokenizer.NewTokenizer(c.input)}).parseSelectClause(q)
if c.expected.err == nil {
if err != nil {
t.Fatalf("\nExpected no error\n Got %v", err)
}
if !reflect.DeepEqual(c.expected.attributes, q.Attributes) {
t.Fatalf("\nExpected %v\n Got %v", c.expected.attributes, q.Attributes)
}
if !reflect.DeepEqual(c.expected.modifiers, q.Modifiers) {
t.Fatalf("\nExpected %v\n Got %v", c.expected.modifiers, q.Modifiers)
}
} else if !reflect.DeepEqual(c.expected.err, err) {
t.Fatalf("\nExpected %v\n Got %v", c.expected.err, err)
}
}
}
func TestParser_ParseFrom(t *testing.T) {
type Expected struct {
sources map[string][]string
aliases map[string]string
err error
}
type Case struct {
input string
expected Expected
}
u, err := user.Current()
if err != nil {
// TODO: If we can't get the current user, should we fatal or just return?
return
}
cases := []Case{
{
input: "WHERE",
expected: Expected{
sources: map[string][]string{
"include": {"."},
"exclude": {},
},
aliases: map[string]string{},
err: nil,
},
},
{
input: "FROM .",
expected: Expected{
sources: map[string][]string{
"include": {"."},
"exclude": {},
},
aliases: map[string]string{},
err: nil,
},
},
{
input: "FROM ~/foo, -./.git/",
expected: Expected{
sources: map[string][]string{
"include": {u.HomeDir + "/foo"},
"exclude": {".git"},
},
aliases: map[string]string{},
err: nil,
},
},
{
input: "FROM ./foo/ AS foo",
expected: Expected{
sources: map[string][]string{
"include": {"foo"},
"exclude": {},
},
aliases: map[string]string{"foo": "foo"},
err: nil,
},
},
{
input: "FROM",
expected: Expected{err: io.ErrUnexpectedEOF},
},
{
input: "FROM WHERE",
expected: Expected{
err: &ErrUnexpectedToken{
Actual: tokenizer.Where,
Expected: tokenizer.Identifier,
},
},
},
}
for _, c := range cases {
q := query.NewQuery()
err := (&parser{tokenizer: tokenizer.NewTokenizer(c.input)}).parseFromClause(q)
if c.expected.err == nil {
if err != nil {
t.Fatalf("\nExpected no error\n Got %v", err)
}
if !reflect.DeepEqual(c.expected.sources, q.Sources) {
t.Fatalf("\nExpected %v\n Got %v", c.expected.sources, q.Sources)
}
if !reflect.DeepEqual(c.expected.aliases, q.SourceAliases) {
t.Fatalf("\nExpected %v\n Got %v", c.expected.aliases, q.SourceAliases)
}
} else if !reflect.DeepEqual(c.expected.err, err) {
t.Fatalf("\nExpected %v\n Got %v", c.expected.err, err)
}
}
}
func TestParser_ParseWhere(t *testing.T) {
type Expected struct {
tree *query.ConditionNode
err error
}
type Case struct {
input string
expected Expected
}
cases := []Case{
{
input: "WHERE name LIKE foo",
expected: Expected{
tree: &query.ConditionNode{
Condition: &query.Condition{
Attribute: "name",
Operator: tokenizer.Like,
Value: "foo",
},
},
err: nil,
},
},
// Our tree is fully-zeroed in this case, so it's easier just to give it
// an empty Expected struct.
{input: "", expected: Expected{}},
{input: "WHERE", expected: Expected{err: io.ErrUnexpectedEOF}},
{
input: "name LIKE foo",
expected: Expected{
err: &ErrUnexpectedToken{
Expected: tokenizer.Where,
Actual: tokenizer.Identifier,
},
},
},
}
for _, c := range cases {
q := query.NewQuery()
err := (&parser{tokenizer: tokenizer.NewTokenizer(c.input)}).parseWhereClause(q)
if c.expected.err == nil {
if err != nil {
t.Fatalf("\nExpected no error\n Got %v", err)
}
if !reflect.DeepEqual(c.expected.tree, q.ConditionTree) {
t.Fatalf("\nExpected %v\n Got %v", c.expected.tree, q.ConditionTree)
}
} else if !reflect.DeepEqual(c.expected.err, err) {
t.Fatalf("\nExpected %v\n Got %v", c.expected.err, err)
}
}
}
func TestParser_Expect(t *testing.T) {
type Case struct {
param tokenizer.TokenType
expected *tokenizer.Token
}
input := "SELECT all FROM . WHERE name = foo OR size <> 100"
p := &parser{tokenizer: tokenizer.NewTokenizer(input)}
cases := []Case{
{
param: tokenizer.Select,
expected: &tokenizer.Token{Type: tokenizer.Select, Raw: "SELECT"},
},
{
param: tokenizer.From,
expected: nil,
},
{
param: tokenizer.Identifier,
expected: &tokenizer.Token{Type: tokenizer.Identifier, Raw: "all"},
},
{
param: tokenizer.Identifier,
expected: nil,
},
{
param: tokenizer.From,
expected: &tokenizer.Token{Type: tokenizer.From, Raw: "FROM"},
},
{
param: tokenizer.Identifier,
expected: &tokenizer.Token{Type: tokenizer.Identifier, Raw: "."},
},
{
param: tokenizer.Identifier,
expected: nil,
},
{
param: tokenizer.Where,
expected: &tokenizer.Token{Type: tokenizer.Where, Raw: "WHERE"},
},
{
param: tokenizer.Identifier,
expected: &tokenizer.Token{Type: tokenizer.Identifier, Raw: "name"},
},
{
param: tokenizer.Equals,
expected: &tokenizer.Token{Type: tokenizer.Equals, Raw: "="},
},
{
param: tokenizer.Identifier,
expected: &tokenizer.Token{Type: tokenizer.Identifier, Raw: "foo"},
},
{
param: tokenizer.Or,
expected: &tokenizer.Token{Type: tokenizer.Or, Raw: "OR"},
},
{
param: tokenizer.Identifier,
expected: &tokenizer.Token{Type: tokenizer.Identifier, Raw: "size"},
},
{
param: tokenizer.Identifier,
expected: nil,
},
{
param: tokenizer.NotEquals,
expected: &tokenizer.Token{Type: tokenizer.NotEquals, Raw: "<>"},
},
{
param: tokenizer.Identifier,
expected: &tokenizer.Token{Type: tokenizer.Identifier, Raw: "100"},
},
}
for _, c := range cases {
actual := p.expect(c.param)
if !reflect.DeepEqual(c.expected, actual) {
t.Fatalf("\nExpected %v\n Got %v", c.expected, actual)
}
}
}
func TestParser_SelectAllVariations(t *testing.T) {
expected := &query.Query{
Attributes: allAttributes,
Sources: map[string][]string{
"include": {"."},
"exclude": {},
},
ConditionTree: &query.ConditionNode{
Condition: &query.Condition{
Attribute: "name",
Operator: tokenizer.Like,
Value: "foo",
},
},
SourceAliases: map[string]string{},
Modifiers: map[string][]query.Modifier{},
}
cases := []string{
"FROM . WHERE name LIKE foo",
"all FROM . WHERE name LIKE foo",
"SELECT FROM . WHERE name LIKE foo",
"SELECT all FROM . WHERE name LIKE foo",
}
for _, c := range cases {
actual, err := Run(c)
if err != nil {
t.Fatalf("\nExpected no error\n Got %v", err)
}
if !reflect.DeepEqual(expected, actual) {
t.Fatalf("\nExpected %v\n Got %v", expected, actual)
}
}
}
func TestParser_Run(t *testing.T) {
type Expected struct {
q *query.Query
err error
}
type Case struct {
input string
expected Expected
}
// TODO: Add more cases.
cases := []Case{
{
input: "SELECT all FROM . WHERE name LIKE foo",
expected: Expected{
q: &query.Query{
Attributes: allAttributes,
Sources: map[string][]string{
"include": {"."},
"exclude": {},
},
ConditionTree: &query.ConditionNode{
Condition: &query.Condition{
Attribute: "name",
Operator: tokenizer.Like,
Value: "foo",
},
},
SourceAliases: map[string]string{},
Modifiers: map[string][]query.Modifier{},
},
err: nil,
},
},
}
for _, c := range cases {
actual, err := Run(c.input)
if c.expected.err == nil {
if err != nil {
t.Fatalf("\nExpected no error\n Got %v", err)
}
if !reflect.DeepEqual(c.expected.q, actual) {
t.Fatalf("\nExpected %v\n Got %v", c.expected.q, actual)
}
} else if !reflect.DeepEqual(c.expected.err, err) {
t.Fatalf("\nExpected %v\n Got %v", c.expected.err, err)
}
}
}
+45
View File
@@ -0,0 +1,45 @@
package parser
import (
"fmt"
"path/filepath"
"github.com/kashav/fsql/tokenizer"
)
// parseSourceList parses the list of directories passed to the FROM clause. If
// a source is followed by the AS keyword, the following word is registered as
// an alias.
func (p *parser) parseSourceList(sources *map[string][]string,
aliases *map[string]string) error {
for {
// If the next token is a hypen, exclude this directory.
sourceType := "include"
if token := p.expect(tokenizer.Hyphen); token != nil {
sourceType = "exclude"
}
source := p.expect(tokenizer.Identifier)
if source == nil {
return p.currentError()
}
source.Raw = filepath.Clean(source.Raw)
(*sources)[sourceType] = append((*sources)[sourceType], source.Raw)
if token := p.expect(tokenizer.As); token != nil {
alias := p.expect(tokenizer.Identifier)
if alias == nil {
return p.currentError()
}
if sourceType == "exclude" {
return fmt.Errorf("cannot alias excluded directory %s", source.Raw)
}
(*aliases)[alias.Raw] = source.Raw
}
if p.expect(tokenizer.Comma) == nil {
break
}
}
return nil
}
+149
View File
@@ -0,0 +1,149 @@
package parser
import (
"errors"
"io"
"reflect"
"testing"
"github.com/kashav/fsql/tokenizer"
)
func TestSourceParser_ExpectCorrectSources(t *testing.T) {
type Expected struct {
sources map[string][]string
err error
}
type Case struct {
input string
expected Expected
}
cases := []Case{
{
input: ".",
expected: Expected{
sources: map[string][]string{"include": {"."}},
err: nil,
},
},
{
input: "., ~/foo",
expected: Expected{
sources: map[string][]string{"include": {".", "~/foo"}},
err: nil,
},
},
{
input: "., -.bar",
expected: Expected{
sources: map[string][]string{
"include": {"."},
"exclude": {".bar"},
},
err: nil,
},
},
{
input: "-.bar, ., ~/foo AS foo",
expected: Expected{
sources: map[string][]string{
"include": {".", "~/foo"},
"exclude": {".bar"},
},
err: nil,
},
},
{input: "", expected: Expected{err: io.ErrUnexpectedEOF}},
{input: "foo,", expected: Expected{err: io.ErrUnexpectedEOF}},
}
for _, c := range cases {
sources := make(map[string][]string, 0)
aliases := make(map[string]string, 0)
p := &parser{tokenizer: tokenizer.NewTokenizer(c.input)}
err := p.parseSourceList(&sources, &aliases)
if c.expected.err == nil {
if err != nil {
t.Fatalf("\nExpected no error\n Got %v", err)
}
if !reflect.DeepEqual(c.expected.sources, sources) {
t.Fatalf("\nExpected %v\n Got %v", c.expected.sources, sources)
}
} else if !reflect.DeepEqual(c.expected.err, err) {
t.Fatalf("\nExpected %v\n Got %v", c.expected.err, err)
}
}
}
func TestSourceParser_ExpectCorrectAliases(t *testing.T) {
type Expected struct {
aliases map[string]string
err error
}
type Case struct {
input string
expected Expected
}
cases := []Case{
{
input: ".",
expected: Expected{
aliases: map[string]string{},
err: nil,
},
},
{
input: ". AS cwd",
expected: Expected{
aliases: map[string]string{"cwd": "."},
err: nil,
},
},
{
input: "., -.bar, ~/foo AS foo",
expected: Expected{
aliases: map[string]string{"foo": "~/foo"},
err: nil,
},
},
{
input: "-.bar AS bar",
expected: Expected{err: errors.New("cannot alias excluded directory .bar")},
},
{
input: "",
expected: Expected{err: io.ErrUnexpectedEOF},
},
{
input: "foo AS",
expected: Expected{err: io.ErrUnexpectedEOF},
},
}
for _, c := range cases {
sources := make(map[string][]string, 0)
aliases := make(map[string]string, 0)
p := &parser{tokenizer: tokenizer.NewTokenizer(c.input)}
err := p.parseSourceList(&sources, &aliases)
if c.expected.err == nil {
if err != nil {
t.Fatalf("\nExpected no error\n Got %v", err)
}
if !reflect.DeepEqual(c.expected.aliases, aliases) {
t.Fatalf("\nExpected %v\n Got %v", c.expected.aliases, aliases)
}
} else if !reflect.DeepEqual(c.expected.err, err) {
t.Fatalf("\nExpected %v\n Got %v", c.expected.err, err)
}
}
}
+137
View File
@@ -0,0 +1,137 @@
package query
import (
"errors"
"fmt"
"os"
"github.com/kashav/fsql/evaluate"
"github.com/kashav/fsql/tokenizer"
"github.com/kashav/fsql/transform"
)
// ConditionNode represents a single node of a query's WHERE clause tree.
type ConditionNode struct {
Type *tokenizer.TokenType
Left *ConditionNode
Right *ConditionNode
Condition *Condition
}
func (root *ConditionNode) String() string {
if root == nil {
return "<nil>"
}
return fmt.Sprintf("{%v (%v %v) %v}", root.Type, root.Left, root.Right,
root.Condition)
}
// evaluateTree runs pre-order traversal on the ConditionNode tree rooted at
// root and evaluates each conditional along the path with the provided compare
// method.
func (root *ConditionNode) evaluateTree(path string, info os.FileInfo) (bool, error) {
if root == nil {
return true, nil
}
if root.Condition != nil {
if root.Condition.IsSubquery {
// Unevaluated subquery.
// TODO: Handle this case.
return false, errors.New("not implemented")
}
if !root.Condition.Parsed {
if err := root.Condition.applyModifiers(); err != nil {
return false, err
}
}
return root.Condition.evaluate(path, info)
}
if *root.Type == tokenizer.And {
if ok, err := root.Left.evaluateTree(path, info); err != nil {
return false, err
} else if !ok {
return false, nil
}
return root.Right.evaluateTree(path, info)
}
if *root.Type == tokenizer.Or {
if ok, err := root.Left.evaluateTree(path, info); err != nil {
return false, nil
} else if ok {
return true, nil
}
return root.Right.evaluateTree(path, info)
}
return false, nil
}
// Condition represents a WHERE condition.
type Condition struct {
Attribute string
AttributeModifiers []Modifier
Parsed bool
Operator tokenizer.TokenType
Value interface{}
Negate bool
Subquery *Query
IsSubquery bool
}
// ApplyModifiers applies each modifier to the value of this Condition.
func (c *Condition) applyModifiers() error {
value := c.Value
for _, m := range c.AttributeModifiers {
var err error
value, err = transform.Parse(&transform.ParseParams{
Attribute: c.Attribute,
Value: value,
Name: m.Name,
Args: m.Arguments,
})
if err != nil {
return err
}
}
c.Value = value
c.Parsed = true
return nil
}
// evaluate runs the respective evaluate function for this Condition.
func (c *Condition) evaluate(path string, file os.FileInfo) (bool, error) {
// FIXME: This is a bit of a hack. We can't pass c.AttributeModifiers, since
// that'll cause a import cycle, so we have to recreate the attribute
// modifiers slice using a separate type defined in evaluate.
modifiers := make([]evaluate.Modifier, len(c.AttributeModifiers))
for i, m := range c.AttributeModifiers {
modifiers[i] = evaluate.Modifier{Name: m.Name, Arguments: m.Arguments}
}
o := &evaluate.Opts{
Path: path,
File: file,
Attribute: c.Attribute,
Modifiers: modifiers,
Operator: c.Operator,
Value: c.Value,
}
result, err := evaluate.Evaluate(o)
if err != nil {
return false, err
}
if c.Negate {
return !result, nil
}
return result, nil
}
+1
View File
@@ -0,0 +1 @@
package query
+43
View File
@@ -0,0 +1,43 @@
package query
import (
"fmt"
"regexp"
"strings"
)
// Excluder allows us to support different methods of excluding in the future.
type Excluder interface {
shouldExclude(path string) bool
}
// regexpExclude uses regular expressions to tell if a file/path should be
// excluded.
type regexpExclude struct {
exclusions []string
regex *regexp.Regexp
}
// ShouldExclude will return a boolean denoting whether or not the path should
// be excluded based on the given slice of exclusions.
func (r *regexpExclude) shouldExclude(path string) bool {
if r.regex == nil {
r.buildRegex()
}
if r.regex.String() == "" {
return false
}
return r.regex.MatchString(path)
}
// buildRegex builds the regular expression for this RegexpExclude.
func (r *regexpExclude) buildRegex() {
exclusions := make([]string, len(r.exclusions))
for i, exclusion := range r.exclusions {
// Wrap exclusion in ^ and (/.*)?$ AFTER trimming trailing slashes and
// escaping all dots.
exclusions[i] = fmt.Sprintf("^%s(/.*)?$",
strings.Replace(strings.TrimRight(exclusion, "/"), ".", "\\.", -1))
}
r.regex = regexp.MustCompile(strings.Join(exclusions, "|"))
}
+51
View File
@@ -0,0 +1,51 @@
package query
import "testing"
func TestShouldExclude_ExpectAllExcluded(t *testing.T) {
type Case struct {
input string
expected bool
}
exclusions := []string{".git", ".gitignore"}
excluder := regexpExclude{exclusions: exclusions}
cases := []Case{
{input: ".git", expected: true},
{input: ".git/", expected: true},
{input: ".git/some/other/file", expected: true},
{input: ".gitignore", expected: true},
}
for _, c := range cases {
actual := excluder.shouldExclude(c.input)
if actual != c.expected {
t.Fatalf("\nExpected %v\n Got %v", c.expected, actual)
}
}
}
func TestShouldExclude_ExpectNotExcluded(t *testing.T) {
type Case struct {
input string
expected bool
}
exclusions := []string{".git"}
excluder := regexpExclude{exclusions: exclusions}
cases := []Case{
{input: ".git", expected: true},
{input: ".git/", expected: true},
{input: ".git/some/other/file", expected: true},
{input: ".gitignore", expected: false},
}
for _, c := range cases {
actual := excluder.shouldExclude(c.input)
if actual != c.expected {
t.Fatalf("\nExpected %v\n Got %v", c.expected, actual)
}
}
}
+55
View File
@@ -0,0 +1,55 @@
package query
import (
"fmt"
"os"
"strings"
"github.com/kashav/fsql/transform"
)
// Modifier represents an attribute modifier.
type Modifier struct {
Name string
Arguments []string
}
func (m *Modifier) String() string {
return fmt.Sprintf("%s(%s)", m.Name, strings.Join(m.Arguments, ", "))
}
// applyModifiers iterates through each SELECT attribute for this query
// and applies the associated modifier to the attribute's output value.
func (q *Query) applyModifiers(path string, info os.FileInfo) (map[string]interface{}, error) {
results := make(map[string]interface{}, len(q.Attributes))
for _, attribute := range q.Attributes {
value, err := transform.DefaultFormatValue(attribute, path, info)
if err != nil {
return map[string]interface{}{}, err
}
if _, ok := q.Modifiers[attribute]; !ok {
results[attribute] = value
continue
}
for _, m := range q.Modifiers[attribute] {
value, err = transform.Format(&transform.FormatParams{
Attribute: attribute,
Path: path,
Info: info,
Value: value,
Name: m.Name,
Args: m.Arguments,
})
if err != nil {
return map[string]interface{}{}, err
}
}
results[attribute] = value
}
return results, nil
}
+32
View File
@@ -0,0 +1,32 @@
package query
import "testing"
func TestModifier_String(t *testing.T) {
type Case struct {
input Modifier
expected string
}
cases := []Case{
{
input: Modifier{Name: "upper", Arguments: []string{}},
expected: "upper()",
},
{
input: Modifier{Name: "format", Arguments: []string{"upper"}},
expected: "format(upper)",
},
}
for _, c := range cases {
result := c.input.String()
if result != c.expected {
t.Fatalf("\nExpected: %s\n Got: %s", c.expected, result)
}
}
}
func TestModifier_Apply(t *testing.T) {
// TODO
}
+119
View File
@@ -0,0 +1,119 @@
package query
import (
"os"
"path/filepath"
"strings"
)
// Query represents an input query.
type Query struct {
Attributes []string
Modifiers map[string][]Modifier
Sources map[string][]string
SourceAliases map[string]string
ConditionTree *ConditionNode
}
// NewQuery returns a pointer to a Query.
func NewQuery() *Query {
return &Query{
Attributes: make([]string, 0),
Modifiers: make(map[string][]Modifier),
Sources: map[string][]string{
"include": make([]string, 0),
"exclude": make([]string, 0),
},
SourceAliases: make(map[string]string),
ConditionTree: nil,
}
}
// HasAttribute checks if this query contains any of the provided attributes.
func (q *Query) HasAttribute(attributes ...string) bool {
for _, attribute := range attributes {
for _, queryAttribute := range q.Attributes {
if attribute == queryAttribute {
return true
}
}
}
return false
}
// Execute runs the query by walking the full path of each source and
// evaluating the condition tree for each file. This method calls workFunc on
// each "successful" file.
func (q *Query) Execute(workFunc interface{}) error {
seen := map[string]bool{}
excluder := &regexpExclude{exclusions: q.Sources["exclude"]}
for _, src := range q.Sources["include"] {
// TODO: Improve our method of detecting if src is a glob pattern. This
// currently doesn't support usage of square brackets, since the tokenizer
// doesn't recognize these as part of a directory.
//
// Pattern reference: https://golang.org/pkg/path/filepath/#Match.
if strings.ContainsAny(src, "*?") {
// If src does _resemble_ a glob pattern, we find all matches and
// evaluate the condition tree against each.
matches, err := filepath.Glob(src)
if err != nil {
return err
}
for _, match := range matches {
if err = filepath.Walk(match, q.walkFunc(seen, excluder, workFunc)); err != nil {
return err
}
}
continue
}
if err := filepath.Walk(src, q.walkFunc(seen, excluder, workFunc)); err != nil {
return err
}
}
return nil
}
// walkFunc returns a filepath.WalkFunc which evaluates the condition tree
// against the given file.
func (q *Query) walkFunc(seen map[string]bool, excluder Excluder,
workFunc interface{}) filepath.WalkFunc {
return func(path string, info os.FileInfo, err error) error {
if err != nil {
return err
}
if path == "." {
return nil
}
// Avoid walking a single directory more than once.
if _, ok := seen[path]; ok {
return nil
}
seen[path] = true
if excluder.shouldExclude(path) {
return nil
}
if ok, err := q.ConditionTree.evaluateTree(path, info); err != nil {
return err
} else if !ok {
return nil
}
results, err := q.applyModifiers(path, info)
if err != nil {
return err
}
workFunc.(func(string, os.FileInfo, map[string]interface{}))(path, info, results)
return nil
}
}
+1
View File
@@ -0,0 +1 @@
package query
+31
View File
@@ -0,0 +1,31 @@
package pager
import (
"bytes"
"os"
"os/exec"
)
const cmd = "less"
var opts = []string{"-S"}
// CommandExists returns true iff cmd exists on the host machine.
func CommandExists() bool {
_, err := exec.LookPath(cmd)
return err == nil
}
// New invokes cmd with in provided as stdin, if cmd is unavailable, return
// an error.
func New(in []byte) error {
path, err := exec.LookPath(cmd)
if err != nil {
return err
}
pager := exec.Command(path, opts...)
pager.Stdin = bytes.NewReader(in)
pager.Stdout = os.Stdout
pager.Stderr = os.Stderr
return pager.Run()
}
+127
View File
@@ -0,0 +1,127 @@
package terminal
import (
"bytes"
"errors"
"fmt"
"io"
"os"
"strings"
"github.com/kashav/fsql"
"github.com/kashav/fsql/terminal/pager"
"golang.org/x/crypto/ssh/terminal"
)
var fd = int(os.Stdin.Fd())
var query bytes.Buffer
// Start listens for queries via stdin and invokes fsql.Run whenever a
// semicolon is read.
func Start() error {
if !terminal.IsTerminal(fd) {
return errors.New("not a terminal")
}
state, err := terminal.MakeRaw(fd)
if err != nil {
return err
}
defer terminal.Restore(fd, state)
prompt := ">>> "
term := terminal.NewTerminal(os.Stdin, prompt)
// Listen for queries and invoke run whenever a semicolon is read. Continues
// until receiving an EOF (Ctrl-D) or _fatal_ error (i.e. anything not
// caused by the query itself).
for {
line, err := term.ReadLine()
if err == io.EOF {
fmt.Print("\r\nbye\r\n")
break
}
if err != nil {
return err
}
if line == "exit" {
fmt.Print("bye\r\n")
break
}
// TODO: If the previous character was a paren., bracket, or quote, we
// don't want to add a space here (although not necessary, since the
// tokenizer handles excess whitespace).
if query.Len() > 0 {
query.WriteString(" ")
}
query.WriteString(line)
if strings.HasSuffix(line, ";") {
query.Truncate(query.Len() - 1)
b := []byte{}
if out, err := run(query.String()); err != nil {
// This error likely corresponds to the query, so instead of exiting
// interactive mode, we simply write the error to stdout and proceed.
b = append(b, []byte(err.Error())...)
b = append(b, '\a', '\n')
term.Write(b)
} else if len(out) > 0 {
_, h, err := terminal.GetSize(fd)
if err != nil {
return err
}
b = append(b, []byte(out)...)
// Write to stdout if out is less than 3/4 of the height of the
// window OR the `less` command doesn't exist; otherwise, invoke the
// pager.
if float64(strings.Count(out, "\n")) <= 0.75*float64(h) ||
!pager.CommandExists() {
term.Write(b)
} else if err = pager.New(b); err != nil {
return err
}
}
query.Reset()
}
prompt = "... "
if query.Len() == 0 {
prompt = ">>> "
}
term.SetPrompt(prompt)
}
return nil
}
// run invokes fsql.Run with the provided query string.
func run(query string) (out string, err error) {
stdout := os.Stdout
r, w, err := os.Pipe()
if err != nil {
return "", err
}
os.Stdout = w
defer func() { os.Stdout = stdout }()
ch := make(chan string)
go func() {
var buf bytes.Buffer
io.Copy(&buf, r)
ch <- buf.String()
}()
err = fsql.Run(query)
// Must happen after the function call and before we try to read from ch.
if closeErr := w.Close(); closeErr != nil {
return "", closeErr
}
out = <-ch
return
}
+52
View File
@@ -0,0 +1,52 @@
package terminal
import (
"errors"
"reflect"
"testing"
)
func TestRun(t *testing.T) {
type Expected struct {
out string
err error
}
type Case struct {
query string
expected Expected
}
// We already test core functionality in the fsql package, so we can stick
// with _simple_ queries for the following cases.
cases := []Case{
{
query: "select name, hash from ../testdata where name = baz",
expected: Expected{
out: "baz\tda39a3e\n",
err: nil,
},
},
{
query: "select all from",
expected: Expected{
out: "",
err: errors.New("unexpected EOF"),
},
},
}
for _, c := range cases {
actual, err := run(c.query)
if c.expected.err == nil {
if err != nil {
t.Fatalf("\nExpected no error\n Got %v", err)
}
if !reflect.DeepEqual(c.expected.out, actual) {
t.Fatalf("\nExpected %v\n Got %v", c.expected.out, actual)
}
} else if !reflect.DeepEqual(c.expected.err, err) {
t.Fatalf("\nExpected %v\n Got %v", c.expected.err, err)
}
}
}
View File
View File
View File
Vendored Executable
View File
View File
View File
View File
View File
+127
View File
@@ -0,0 +1,127 @@
package tokenizer
import "fmt"
// TokenType represents a Token's type.
type TokenType int8
// All TokenType constants.
const (
Unknown TokenType = iota
Identifier
Subquery
Select
From
Where
As
Or
And
Not
In
Is
Like
RLike
Equals
NotEquals
GreaterThanEquals
GreaterThan
LessThanEquals
LessThan
Comma
Hyphen
ExclamationMark
OpenParen
CloseParen
OpenBracket
CloseBracket
)
func (t TokenType) String() string {
switch t {
case Identifier:
return "identifier"
case Subquery:
return "subquery"
case Select:
return "select"
case From:
return "from"
case As:
return "as"
case Where:
return "where"
case Or:
return "or"
case And:
return "and"
case Not:
return "not"
case In:
return "in"
case Is:
return "is"
case Like:
return "like"
case RLike:
return "RLike"
case Equals:
return "equal"
case NotEquals:
return "not-equal"
case GreaterThanEquals:
return "greater-than-or-equal"
case GreaterThan:
return "greater-than"
case LessThanEquals:
return "less-than-or-equal"
case LessThan:
return "less-than"
case Comma:
return "comma"
case Hyphen:
return "hyphen"
case ExclamationMark:
return "exclamation-mark"
case OpenParen:
return "open-parentheses"
case CloseParen:
return "close-parentheses"
case OpenBracket:
return "open-bracket"
case CloseBracket:
return "close-bracket"
default:
return "unknown"
}
}
// Token represents a single token.
type Token struct {
Type TokenType
Raw string
}
func (t *Token) String() string {
return fmt.Sprintf("{type: %s, raw: \"%s\"}",
t.Type.String(), t.Raw)
}
// Tokenizer represents a token worker.
type Tokenizer struct {
input []rune
tokens []*Token
}
// NewTokenizer initializes a new Tokenizer.
func NewTokenizer(input string) *Tokenizer {
return &Tokenizer{
input: []rune(input),
tokens: make([]*Token, 0),
}
}
+87
View File
@@ -0,0 +1,87 @@
package tokenizer
import "testing"
func TestToken_TokenTypeString(t *testing.T) {
type Case struct {
tt TokenType
expected string
}
cases := []Case{
{tt: Identifier, expected: "identifier"},
{tt: Subquery, expected: "subquery"},
{tt: Select, expected: "select"},
{tt: From, expected: "from"},
{tt: As, expected: "as"},
{tt: Where, expected: "where"},
{tt: Or, expected: "or"},
{tt: And, expected: "and"},
{tt: Not, expected: "not"},
{tt: In, expected: "in"},
{tt: Is, expected: "is"},
{tt: Like, expected: "like"},
{tt: RLike, expected: "RLike"},
{tt: Equals, expected: "equal"},
{tt: NotEquals, expected: "not-equal"},
{tt: GreaterThanEquals, expected: "greater-than-or-equal"},
{tt: GreaterThan, expected: "greater-than"},
{tt: LessThanEquals, expected: "less-than-or-equal"},
{tt: LessThan, expected: "less-than"},
{tt: Comma, expected: "comma"},
{tt: Hyphen, expected: "hyphen"},
{tt: ExclamationMark, expected: "exclamation-mark"},
{tt: OpenParen, expected: "open-parentheses"},
{tt: CloseParen, expected: "close-parentheses"},
{tt: OpenBracket, expected: "open-bracket"},
{tt: CloseBracket, expected: "close-bracket"},
{tt: Unknown, expected: "unknown"},
}
for _, c := range cases {
actual := c.tt.String()
if c.expected != actual {
t.Fatalf("\nExpected %v\n Got %v", c.expected, actual)
}
}
}
func TestToken_String(t *testing.T) {
type Case struct {
token Token
expected string
}
cases := []Case{
{
token: Token{Type: Identifier, Raw: "name"},
expected: "{type: identifier, raw: \"name\"}",
},
{
token: Token{Type: Comma, Raw: ","},
expected: "{type: comma, raw: \",\"}",
},
}
for _, c := range cases {
actual := c.token.String()
if c.expected != actual {
t.Fatalf("\nExpected %v\n Got %v", c.expected, actual)
}
}
}
func TestToken_NewTokenizer(t *testing.T) {
input := "SELECT all FROM ."
inputLength, tokenLength := len([]rune(input)), 0
tokenizer := NewTokenizer(input)
if len(tokenizer.input) != inputLength {
t.Fatalf("len(tokenizer.input)\nExpected %v\n Got %v", inputLength,
len(tokenizer.input))
}
if len(tokenizer.tokens) != tokenLength {
t.Fatalf("len(tokenizer.tokens)\nExpected %v\n Got %v", tokenLength,
len(tokenizer.tokens))
}
}
+244
View File
@@ -0,0 +1,244 @@
package tokenizer
import (
"fmt"
"strings"
"unicode"
)
// All parses all tokens for this Tokenizer.
func (t *Tokenizer) All() []Token {
tokens := []Token{}
for tok := t.Next(); tok != nil; tok = t.Next() {
tokens = append(tokens, *tok)
}
return tokens
}
// Next finds and returns the next Token in the input string.
func (t *Tokenizer) Next() *Token {
for unicode.IsSpace(t.current()) {
t.input = t.input[1:]
}
current := t.current()
if current == -1 {
return nil
}
switch current {
case '(':
t.input = t.input[1:]
return t.setToken(&Token{Type: OpenParen, Raw: "("})
case ')':
t.input = t.input[1:]
return t.setToken(&Token{Type: CloseParen, Raw: ")"})
case '[':
t.input = t.input[1:]
return t.setToken(&Token{Type: OpenBracket, Raw: "["})
case ']':
t.input = t.input[1:]
return t.setToken(&Token{Type: CloseBracket, Raw: "]"})
case ',':
t.input = t.input[1:]
return t.setToken(&Token{Type: Comma, Raw: ","})
case '-':
t.input = t.input[1:]
return t.setToken(&Token{Type: Hyphen, Raw: "-"})
case '!':
if t.getRuneAt(1) == '=' {
t.input = t.input[2:]
return t.setToken(&Token{Type: NotEquals, Raw: "!="})
}
return t.setToken(&Token{Type: ExclamationMark, Raw: "!"})
case '=':
t.input = t.input[1:]
return t.setToken(&Token{Type: Equals, Raw: "="})
case '>':
if t.getRuneAt(1) == '=' {
t.input = t.input[2:]
return t.setToken(&Token{Type: GreaterThanEquals, Raw: ">="})
}
t.input = t.input[1:]
return t.setToken(&Token{Type: GreaterThan, Raw: ">"})
case '<':
if t.getRuneAt(1) == '=' {
t.input = t.input[2:]
return t.setToken(&Token{Type: LessThanEquals, Raw: "<="})
}
if t.getRuneAt(1) == '>' {
t.input = t.input[2:]
return t.setToken(&Token{Type: NotEquals, Raw: "<>"})
}
t.input = t.input[1:]
return t.setToken(&Token{Type: LessThan, Raw: "<"})
}
if !t.currentIs(-1, ',', '\'', '"', '`', '(', ')', '[', ']') {
word := t.readWord()
tok := &Token{Raw: word}
switch strings.ToUpper(word) {
case "SELECT":
tok.Type = Select
case "FROM":
tok.Type = From
case "WHERE":
tok.Type = Where
case "AS":
tok.Type = As
case "OR":
tok.Type = Or
case "AND":
tok.Type = And
case "NOT":
tok.Type = Not
case "IN":
tok.Type = In
case "IS":
tok.Type = Is
case "LIKE":
tok.Type = Like
case "REGEXP", "RLIKE":
tok.Type = RLike
default:
tok.Type = Identifier
}
if t.getPreviousToken() != nil && t.getPreviousToken().Type == OpenParen &&
t.getTokenAt(1) != nil && t.getTokenAt(1).Type == In {
// The two previous tokens were: `IN` and `(`, so we're at a subquery.
tok.Type = Subquery
tok.Raw = fmt.Sprintf("%s %s", word, t.readQuery())
}
return t.setToken(tok)
}
tok := &Token{Type: Unknown, Raw: string(current)}
// If the current rune is a single/double quote or backtick, we want to keep
// reading until we reach the matching closing symbol.
if t.currentIs('\'', '"', '`') {
t.input = t.input[1:]
tok.Raw = t.readWord() + t.readUntil(current)
tok.Type = Identifier
}
t.input = t.input[1:]
return t.setToken(tok)
}
// setToken adds token to the list of this Tokenizer's tokens.
func (t *Tokenizer) setToken(token *Token) *Token {
t.tokens = append(t.tokens, token)
return token
}
// getPreviousToken returns the token that was most-recently read.
func (t *Tokenizer) getPreviousToken() *Token {
return t.getTokenAt(0)
}
// getTokenAt returns the token at index i from the end of the tokens slice.
func (t *Tokenizer) getTokenAt(i int) *Token {
j := len(t.tokens) - 1 - i
if j < 0 {
return nil
}
return t.tokens[j]
}
// current returns the run at the 0th index of the input.
func (t *Tokenizer) current() rune {
return t.getRuneAt(0)
}
// getRuneAt returns the rune at the ith index of the input.
func (t *Tokenizer) getRuneAt(i int) rune {
if len(t.input) == i {
return -1
}
return t.input[i]
}
// currentIs returns true iff the input's current rune (at index 0) is in runes.
func (t *Tokenizer) currentIs(runes ...rune) bool {
for _, r := range runes {
if r == t.current() {
return true
}
}
return false
}
// readWord reads a single word from the input. Returns when the next rune is
// any of: nil (-1), empty space, comma, single/double quote, backtick,
// or opening/closing parenthesis/bracket.
func (t *Tokenizer) readWord() string {
word := []rune{}
for {
if unicode.IsSpace(t.current()) ||
t.currentIs(-1, ',', '\'', '"', '`', '(', ')', '[', ']') {
return string(word)
}
word = append(word, t.current())
t.input = t.input[1:]
}
}
// readQuery reads a full string until reaching a closing parentheses. Counts
// opening parens to ensure that balance is maintained.
func (t *Tokenizer) readQuery() string {
var query string
var count = 1
for count > 0 {
for unicode.IsSpace(t.current()) {
t.input = t.input[1:]
}
word := fmt.Sprintf("%s", t.readWord())
if t.current() == -1 {
break
}
if t.current() == '(' {
count++
word = "("
} else if t.current() == ')' {
count--
if count <= 0 {
query += word
break
}
word = ")"
} else if t.currentIs('\'', '`') {
word += string(t.current())
} else {
word += " "
}
query += word
t.input = t.input[1:]
}
return query
}
// readUnitl reads the input starting at start, until reaching a rune in runes.
func (t *Tokenizer) readUntil(runes ...rune) string {
var word string
for !t.currentIs(runes...) {
for unicode.IsSpace(t.current()) {
t.input = t.input[1:]
}
word = fmt.Sprintf("%s %s", word, t.readWord())
}
return word
}
+202
View File
@@ -0,0 +1,202 @@
package tokenizer
import (
"reflect"
"testing"
)
func TestTokenizer_NextTokenType(t *testing.T) {
type Case struct {
input string
expected TokenType
}
cases := []Case{
{input: "SELECT", expected: Select},
{input: "FROM", expected: From},
{input: "WHERE", expected: Where},
{input: "AS", expected: As},
{input: "OR", expected: Or},
{input: "AND", expected: And},
{input: "NOT", expected: Not},
{input: "IN", expected: In},
{input: "IS", expected: Is},
{input: "LIKE", expected: Like},
{input: "RLIKE", expected: RLike},
{input: "foo", expected: Identifier},
{input: "(", expected: OpenParen},
{input: ")", expected: CloseParen},
{input: ",", expected: Comma},
{input: "-", expected: Hyphen},
{input: "=", expected: Equals},
{input: "<>", expected: NotEquals},
{input: "<", expected: LessThan},
{input: "<=", expected: LessThanEquals},
{input: ">", expected: GreaterThan},
{input: ">=", expected: GreaterThanEquals},
}
for _, c := range cases {
actual := NewTokenizer(c.input).Next()
expected := &Token{Type: c.expected, Raw: c.input}
if !reflect.DeepEqual(actual, expected) {
t.Fatalf("\nExpected: %v\n Got: %v", expected, actual)
}
}
}
func TestTokenizer_NextRaw(t *testing.T) {
type Case struct {
input string
expected string
}
// TODO: Fix the last 2 cases, they're currently hanging.
cases := []Case{
{input: "foo", expected: "foo"},
{input: " foo ", expected: "foo"},
{input: "\" foo \"", expected: " foo "},
{input: "' foo '", expected: " foo "},
{input: "` foo `", expected: " foo "},
// Case{input: "\"foo'bar\"", expected: "foo'bar"},
// Case{input: "\"()\"", expected: "()"},
}
for _, c := range cases {
actual := NewTokenizer(c.input).Next()
expected := &Token{Type: Identifier, Raw: c.expected}
if !reflect.DeepEqual(actual, expected) {
t.Fatalf("\nExpected: %v\n Got: %v", expected, actual)
}
}
}
func TestTokenizer_AllSimple(t *testing.T) {
input := `
SELECT
name, size
FROM
~/Desktop
WHERE
name LIKE %go
`
actual := NewTokenizer(input).All()
expected := []Token{
{Type: Select, Raw: "SELECT"},
{Type: Identifier, Raw: "name"},
{Type: Comma, Raw: ","},
{Type: Identifier, Raw: "size"},
{Type: From, Raw: "FROM"},
{Type: Identifier, Raw: "~/Desktop"},
{Type: Where, Raw: "WHERE"},
{Type: Identifier, Raw: "name"},
{Type: Like, Raw: "LIKE"},
{Type: Identifier, Raw: "%go"},
}
for i := range expected {
if !reflect.DeepEqual(actual[i], expected[i]) {
t.Fatalf("\nExpected: %v\n Got: %v", expected[i], actual[i])
}
}
}
func TestTokenizer_AllSubquery(t *testing.T) {
input := `
SELECT
name, size
FROM
~/Desktop
WHERE
name LIKE %go OR
name IN (
SELECT
name
FROM
$GOPATH/src/github.com
WHERE
name RLIKE .*_test\.go)
`
actual := NewTokenizer(input).All()
expected := []Token{
{Type: Select, Raw: "SELECT"},
{Type: Identifier, Raw: "name"},
{Type: Comma, Raw: ","},
{Type: Identifier, Raw: "size"},
{Type: From, Raw: "FROM"},
{Type: Identifier, Raw: "~/Desktop"},
{Type: Where, Raw: "WHERE"},
{Type: Identifier, Raw: "name"},
{Type: Like, Raw: "LIKE"},
{Type: Identifier, Raw: "%go"},
{Type: Or, Raw: "OR"},
{Type: Identifier, Raw: "name"},
{Type: In, Raw: "IN"},
{Type: OpenParen, Raw: "("},
{Type: Subquery, Raw: "SELECT name FROM $GOPATH/src/github.com WHERE name RLIKE .*_test\\.go"},
{Type: CloseParen, Raw: ")"},
}
for i := range expected {
if !reflect.DeepEqual(actual[i], expected[i]) {
t.Fatalf("\nExpected: %v\n Got: %v", expected[i], actual[i])
}
}
}
func TestTokenizer_ReadWord(t *testing.T) {
type Case struct {
input string
expected string
}
cases := []Case{
{input: "foo", expected: "foo"},
{input: "foo bar", expected: "foo"},
{input: "", expected: ""},
}
for _, c := range cases {
actual := NewTokenizer(c.input).readWord()
if !reflect.DeepEqual(actual, c.expected) {
t.Fatalf("\nExpected: %v\n Got: %v", c.expected, actual)
}
}
}
func TestTokenizer_ReadQuery(t *testing.T) {
type Case struct {
input string
expected string
}
// TODO: Complete these cases.
cases := []Case{}
for _, c := range cases {
actual := NewTokenizer(c.input).readQuery()
if !reflect.DeepEqual(actual, c.expected) {
t.Fatalf("\nExpected: %v\n Got: %v", c.expected, actual)
}
}
}
func TestTokenizer_ReadUntil(t *testing.T) {
type Case struct {
input string
until []rune
expected string
}
// TODO: Complete these cases.
cases := []Case{}
for _, c := range cases {
actual := NewTokenizer(c.input).readUntil(c.until...)
if !reflect.DeepEqual(actual, c.expected) {
t.Fatalf("\nExpected: %v\n Got: %v", c.expected, actual)
}
}
}
+82
View File
@@ -0,0 +1,82 @@
package transform
import (
"crypto/sha1"
"encoding/hex"
"hash"
"os"
"path/filepath"
"strings"
)
// formatName runs the correct name format function based on the value of arg.
func formatName(arg, name string) interface{} {
switch strings.ToUpper(arg) {
case "UPPER":
return upper(name)
case "LOWER":
return lower(name)
}
return nil
}
// upper returns the uppercased version of name.
func upper(name string) interface{} {
return strings.ToUpper(name)
}
// lower returns the lowercase version of name.
func lower(name string) interface{} {
return strings.ToLower(name)
}
// truncate returns the first n characters of str. If n is greater than the
// length of str or less than 0, return str.
func truncate(str string, n int) string {
if len(str) < n || n < 0 {
return str
}
return str[0:n]
}
// FindHash returns a func to create a new hash based on the provided name.
func FindHash(name string) func() hash.Hash {
switch strings.ToUpper(name) {
case "SHA1":
return sha1.New
}
return nil
}
// ComputeHash applies the hash h to the file located at path. Returns a line
// of dashes for directories.
func ComputeHash(info os.FileInfo, path string, h hash.Hash) (interface{}, error) {
fallback := strings.Repeat("-", h.Size()*2)
// If the current file is a symlink, attempt to evaluate the link and
// stat the resultant file. If either process fails, ignore the error and
// return the fallback.
if info.Mode()&os.ModeSymlink == os.ModeSymlink {
var err error
if path, err = filepath.EvalSymlinks(path); err != nil {
return fallback, nil
}
if info, err = os.Stat(path); err != nil {
return fallback, nil
}
}
if info.IsDir() {
return fallback, nil
}
b, err := os.ReadFile(path)
if err != nil {
return nil, err
}
if _, err := h.Write(b); err != nil {
return nil, err
}
return hex.EncodeToString(h.Sum(nil)), nil
}
+144
View File
@@ -0,0 +1,144 @@
package transform
import (
"crypto/sha1"
"hash"
"os"
"reflect"
"strings"
"testing"
)
func TestCommon_FormatName(t *testing.T) {
type Case struct {
arg string
name string
expected string
}
cases := []Case{
{arg: "upper", name: "foo", expected: "FOO"},
{arg: "upper", name: "FOO", expected: "FOO"},
{arg: "lower", name: "foo", expected: "foo"},
{arg: "lower", name: "FOO", expected: "foo"},
}
for _, c := range cases {
result := formatName(c.arg, c.name)
if result != c.expected {
t.Fatalf("\nExpected: %s\n Got: %s", c.expected, result)
}
}
}
func TestCommon_Upper(t *testing.T) {
type Case struct {
name string
expected string
}
cases := []Case{
{name: "foo", expected: "FOO"},
{name: "FOO", expected: "FOO"},
}
for _, c := range cases {
result := upper(c.name)
if result != c.expected {
t.Fatalf("\nExpected: %s\n Got: %s", c.expected, result)
}
}
}
func TestCommon_Lower(t *testing.T) {
type Case struct {
name string
expected string
}
cases := []Case{
{name: "foo", expected: "foo"},
{name: "FOO", expected: "foo"},
}
for _, c := range cases {
result := lower(c.name)
if result != c.expected {
t.Fatalf("\nExpected: %s\n Got: %s", c.expected, result)
}
}
}
func TestCommon_Truncate(t *testing.T) {
input := "foo-bar-baz"
type Case struct {
n int
expected string
}
cases := []Case{
{n: 3, expected: "foo"},
{n: 7, expected: "foo-bar"},
{n: 100, expected: input},
{n: -1, expected: input},
{n: len(input), expected: "foo-bar-baz"},
{n: len(input) - 1, expected: "foo-bar-ba"},
}
for _, c := range cases {
actual := truncate(input, c.n)
if c.expected != actual {
t.Fatalf("\nExpected: %s\n Got: %s", c.expected, actual)
}
}
}
func TestCommon_FindHash(t *testing.T) {
type Case struct {
name string
expected hash.Hash
}
cases := []Case{
{name: "SHA1", expected: sha1.New()},
{name: "FOO", expected: nil},
}
for _, c := range cases {
actual := FindHash(c.name)
if actual == nil {
if c.expected != nil {
t.Fatalf("\nExpected: %s\n Got: nil", c.expected)
}
} else if h := actual(); !reflect.DeepEqual(c.expected, h) {
t.Fatalf("\nExpected: %s\n Got: %s", c.expected, h)
}
}
}
func TestCommon_ComputeHash(t *testing.T) {
type Case struct {
path string
expected string
}
cases := []Case{
{path: "../testdata/foo", expected: strings.Repeat("-", 40)},
{path: "../testdata/baz", expected: "da39a3ee5e6b4b0d3255bfef95601890afd80709"},
}
for _, c := range cases {
info, err := os.Stat(c.path)
if err != nil {
t.Fatalf("\nExpected no error\n Got: %s", err.Error())
}
actual, err := ComputeHash(info, c.path, sha1.New())
if err != nil {
t.Fatalf("\nExpected no error\n Got: %s", err.Error())
}
if !reflect.DeepEqual(c.expected, actual) {
t.Fatalf("\nExpected: %s\n Got: %s", c.expected, actual)
}
}
}
+28
View File
@@ -0,0 +1,28 @@
package transform
import (
"fmt"
"strings"
)
// ErrNotImplemented used for non-implemented modifier functions.
type ErrNotImplemented struct {
Name string
Attribute string
}
func (e *ErrNotImplemented) Error() string {
return fmt.Sprintf("function %s is not implemented for attribute %s",
strings.ToUpper(e.Name), e.Attribute)
}
// ErrUnsupportedFormat used for unsupport arguments for FORMAT functions.
type ErrUnsupportedFormat struct {
Format string
Attribute string
}
func (e *ErrUnsupportedFormat) Error() string {
return fmt.Sprintf("unsupported format type %s for attribute %s",
e.Format, e.Attribute)
}
+22
View File
@@ -0,0 +1,22 @@
package transform
import "testing"
func TestTransform_ErrNotImplemented(t *testing.T) {
err := &ErrNotImplemented{"n", "a"}
expected := "function N is not implemented for attribute a"
actual := err.Error()
if expected != actual {
t.Fatalf("\nExpected: %s\n Got: %s", expected, actual)
}
}
func TestTransform_ErrUnsupportedFormat(t *testing.T) {
err := &ErrUnsupportedFormat{"f", "a"}
expected := "unsupported format type f for attribute a"
actual := err.Error()
if expected != actual {
t.Fatalf("\nExpected: %s\n Got: %s", expected, actual)
}
}
+159
View File
@@ -0,0 +1,159 @@
package transform
import (
"fmt"
"hash"
"os"
"strconv"
"strings"
"time"
)
const defaultHashLength = 7
// FormatParams holds the params for a format-modifier function.
type FormatParams struct {
Attribute string
Path string
Info os.FileInfo
Value interface{}
Name string
Args []string
}
// Format runs the respective format function on the provided parameters.
func Format(p *FormatParams) (val interface{}, err error) {
switch strings.ToUpper(p.Name) {
case "FORMAT":
val, err = p.format()
case "UPPER":
val = upper(p.Value.(string))
case "LOWER":
val = lower(p.Value.(string))
case "FULLPATH":
val, err = p.fullPath()
case "SHORTPATH":
val, err = p.shortPath()
case "SHA1":
val, err = p.hash(FindHash(p.Name)())
}
if err != nil {
return nil, err
}
if val == nil {
return nil, &ErrNotImplemented{p.Name, p.Attribute}
}
return val, nil
}
// format runs a format function based on the value of the provided attribute.
func (p *FormatParams) format() (val interface{}, err error) {
switch p.Attribute {
case "name":
val = formatName(p.Args[0], p.Value.(string))
case "size":
val, err = p.formatSize()
case "time":
val, err = p.formatTime()
}
if err != nil {
return nil, err
}
if val == nil {
return nil, &ErrUnsupportedFormat{p.Args[0], p.Attribute}
}
return val, nil
}
// formatSize formats a size. Valid arguments include `KB`, `MB`, `GB` (case
// insensitive).
func (p *FormatParams) formatSize() (interface{}, error) {
size := p.Value.(int64)
switch strings.ToUpper(p.Args[0]) {
case "KB":
return fmt.Sprintf("%fkb", float64(size)/(1<<10)), nil
case "MB":
return fmt.Sprintf("%fmb", float64(size)/(1<<20)), nil
case "GB":
return fmt.Sprintf("%fgb", float64(size)/(1<<30)), nil
}
return nil, nil
}
// formatTime formats a time. Valid arguments include `UNIX` and `ISO` (case
// insensitive), or a custom layout layout. If a custom layout is provided, it
// must be set according to 2006-01-02T15:04:05.999999-07:00.
func (p *FormatParams) formatTime() (interface{}, error) {
switch strings.ToUpper(p.Args[0]) {
case "ISO":
return p.Info.ModTime().Format(time.RFC3339), nil
case "UNIX":
return p.Info.ModTime().Format(time.UnixDate), nil
default:
return p.Info.ModTime().Format(p.Args[0]), nil
}
}
// fullPath returns the full path of the current file. Only supports the
// `name` attribute.
func (p *FormatParams) fullPath() (interface{}, error) {
if p.Attribute != "name" {
return nil, nil
}
return p.Path, nil
}
// shortPath returns the short path of the current file. Only supports the
// `name` attribute.
func (p *FormatParams) shortPath() (interface{}, error) {
if p.Attribute != "name" {
return nil, nil
}
return p.Info.Name(), nil
}
// hash applies the provided hash algorithm h with ComputeHash.
func (p *FormatParams) hash(h hash.Hash) (interface{}, error) {
var (
err error
n int
result interface{}
)
if len(p.Args) == 0 || p.Args[0] == "" {
n = defaultHashLength
} else if strings.ToUpper(p.Args[0]) == "FULL" {
n = -1
} else if n, err = strconv.Atoi(p.Args[0]); err != nil {
return nil, err
}
if result, err = ComputeHash(p.Info, p.Path, h); err != nil {
return nil, err
}
return truncate(result.(string), n), nil
}
// DefaultFormatValue returns the default format value for the provided
// attribute attr based on path and info.
func DefaultFormatValue(attr, path string, info os.FileInfo) (value interface{}, err error) {
switch attr {
case "mode":
value = info.Mode()
case "name":
value = info.Name()
case "size":
value = info.Size()
case "time":
value = info.ModTime().Format(time.Stamp)
case "hash":
if value, err = ComputeHash(info, path, FindHash("SHA1")()); value != nil {
value = truncate(value.(string), defaultHashLength)
}
default:
err = fmt.Errorf("unknown attribute %s", attr)
}
return value, err
}
+94
View File
@@ -0,0 +1,94 @@
package transform
import (
"fmt"
"reflect"
"testing"
)
func TestTransform_Format(t *testing.T) {
type Expected struct {
val interface{}
err error
}
type Case struct {
params *FormatParams
expected Expected
}
// TODO: Add tests for the time and hash attribute.
cases := []Case{
{
params: &FormatParams{
Attribute: "size",
Path: "path",
Info: nil,
Value: int64(300),
Name: "format",
Args: []string{"kb"},
},
expected: Expected{
val: fmt.Sprintf("%fkb", float64(300)/(1<<10)),
err: nil,
},
},
{
params: &FormatParams{
Attribute: "size",
Path: "path",
Info: nil,
Value: int64(300),
Name: "format",
Args: []string{"kilobytes"},
},
expected: Expected{
val: nil,
err: &ErrUnsupportedFormat{"kilobytes", "size"},
},
},
{
params: &FormatParams{
Attribute: "name",
Path: "path",
Info: nil,
Value: "VALUE",
Name: "format",
Args: []string{"lower"},
},
expected: Expected{val: "value", err: nil},
},
{
params: &FormatParams{
Attribute: "name",
Path: "path",
Info: nil,
Value: "value",
Name: "upper",
Args: []string{},
},
expected: Expected{val: "VALUE", err: nil},
},
{
params: &FormatParams{
Attribute: "name",
Path: "path",
Info: nil,
Value: "value",
Name: "fullpath",
Args: []string{},
},
expected: Expected{val: "path", err: nil},
},
}
for _, c := range cases {
val, err := Format(c.params)
if !(reflect.DeepEqual(val, c.expected.val) &&
reflect.DeepEqual(err, c.expected.err)) {
t.Fatalf("\nExpected: %v, %v\n Got: %v, %v",
c.expected.val, c.expected.err,
val, err)
}
}
}
+148
View File
@@ -0,0 +1,148 @@
package transform
import (
"hash"
"os"
"reflect"
"strconv"
"strings"
"time"
)
// ParseParams holds the params for a parse-modifier function.
type ParseParams struct {
Attribute string
Value interface{}
Name string
Args []string
}
// Parse runs the associated modifier function for the provided parameters.
// Depending on the type of p.Value, we may recursively run this method
// on every element of the structure.
//
// We're using reflect _quite_ heavily for this, meaning it's kind of unsafe,
// it'd be great if we could find another solution while keeping it as
// abstract as it is.
func Parse(p *ParseParams) (val interface{}, err error) {
kind := reflect.TypeOf(p.Value).Kind()
// If we have a slice/array, recursively run Parse on each element.
if kind == reflect.Slice || kind == reflect.Array {
s := reflect.ValueOf(p.Value)
for i := 0; i < s.Len(); i++ {
p.Value = s.Index(i).Interface()
if val, err = Parse(p); err != nil {
return nil, err
}
s.Index(i).Set(reflect.ValueOf(val))
}
return s.Interface(), nil
}
// If we have a map, recursively run Parse on each *key* and create a new
// map out of the return values.
if kind == reflect.Map {
result := reflect.MakeMap(reflect.TypeOf(p.Value))
for _, key := range reflect.ValueOf(p.Value).MapKeys() {
p.Value = key.Interface()
if val, err = Parse(p); err != nil {
return nil, err
}
result.SetMapIndex(reflect.ValueOf(val), reflect.ValueOf(true))
}
return result.Interface(), nil
}
// Not a slice nor a map.
switch strings.ToUpper(p.Name) {
case "FORMAT":
val, err = p.format()
case "UPPER":
val = upper(p.Value.(string))
case "LOWER":
val = lower(p.Value.(string))
case "SHA1":
val, err = p.hash(FindHash(p.Name)())
}
if err != nil {
return nil, err
}
if val == nil {
return nil, &ErrNotImplemented{p.Name, p.Attribute}
}
return val, nil
}
// format runs the correct format function based on the provided attribute.
func (p *ParseParams) format() (val interface{}, err error) {
switch p.Attribute {
case "name":
val = formatName(p.Args[0], p.Value.(string))
case "size":
val, err = p.formatSize()
case "time":
val, err = p.formatTime()
}
if err != nil {
return nil, err
}
if val == nil {
return nil, &ErrUnsupportedFormat{p.Args[0], p.Attribute}
}
return val, nil
}
// formatSize formats the size attribute. Valid arguments include `B`, `KB`,
// `MB`, and `GB` (case insensitive).
func (p *ParseParams) formatSize() (interface{}, error) {
size, err := strconv.ParseFloat(p.Value.(string), 64)
if err != nil {
return nil, err
}
switch strings.ToUpper(p.Args[0]) {
case "B":
size *= 1
case "KB":
size *= 1 << 10
case "MB":
size *= 1 << 20
case "GB":
size *= 1 << 30
default:
return nil, nil
}
return size, nil
}
// formatTime formats the time attribute. Valid arguments include `ISO`,
// `UNIX`, (case insensitive) or a custom layout. If a custom layout is
// provided, it must be set according to 2006-01-02T15:04:05.999999-07:00.
func (p *ParseParams) formatTime() (interface{}, error) {
var (
t time.Time
err error
)
switch strings.ToUpper(p.Args[0]) {
case "ISO":
t, err = time.Parse(time.RFC3339, p.Value.(string))
case "UNIX":
t, err = time.Parse(time.UnixDate, p.Value.(string))
default:
t, err = time.Parse(p.Args[0], p.Value.(string))
}
if err != nil {
return nil, err
}
return t, nil
}
func (p *ParseParams) hash(h hash.Hash) (interface{}, error) {
info, err := os.Stat(p.Value.(string))
if err != nil {
return nil, err
}
return ComputeHash(info, p.Value.(string), h)
}
+31
View File
@@ -0,0 +1,31 @@
package transform
import (
"reflect"
"testing"
)
type ParseOutput struct {
val interface{}
err error
}
type ParseCase struct {
params *ParseParams
expected ParseOutput
}
func TestTransform_Parse(t *testing.T) {
// TODO: Complete this.
cases := []ParseCase{}
for _, c := range cases {
val, err := Parse(c.params)
if !(reflect.DeepEqual(val, c.expected.val) &&
reflect.DeepEqual(err, c.expected.err)) {
t.Fatalf("\nExpected: %v, %v\n Got: %v, %v",
c.expected.val, c.expected.err,
val, err)
}
}
}