commit c58133635692b65c12bd174e4916715d63e4ea2b Author: wehub-resource-sync Date: Mon Jul 13 12:32:14 2026 +0800 chore: import upstream snapshot with attribution diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml new file mode 100644 index 0000000..eda422b --- /dev/null +++ b/.github/workflows/go.yml @@ -0,0 +1,31 @@ +name: Go + +on: + push: + branches: [ master ] + pull_request: + branches: [ master ] + +jobs: + build: + runs-on: ubuntu-latest + strategy: + matrix: + go-version: [ '1.19', '1.20', '1.21' ] + + steps: + - uses: actions/checkout@v3 + + - name: Set up Go ${{ matrix.go-version }} + uses: actions/setup-go@v3 + with: + go-version: ${{ matrix.go-version }} + + - name: Install dependencies + run: go get . + + - name: Build + run: make + + - name: Test + run: make test diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..6b1f1ba --- /dev/null +++ b/.gitignore @@ -0,0 +1,45 @@ +# Compiled Object files, Static and Dynamic libs (Shared Objects) +*.o +*.a +*.so + +# Folders +_obj +_test + +# Architecture specific extensions/prefixes +*.[568vq] +[568vq].out + +*.cgo1.go +*.cgo2.c +_cgo_defun.c +_cgo_gotypes.go +_cgo_export.* + +_testmain.go + +*.exe +*.test +*.prof + +# Output of the go coverage tool, specifically when used with LiteIDE +*.out + +# Project-local glide cache, RE: https://github.com/Masterminds/glide/issues/736 +.glide/ + +# Compiled binaries +fsql +debug +main +dist/ + +# Allow cmd/fsql +!cmd/fsql + +# Editor workspace directories +.vscode + +# Coverage output +coverage.txt diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..7d65518 --- /dev/null +++ b/LICENSE @@ -0,0 +1,21 @@ +The MIT License (MIT) + +Copyright (c) 2017 Kashav Madan + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..39b95cb --- /dev/null +++ b/Makefile @@ -0,0 +1,105 @@ +PREFIX ?= $(shell pwd) + +NAME = fsql +PKG = github.com/kashav/$(NAME) +MAIN = $(PKG)/cmd/$(NAME) + +DIST_DIR := ${PREFIX}/dist +DIST_DIRS := find . -type d | sed "s|^\./||" | grep -v \\. | tr '\n' '\0' | xargs -0 -I '{}' + +SRCS := $(shell find . -type f -name '*.go') +PKGS := $(shell go list ./...) + +VERSION := $(shell cat VERSION) +GITCOMMIT := $(shell git rev-parse --short HEAD) +ifneq ($(shell git status --porcelain --untracked-files=no),) + GITCOMMIT := $(GITCOMMIT)-dirty +endif + +LDFLAGS := ${LDFLAGS} \ + -X $(PKG)/meta.GITCOMMIT=${GITCOMMIT} \ + -X $(PKG)/meta.VERSION=${VERSION} + +.PHONY: all +all: $(NAME) + +$(NAME): $(SRCS) VERSION + @echo "+ $@" + @go build -ldflags "${LDFLAGS}" -o $(NAME) -v $(MAIN) + +.PHONY: install +install: + @echo "+ $@" + @go install $(PKGS) + +.PHONY: get-tools +get-tools: + @echo "+ $@" + @go get -u -v golang.org/x/lint/golint + +.PHONY: clean +clean: + @echo "+ $@" + $(RM) $(NAME) + $(RM) -r $(DIST_DIR) + +.PHONY: fmt +fmt: + @echo "+ $@" + @test -z "$$(gofmt -s -l . 2>&1 | tee /dev/stderr)" || \ + (echo >&2 "+ please format Go code with 'gofmt -s', or use 'make fmt-save'" && false) + +.PHONY: fmt-save +fmt-save: + @echo "+ $@" + @gofmt -s -l . 2>&1 | xargs gofmt -s -l -w + +.PHONY: vet +vet: + @echo "+ $@" + @go vet $(PKGS) + +.PHONY: lint +lint: + @echo "+ $@" + $(if $(shell which golint || echo ''),, \ + $(error Please install golint: `make get-tools`)) + @test -z "$$(golint ./... 2>&1 | grep -v mock/ | tee /dev/stderr)" + +.PHONY: test +test: + @echo "+ $@" + @go test -race $(PKGS) + +.PHONY: coverage +coverage: + @echo "+ $@" + @for pkg in $(PKGS); do \ + go test -test.short -race -coverprofile="../../../$$pkg/coverage.txt" $${pkg} || exit 1; \ + done + +.PHONY: bootstrap-dist +bootstrap-dist: + @echo "+ $@" + @go get -u -v github.com/franciscocpg/gox + +.PHONY: build-all +build-all: $(SRCS) VERSION + @echo "+ $@" + @gox -verbose \ + -ldflags "${LDFLAGS}" \ + -os="darwin freebsd netbsd openbsd linux solaris windows" \ + -arch="386 amd64 arm arm64" \ + -osarch="!darwin/386 !darwin/arm" \ + -output="$(DIST_DIR)/$(NAME)-{{.OS}}-{{.Arch}}/{{.Dir}}" $(MAIN) + +.PHONY: dist +dist: clean build-all + @echo "+ $@" + @cd $(DIST_DIR) && \ + $(DIST_DIRS) cp ../LICENSE {} && \ + $(DIST_DIRS) cp ../README.md {} && \ + $(DIST_DIRS) tar -zcf fsql-${VERSION}-{}.tar.gz {} && \ + $(DIST_DIRS) zip -r -q fsql-${VERSION}-{}.zip {} && \ + $(DIST_DIRS) rm -rf {} && \ + cd .. diff --git a/README.md b/README.md new file mode 100644 index 0000000..c9319dc --- /dev/null +++ b/README.md @@ -0,0 +1,341 @@ +# fsql [![Go](https://github.com/kashav/fsql/actions/workflows/go.yml/badge.svg)](https://github.com/kashav/fsql/actions/workflows/go.yml) + +>Search through your filesystem with SQL-esque queries. + +## Contents + +- [Demo](#demo) +- [Installation](#installation) +- [Usage](#usage) +- [Query Syntax](#query-syntax) +- [Examples](#usage-examples) +- [Contribute](#contribute) +- [License](#license) + +## Demo + +[![fsql.gif](./media/fsql.gif)](https://asciinema.org/a/120534) + +## Installation + +#### Binaries + +[View latest release](https://github.com/kashav/fsql/releases/latest). + +#### Via Go + +```sh +$ go get -u -v github.com/kashav/fsql/... +$ which fsql +$GOPATH/bin/fsql +``` + +#### Via Homebrew + +```sh +$ brew install fsql +$ which fsql +/usr/local/bin/fsql +``` + +#### Build manually + +```sh +$ git clone https://github.com/kashav/fsql.git $GOPATH/src/github.com/kashav/fsql +$ cd $_ # $GOPATH/src/github.com/kashav/fsql +$ make +$ ./fsql +``` + +## Usage + +fsql expects a single query via stdin. You may also choose to use fsql in interactive mode. + +View the usage dialogue with the `-help` flag. + +```sh +$ fsql -help +usage: fsql [options] [query] + -v print version and exit (shorthand) + -version + print version and exit +``` + +## Query syntax + +In general, each query requires a `SELECT` clause (to specify which attributes will be shown), a `FROM` clause (to specify which directories to search), and a `WHERE` clause (to specify conditions to test against). + +```console +>>> SELECT attribute, ... FROM source, ... WHERE condition; +``` + +You may choose to omit the `SELECT` and `WHERE` clause. + +If you're providing your query via stdin, quotes are **not** required, however you'll have to escape _reserved_ characters (e.g. `*`, `<`, `>`, etc). + +### Attribute + +Currently supported attributes include `name`, `size`, `time`, `hash`, `mode`. + +Use `all` or `*` to choose all; if no attribute is provided, this is chosen by default. + +**Examples**: + +Each group features a set of equivalent clauses. + +```console +>>> SELECT name, size, time ... +>>> name, size, time ... +``` + +```console +>>> SELECT all FROM ... +>>> all FROM ... +>>> FROM ... +``` + +### Source + +Each source should be a relative or absolute path to a directory on your machine. + +Source paths may include environment variables (e.g. `$GOPATH`) or tildes (`~`). Use a hyphen (`-`) to exclude a directory. Source paths also support usage of [glob patterns](https://en.wikipedia.org/wiki/Glob_(programming)). + +In the case that a directory begins with a hyphen (e.g. `-foo`), use the following to include it as a source: + +```console +>>> ... FROM ./-foo ... +``` + +**Examples**: + +```console +>>> ... FROM . ... +``` + +```console +>>> ... FROM ~/Desktop, ./*/**.go ... +``` + +```console +>>> ... FROM $GOPATH, -.git/ ... +``` + +### Condition + +#### Condition syntax + +A single condition is made up of 3 parts: an attribute, an operator, and a value. + +- **Attribute**: + + A valid attribute is any of the following: `name`, `size`, `mode`, `time`. + +- **Operator**: + + Each attribute has a set of associated operators. + + - `name`: + + | Operator | Description | + | :---: | --- | + | `=` | String equality | + | `<>` / `!=` | Synonymous to using `"NOT ... = ..."` | + | `IN` | Basic list inclusion | + | `LIKE` | Simple pattern matching. Use `%` to match zero, one, or multiple characters. Check that a string begins with a value: `%`, ends with a value: `%`, or contains a value: `%%`. | + | `RLIKE` | Pattern matching with regular expressions. | + + - `size` / `time`: + + - All basic algebraic operators: `>`, `>=`, `<`, `<=`, `=`, and `<>` / `!=`. + + - `hash`: + + - `=` or `<>` / `!=` + + - `mode`: + + - `IS` + + +- **Value**: + + If the value contains spaces, wrap the value in quotes (either single or double) or backticks. + + The default unit for `size` is bytes. + + The default format for `time` is `MMM DD YYYY HH MM` (e.g. `"Jan 02 2006 15 04"`). + + Use `mode` to test if a file is regular (`IS REG`) or if it's a directory (`IS DIR`). + + Use `hash` to compute and/or compare the hash value of a file. The default algorithm is `SHA1` + +#### Conjunction / Disjunction + +Use `AND` / `OR` to join conditions. Note that precedence is assigned based on order of appearance. + +This means `WHERE a AND b OR c` is **not** the same as `WHERE c OR b AND a`. Use parentheses to get around this behaviour, i.e. `WHERE a AND b OR c` **is** the same as `WHERE c OR (b AND a)`. + +**Examples**: + +```console +>>> ... WHERE name = main.go OR size = 5 ... +``` + +```console +>>> ... WHERE name = main.go AND size > 20 ... +``` + +#### Negation + +Use `NOT` to negate a condition. This keyword **must** precede the condition (e.g. `... WHERE NOT a ...`). + +Note that negating parenthesized conditions is currently not supported. However, this can easily be resolved by applying [De Morgan's laws](https://en.wikipedia.org/wiki/De_Morgan%27s_laws) to your query. For example, `... WHERE NOT (a AND b) ...` is _logically equivalent_ to `... WHERE NOT a OR NOT b ...` (the latter is actually more optimal, due to [lazy evaluation](https://en.wikipedia.org/wiki/Lazy_evaluation)). + +**Examples**: + +```console +>>> ... WHERE NOT name = main.go ... +``` + +### Attribute Modifiers + +Attribute modifiers are used to specify how input and output values should be processed. These functions are applied directly to attributes in the `SELECT` and `WHERE` clauses. + +The table below lists currently-supported modifiers. Note that the first parameter to `FORMAT` is always the attribute name. + +| Attribute | Modifier | Supported in `SELECT` | Supported in `WHERE` | +| :---: | --- | :---: | :---: | +| `hash` | `SHA1(, n)` | ✔️ | ✔️ | +| `name` | `UPPER` (synonymous to `FORMAT(, UPPER)`) | ✔️ | ✔️ | +| | `LOWER` (synonymous to `FORMAT(, LOWER)`) | ✔️ | ✔️ | +| | `FULLPATH` | ✔️ | | +| | `SHORTPATH` | ✔️ | | +| `size` | `FORMAT(, unit)` | ✔️ | ✔️ | +| `time` | `FORMAT(, layout)` | ✔️ | ✔️ | + + +- **`n`**: + + Specify the length of the hash value. Use a negative integer or `ALL` to display all digits. + +- **`unit`**: + + Specify the size unit. One of: `B` (byte), `KB` (kilobyte), `MB` (megabyte), or `GB` (gigabyte). + +- **`layout`**: + + Specify the time layout. One of: [`ISO`](https://en.wikipedia.org/wiki/ISO_8601), [`UNIX`](https://en.wikipedia.org/wiki/Unix_time), or [custom](https://golang.org/pkg/time/#Time.Format). Custom layouts must be provided in reference to the following date: `Mon Jan 2 15:04:05 -0700 MST 2006`. + +**Examples**: + +```console +>>> SELECT SHA1(hash, 20) ... +``` + +```console +>>> ... WHERE UPPER(name) ... +``` + +```console +>>> SELECT FORMAT(size, MB) ... +``` + +```console +>>> ... WHERE FORMAT(time, "Mon Jan 2 2006 15:04:05") ... +``` + +### Subqueries + +Subqueries allow for more complex condition statements. These queries are recursively evaluated while parsing. SELECTing multiple attributes in a subquery is not currently supported; if more than one attribute (or `all`) is provided, only the first attribute is used. + +Support for referencing superqueries is not yet implemented, see [#4](https://github.com/kashav/fsql/issues/4) if you'd like to help with this. + +**Examples**: + +```console +>>> ... WHERE name IN (SELECT name FROM ../foo) ... +``` + +## Usage Examples + +List all attributes of each directory in your home directory (note the escaped `*`): + +```console +$ fsql SELECT \* FROM ~ WHERE mode IS DIR +``` + +List the names of all files in the Desktop and Downloads directory that contain `csc` in the name: + +```console +$ fsql "SELECT name FROM ~/Desktop, ~/Downloads WHERE name LIKE %csc%" +``` + +List all files in the current directory that are also present in some other directory: + +```console +$ fsql +>>> SELECT all FROM . WHERE name IN ( +... SELECT name FROM ~/Desktop/files.bak/ +... ); +``` + +Passing queries via stdin without quotes is a bit of a pain, hopefully the next examples highlight that, my suggestion is to use interactive mode or wrap the query in quotes if you're doing anything with subqueries or attribute modifiers. + +List all files named `main.go` in `$GOPATH` which are larger than 10.5 kilobytes or smaller than 100 bytes: + +```console +$ fsql SELECT all FROM $GOPATH WHERE name = main.go AND \(FORMAT\(size, KB\) \>= 10.5 OR size \< 100\) +$ fsql "SELECT all FROM $GOPATH WHERE name = main.go AND (FORMAT(size, KB) >= 10.5 OR size < 100)" +$ fsql +>>> SELECT +... all +... FROM +... $GOPATH +... WHERE +... name = main.go +... AND ( +... FORMAT(size, KB) >= 10.5 +... OR size < 100 +... ) +... ; +``` + +List the name, size, and modification time of JavaScript files in the current directory that were modified after April 1st 2017: + +```console +$ fsql SELECT UPPER\(name\), FORMAT\(size, KB\), FORMAT\(time, ISO\) FROM . WHERE name LIKE %.js AND time \> \'Apr 01 2017 00 00\' +$ fsql "SELECT UPPER(name), FORMAT(size, KB), FORMAT(time, ISO) FROM . WHERE name LIKE %.js AND time > 'Apr 01 2017 00 00'" +$ fsql +>>> SELECT +... UPPER(name), +... FORMAT(size, KB), +... FORMAT(time, ISO) +... FROM +... . +... WHERE +... name LIKE %.js +... AND time > 'Apr 01 2017 00 00' +... ; +``` + +## Contribute + +This project is completely open source, feel free to [open an issue](https://github.com/kashav/fsql/issues) or [submit a pull request](https://github.com/kashav/fsql/pulls). + +Before submitting code, please ensure that tests are passing and the linter is happy. The following commands may be of use, refer to the [Makefile](./Makefile) to see what they do. + +```sh +$ make install \ + get-tools +$ make fmt \ + vet \ + lint +$ make test \ + coverage +$ make bootstrap-dist \ + dist +``` + +## License + +fsql source code is available under the [MIT license](./LICENSE). diff --git a/README.wehub.md b/README.wehub.md new file mode 100644 index 0000000..63ca040 --- /dev/null +++ b/README.wehub.md @@ -0,0 +1,7 @@ +# WeHub 来源说明 + +- 原始项目：`kashav/fsql` +- 原始仓库：https://github.com/kashav/fsql +- 导入方式：上游默认分支的最新快照 +- 原作者、版权和许可证信息以原始仓库及本仓库 LICENSE 为准 +- 本文件仅用于记录来源，不代表 WeHub 是原项目作者 diff --git a/VERSION b/VERSION new file mode 100644 index 0000000..cb0c939 --- /dev/null +++ b/VERSION @@ -0,0 +1 @@ +0.5.2 diff --git a/cmd/fsql/main.go b/cmd/fsql/main.go new file mode 100644 index 0000000..36c8c50 --- /dev/null +++ b/cmd/fsql/main.go @@ -0,0 +1,53 @@ +package main + +import ( + "flag" + "fmt" + "log" + "os" + "strings" + + "github.com/kashav/fsql" + "github.com/kashav/fsql/meta" + "github.com/kashav/fsql/terminal" +) + +var options struct { + version bool +} + +func readInput() string { + if len(flag.Args()) > 1 { + return strings.Join(flag.Args(), " ") + } + + return flag.Args()[0] +} + +func main() { + flag.Usage = func() { + fmt.Printf("usage: %s [options] [query]\n", os.Args[0]) + flag.PrintDefaults() + } + + flag.BoolVar(&options.version, "version", false, "print version and exit") + flag.BoolVar(&options.version, "v", false, + "print version and exit (shorthand)") + flag.Parse() + + if options.version { + fmt.Printf("%s\n", meta.Meta()) + os.Exit(0) + } + + if len(flag.Args()) == 0 { + if err := terminal.Start(); err != nil { + log.Fatal(err.Error()) + } + os.Exit(0) + } + + if err := fsql.Run(readInput()); err != nil { + log.Fatal(err.Error()) + } +} diff --git a/evaluate/compare.go b/evaluate/compare.go new file mode 100644 index 0000000..e67c7b4 --- /dev/null +++ b/evaluate/compare.go @@ -0,0 +1,150 @@ +package evaluate + +import ( + "fmt" + "regexp" + "strings" + "time" + + "github.com/kashav/fsql/tokenizer" + "github.com/kashav/fsql/transform" +) + +// cmpAlpha performs alphabetic comparison on a and b. +func cmpAlpha(o *Opts, a, b interface{}) (result bool, err error) { + switch o.Operator { + case tokenizer.Equals: + result = a.(string) == b.(string) + case tokenizer.NotEquals: + result = a.(string) != b.(string) + case tokenizer.Like: + aStr, bStr := a.(string), b.(string) + if strings.HasPrefix(bStr, "%") && strings.HasSuffix(bStr, "%") { + result = strings.Contains(aStr, bStr[1:len(bStr)-1]) + } else if strings.HasPrefix(bStr, "%") { + result = strings.HasSuffix(aStr, bStr[1:]) + } else if strings.HasSuffix(bStr, "%") { + result = strings.HasPrefix(aStr, bStr[:len(bStr)-1]) + } else { + result = strings.Contains(aStr, bStr) + } + case tokenizer.RLike: + result = regexp.MustCompile(b.(string)).MatchString(a.(string)) + case tokenizer.In: + switch t := b.(type) { + case map[interface{}]bool: + if _, ok := t[a.(string)]; ok { + result = true + } + case []string: + for _, el := range t { + if a.(string) == el { + result = true + } + } + case string: + for _, el := range strings.Split(t, ",") { + if a.(string) == el { + result = true + } + } + } + default: + err = &ErrUnsupportedOperator{o.Attribute, o.Operator} + } + return result, err +} + +// cmpNumeric performs numeric comparison on a and b. +func cmpNumeric(o *Opts, a, b interface{}) (result bool, err error) { + switch o.Operator { + case tokenizer.Equals: + result = a.(int64) == b.(int64) + case tokenizer.NotEquals: + result = a.(int64) != b.(int64) + case tokenizer.GreaterThanEquals: + result = a.(int64) >= b.(int64) + case tokenizer.GreaterThan: + result = a.(int64) > b.(int64) + case tokenizer.LessThanEquals: + result = a.(int64) <= b.(int64) + case tokenizer.LessThan: + result = a.(int64) < b.(int64) + case tokenizer.In: + if _, ok := b.(map[interface{}]bool)[a.(int64)]; ok { + result = true + } + default: + err = &ErrUnsupportedOperator{o.Attribute, o.Operator} + } + return result, err +} + +// cmpTime performs time comparison on a and b. +func cmpTime(o *Opts, a, b interface{}) (result bool, err error) { + switch o.Operator { + case tokenizer.Equals: + result = a.(time.Time).Equal(b.(time.Time)) + case tokenizer.NotEquals: + result = !a.(time.Time).Equal(b.(time.Time)) + case tokenizer.GreaterThanEquals: + result = a.(time.Time).After(b.(time.Time)) || a.(time.Time).Equal(b.(time.Time)) + case tokenizer.GreaterThan: + result = a.(time.Time).After(b.(time.Time)) + case tokenizer.LessThanEquals: + result = a.(time.Time).Before(b.(time.Time)) || a.(time.Time).Equal(b.(time.Time)) + case tokenizer.LessThan: + result = a.(time.Time).Before(b.(time.Time)) + case tokenizer.In: + if _, ok := b.(map[interface{}]bool)[a.(time.Time)]; ok { + result = true + } + default: + err = &ErrUnsupportedOperator{o.Attribute, o.Operator} + } + return result, err +} + +// cmpMode performs mode comparison with info and typ. +func cmpMode(o *Opts) (result bool, err error) { + if o.Operator != tokenizer.Is { + return false, &ErrUnsupportedOperator{o.Attribute, o.Operator} + } + switch strings.ToUpper(o.Value.(string)) { + case "DIR": + result = o.File.Mode().IsDir() + case "REG": + result = o.File.Mode().IsRegular() + default: + result = false + } + return result, err +} + +// cmpHash computes the hash of the current file and compares it with the +// provided value. +func cmpHash(o *Opts) (result bool, err error) { + hashType := "SHA1" + if len(o.Modifiers) > 0 { + hashType = o.Modifiers[0].Name + } + + hashFunc := transform.FindHash(hashType) + if hashFunc == nil { + return false, fmt.Errorf("unexpected hash algorithm %s", hashType) + } + h, err := transform.ComputeHash(o.File, o.Path, hashFunc()) + if err != nil { + return false, err + } + + switch o.Operator { + case tokenizer.Equals: + result = h == o.Value + case tokenizer.NotEquals: + result = h != o.Value + default: + err = &ErrUnsupportedOperator{o.Attribute, o.Operator} + } + return result, err +} diff --git a/evaluate/compare_test.go b/evaluate/compare_test.go new file mode 100644 index 0000000..759b310 --- /dev/null +++ b/evaluate/compare_test.go @@ -0,0 +1,423 @@ +package evaluate + +import ( + "os" + "reflect" + "testing" + "time" + + "github.com/kashav/fsql/tokenizer" +) + +func TestCmpAlpha(t *testing.T) { + type Input struct { + o Opts + a, b interface{} + } + + type Expected struct { + result bool + err error + } + + type Case struct { + input Input + expected Expected + } + + // TODO: Test for errors. + cases := []Case{ + { + input: Input{o: Opts{Operator: tokenizer.Equals}, a: "a", b: "a"}, + expected: Expected{result: true, err: nil}, + }, + { + input: Input{o: Opts{Operator: tokenizer.Equals}, a: "a", b: "b"}, + expected: Expected{result: false, err: nil}, + }, + { + input: Input{o: Opts{Operator: tokenizer.Equals}, a: "a", b: "A"}, + expected: Expected{result: false, err: nil}, + }, + + { + input: Input{o: Opts{Operator: tokenizer.NotEquals}, a: "a", b: "a"}, + expected: Expected{result: false, err: nil}, + }, + { + input: Input{o: Opts{Operator: tokenizer.NotEquals}, a: "a", b: "b"}, + expected: Expected{result: true, err: nil}, + }, + { + input: Input{o: Opts{Operator: tokenizer.NotEquals}, a: "a", b: "A"}, + expected: Expected{result: true, err: nil}, + }, + + { + input: Input{o: Opts{Operator: tokenizer.Like}, a: "abc", b: "%a%"}, + expected: Expected{result: true, err: nil}, + }, + { + input: Input{o: Opts{Operator: tokenizer.Like}, a: "aaa", b: "%b%"}, + expected: Expected{result: false, err: nil}, + }, + { + input: Input{o: Opts{Operator: tokenizer.Like}, a: "aaa", b: "%a"}, + expected: Expected{result: true, err: nil}, + }, + { + input: Input{o: Opts{Operator: tokenizer.Like}, a: "abc", b: "%a"}, + expected: Expected{result: false, err: nil}, + }, + { + input: Input{o: Opts{Operator: tokenizer.Like}, a: "abc", b: "a%"}, + expected: Expected{result: true, err: nil}, + }, + { + input: Input{o: Opts{Operator: tokenizer.Like}, a: "cba", b: "a%"}, + expected: Expected{result: false, err: nil}, + }, + { + input: Input{o: Opts{Operator: tokenizer.Like}, a: "a", b: "a"}, + expected: Expected{result: true, err: nil}, + }, + { + input: Input{o: Opts{Operator: tokenizer.Like}, a: "a", b: "b"}, + expected: Expected{result: false, err: nil}, + }, + + { + input: Input{o: Opts{Operator: tokenizer.RLike}, a: "a", b: ".*a.*"}, + expected: Expected{result: true, err: nil}, + }, + { + input: Input{o: Opts{Operator: tokenizer.RLike}, a: "a", b: "^$"}, + expected: Expected{result: false, err: nil}, + }, + { + input: Input{o: Opts{Operator: tokenizer.RLike}, a: "", b: "^$"}, + expected: Expected{result: true, err: nil}, + }, + { + input: Input{o: Opts{Operator: tokenizer.RLike}, a: "...", b: "[\\.]{3}"}, + expected: Expected{result: true, err: nil}, + }, + { + input: Input{o: Opts{Operator: tokenizer.RLike}, a: "aaa", b: "\\s+"}, + expected: Expected{result: false, err: nil}, + }, + + { + input: Input{o: Opts{Operator: tokenizer.In}, a: "a", b: map[interface{}]bool{"a": true}}, + expected: Expected{result: true, err: nil}, + }, + { + input: Input{o: Opts{Operator: tokenizer.In}, a: "a", b: map[interface{}]bool{"a": false}}, + expected: Expected{result: true, err: nil}, + }, + { + input: Input{o: Opts{Operator: tokenizer.In}, a: "a", b: map[interface{}]bool{"b": true}}, + expected: Expected{result: false, err: nil}, + }, + { + input: Input{o: Opts{Operator: tokenizer.In}, a: "a", b: map[interface{}]bool{}}, + expected: Expected{result: false, err: nil}, + }, + } + + for _, c := range cases { + actual, err := cmpAlpha(&c.input.o, c.input.a, c.input.b) + if c.expected.err == nil { + if err != nil { + t.Fatalf("\nExpected no error\n Got %v", err) + } + if !reflect.DeepEqual(c.expected.result, actual) { + t.Fatalf("%v, %v, %v\nExpected: %v\n Got: %v", + c.input.o.Operator, c.input.a, c.input.b, c.expected.result, actual) + } + } else if !reflect.DeepEqual(c.expected.err, err) { + t.Fatalf("\nExpected %v\n Got %v", c.expected.err, err) + } + } +} + +func TestCmpNumeric(t *testing.T) { + type Input struct { + o Opts + a, b interface{} + } + + type Expected struct { + result bool + err error + } + + type Case struct { + input Input + expected Expected + } + + // TODO: Test for errors. + cases := []Case{ + { + input: Input{o: Opts{Operator: tokenizer.Equals}, a: int64(1), b: int64(1)}, + expected: Expected{result: true, err: nil}, + }, + { + input: Input{o: Opts{Operator: tokenizer.Equals}, a: int64(1), b: int64(2)}, + expected: Expected{result: false, err: nil}, + }, + + { + input: Input{o: Opts{Operator: tokenizer.NotEquals}, a: int64(1), b: int64(1)}, + expected: Expected{result: false, err: nil}, + }, + { + input: Input{o: Opts{Operator: tokenizer.NotEquals}, a: int64(1), b: int64(2)}, + expected: Expected{result: true, err: nil}, + }, + + { + input: Input{o: Opts{Operator: tokenizer.GreaterThanEquals}, a: int64(1), b: int64(1)}, + expected: Expected{result: true, err: nil}, + }, + { + input: Input{o: Opts{Operator: tokenizer.GreaterThanEquals}, a: int64(2), b: int64(1)}, + expected: Expected{result: true, err: nil}, + }, + { + input: Input{o: Opts{Operator: tokenizer.GreaterThanEquals}, a: int64(1), b: int64(2)}, + expected: Expected{result: false, err: nil}, + }, + + { + input: Input{o: Opts{Operator: tokenizer.GreaterThan}, a: int64(1), b: int64(1)}, + expected: Expected{result: false, err: nil}, + }, + { + input: Input{o: Opts{Operator: tokenizer.GreaterThan}, a: int64(2), b: int64(1)}, + expected: Expected{result: true, err: nil}, + }, + { + input: Input{o: Opts{Operator: tokenizer.GreaterThan}, a: int64(1), b: int64(2)}, + expected: Expected{result: false, err: nil}, + }, + + { + input: Input{o: Opts{Operator: tokenizer.LessThanEquals}, a: int64(1), b: int64(1)}, + expected: Expected{result: true, err: nil}, + }, + { + input: Input{o: Opts{Operator: tokenizer.LessThanEquals}, a: int64(2), b: int64(1)}, + expected: Expected{result: false, err: nil}, + }, + { + input: Input{o: Opts{Operator: tokenizer.LessThanEquals}, a: int64(1), b: int64(2)}, + expected: Expected{result: true, err: nil}, + }, + + { + input: Input{o: Opts{Operator: tokenizer.LessThan}, a: int64(1), b: int64(1)}, + expected: Expected{result: false, err: nil}, + }, + { + input: Input{o: Opts{Operator: tokenizer.LessThan}, a: int64(2), b: int64(1)}, + expected: Expected{result: false, err: nil}, + }, + { + input: Input{o: Opts{Operator: tokenizer.LessThan}, a: int64(1), b: int64(2)}, + expected: Expected{result: true, err: nil}, + }, + + { + input: Input{o: Opts{Operator: tokenizer.In}, a: int64(1), b: map[interface{}]bool{int64(1): true}}, + expected: Expected{result: true, err: nil}, + }, + { + input: Input{o: Opts{Operator: tokenizer.In}, a: int64(1), b: map[interface{}]bool{int64(1): false}}, + expected: Expected{result: true, err: nil}, + }, + { + input: Input{o: Opts{Operator: tokenizer.In}, a: int64(1), b: map[interface{}]bool{int64(2): true}}, + expected: Expected{result: false, err: nil}, + }, + { + input: Input{o: Opts{Operator: tokenizer.In}, a: int64(1), b: map[interface{}]bool{}}, + expected: Expected{result: false, err: nil}, + }, + } + + for _, c := range cases { + actual, err := cmpNumeric(&c.input.o, c.input.a, c.input.b) + if c.expected.err == nil { + if err != nil { + t.Fatalf("\nExpected no error\n Got %v", err) + } + if !reflect.DeepEqual(c.expected.result, actual) { + t.Fatalf("%v, %v, %v\nExpected: %v\n Got: %v", + c.input.o.Operator, c.input.a, c.input.b, c.expected.result, actual) + } + } else if !reflect.DeepEqual(c.expected.err, err) { + t.Fatalf("\nExpected %v\n Got %v", c.expected.err, err) + } + } +} + +func TestCmpTime(t *testing.T) { + type Input struct { + o Opts + a, b interface{} + } + + type Expected struct { + result bool + err error + } + + type Case struct { + input Input + expected Expected + } + + // TODO: Test for errors. + cases := []Case{ + { + input: Input{ + o: Opts{Operator: tokenizer.Equals}, + a: time.Now().Round(time.Minute), + b: time.Now().Round(time.Minute), + }, + expected: Expected{result: true, err: nil}, + }, + { + input: Input{ + o: Opts{Operator: tokenizer.Equals}, + a: time.Now().Add(time.Hour), + b: time.Now().Add(-1 * time.Hour), + }, + expected: Expected{result: false, err: nil}, + }, + + { + input: Input{ + o: Opts{Operator: tokenizer.NotEquals}, + a: time.Now().Round(time.Minute), + b: time.Now().Round(time.Minute), + }, + expected: Expected{result: false, err: nil}, + }, + { + input: Input{ + o: Opts{Operator: tokenizer.NotEquals}, + a: time.Now().Add(time.Hour), + b: time.Now().Add(-1 * time.Hour), + }, + expected: Expected{result: true, err: nil}, + }, + + { + input: Input{ + o: Opts{Operator: tokenizer.In}, + a: time.Now().Round(time.Minute), + b: map[interface{}]bool{time.Now().Round(time.Minute): true}, + }, + expected: Expected{result: true, err: nil}, + }, + { + input: Input{ + o: Opts{Operator: tokenizer.In}, + a: time.Now().Round(time.Minute), + b: map[interface{}]bool{time.Now().Add(time.Hour): true}, + }, + expected: Expected{result: false, err: nil}, + }, + } + + for _, c := range cases { + actual, err := cmpTime(&c.input.o, c.input.a, c.input.b) + if c.expected.err == nil { + if err != nil { + t.Fatalf("\nExpected no error\n Got %v", err) + } + if !reflect.DeepEqual(c.expected.result, actual) { + t.Fatalf("%v, %v, %v\nExpected: %v\n Got: %v", + c.input.o.Operator, c.input.a, c.input.b, c.expected.result, actual) + } + } else if !reflect.DeepEqual(c.expected.err, err) { + t.Fatalf("\nExpected %v\n Got %v", c.expected.err, err) + } + } + +} + +func TestCmpMode(t *testing.T) { + type Input struct { + o Opts + file os.FileInfo + typ interface{} + } + + type Expected struct { + result bool + err error + } + + type Case struct { + input Input + expected Expected + } + + // TODO: Complete these cases. + cases := []Case{} + + for _, c := range cases { + actual, err := cmpMode(&c.input.o) + if c.expected.err == nil { + if err != nil { + t.Fatalf("\nExpected no error\n Got %v", err) + } + if !reflect.DeepEqual(c.expected.result, actual) { + t.Fatalf("%v, %v, %v\nExpected: %v\n Got: %v", + c.input.o.Operator, c.input.file, c.input.typ, c.expected.result, actual) + } + } else if !reflect.DeepEqual(c.expected.err, err) { + t.Fatalf("\nExpected %v\n Got %v", c.expected.err, err) + } + } +} + +func TestCmpHash(t *testing.T) { + type Input struct { + o Opts + file os.FileInfo + typ interface{} + } + + type Expected struct { + result bool + err error + } + + type Case struct { + input Input + expected Expected + } + + // TODO: Complete these cases. + cases := []Case{} + + for _, c := range cases { + actual, err := cmpHash(&c.input.o) + if c.expected.err == nil { + if err != nil { + t.Fatalf("\nExpected no error\n Got %v", err) + } + if !reflect.DeepEqual(c.expected.result, actual) { + t.Fatalf("%v, %v, %v\nExpected: %v\n Got: %v", + c.input.o.Operator, c.input.file, c.input.typ, c.expected.result, actual) + } + } else if !reflect.DeepEqual(c.expected.err, err) { + t.Fatalf("\nExpected %v\n Got %v", c.expected.err, err) + } + } +} diff --git a/evaluate/error.go b/evaluate/error.go new file mode 100644 index 0000000..5884064 --- /dev/null +++ b/evaluate/error.go @@ -0,0 +1,38 @@ +package evaluate + +import ( + "fmt" + + "github.com/kashav/fsql/tokenizer" +) + +// ErrUnsupportedAttribute represents an unsupported attribute error. +type ErrUnsupportedAttribute struct { + Attribute string +} + +func (e *ErrUnsupportedAttribute) Error() string { + return fmt.Sprintf("unsupported attribute %s", e.Attribute) +} + +// ErrUnsupportedType represents an unsupported type error. +type ErrUnsupportedType struct { + Attribute string + Value interface{} +} + +func (e *ErrUnsupportedType) Error() string { + return fmt.Sprintf("unsupported type %T for for attribute %s", e.Value, + e.Attribute) +} + +// ErrUnsupportedOperator represents an unsupported operator error. +type ErrUnsupportedOperator struct { + Attribute string + Operator tokenizer.TokenType +} + +func (e *ErrUnsupportedOperator) Error() string { + return fmt.Sprintf("unsupported operator %s for attribute %s", + e.Operator.String(), e.Attribute) +} diff --git a/evaluate/error_test.go b/evaluate/error_test.go new file mode 100644 index 0000000..cb6815f --- /dev/null +++ b/evaluate/error_test.go @@ -0,0 +1,3 @@ +package evaluate + +// TODO diff --git a/evaluate/evaluate.go b/evaluate/evaluate.go new file mode 100644 index 0000000..2b07cde --- /dev/null +++ b/evaluate/evaluate.go @@ -0,0 +1,104 @@ +package evaluate + +import ( + "os" + "strconv" + "time" + + "github.com/kashav/fsql/tokenizer" +) + +// Opts represents a set of options used in the evaluate functions. +type Opts struct { + Path string + File os.FileInfo + Attribute string + Modifiers []Modifier + Operator tokenizer.TokenType + Value interface{} +} + +// Modifier represents an attribute modifier. +type Modifier struct { + Name string + Arguments []string +} + +// Evaluate runs the respective evaluate function for the provided options. +func Evaluate(o *Opts) (bool, error) { + switch o.Attribute { + case "name": + return evaluateName(o) + case "size": + return evaluateSize(o) + case "time": + return evaluateTime(o) + case "mode": + return evaluateMode(o) + case "hash": + return evaluateHash(o) + } + return false, &ErrUnsupportedAttribute{o.Attribute} +} + +// evaluateName evaluates a Condition with attribute `name`. +func evaluateName(o *Opts) (bool, error) { + var a, b interface{} + switch o.Value.(type) { + case string, []string, map[interface{}]bool: + a = o.File.Name() + b = o.Value + default: + return false, &ErrUnsupportedType{o.Attribute, o.Value} + } + return cmpAlpha(o, a, b) +} + +// evaluateSize evaluates a Condition with attribute `size`. +func evaluateSize(o *Opts) (bool, error) { + var a, b interface{} + switch o.Value.(type) { + case float64: + a = o.File.Size() + b = int64(o.Value.(float64)) + case map[interface{}]bool: + a = o.File.Size() + b = o.Value + case string: + size, err := strconv.ParseFloat(o.Value.(string), 10) + if err != nil { + return false, err + } + a = o.File.Size() + b = int64(size) + default: + return false, &ErrUnsupportedType{o.Attribute, o.Value} + } + return cmpNumeric(o, a, b) +} + +// evaluateTime evaluates a Condition with attribute `time`. +func evaluateTime(o *Opts) (bool, error) { + var a, b interface{} + switch o.Value.(type) { + case string: + t, err := time.Parse("Jan 02 2006 15 04", o.Value.(string)) + if err != nil { + return false, err + } + a = o.File.ModTime() + b = t + case map[interface{}]bool, time.Time: + a = o.File.ModTime() + b = o.Value + default: + return false, &ErrUnsupportedType{o.Attribute, o.Value} + } + return cmpTime(o, a, b) +} + +// evaluateMode evaluates a Condition with attribute `mode`. +func evaluateMode(o *Opts) (bool, error) { return cmpMode(o) } + +// evaluateHash evaluates a Condition with attribute `hash`. +func evaluateHash(o *Opts) (bool, error) { return cmpHash(o) } diff --git a/evaluate/evaluate_test.go b/evaluate/evaluate_test.go new file mode 100644 index 0000000..cb6815f --- /dev/null +++ b/evaluate/evaluate_test.go @@ -0,0 +1,3 @@ +package evaluate + +// TODO diff --git a/fsql.go b/fsql.go new file mode 100644 index 0000000..1bb32b3 --- /dev/null +++ b/fsql.go @@ -0,0 +1,55 @@ +package fsql + +import ( + "bytes" + "fmt" + "os" + + "github.com/kashav/fsql/parser" +) + +// Run parses the input and executes the resultant query. +func Run(input string) error { + q, err := parser.Run(input) + if err != nil { + return err + } + + // Find length of the longest name to normalize name output. + var max = 0 + var results = make([]map[string]interface{}, 0) + + err = q.Execute( + func(path string, info os.FileInfo, result map[string]interface{}) { + results = append(results, result) + if !q.HasAttribute("name") { + return + } + if s, ok := result["name"].(string); ok && len(s) > max { + max = len(s) + } + }, + ) + if err != nil { + return err + } + + for _, result := range results { + var buf bytes.Buffer + for j, attribute := range q.Attributes { + // If the current attribute is "name", pad the output string by `max` + // spaces. + format := "%v" + if attribute == "name" { + format = fmt.Sprintf("%%-%ds", max) + } + buf.WriteString(fmt.Sprintf(format, result[attribute])) + if j != len(q.Attributes)-1 { + buf.WriteString("\t") + } + } + fmt.Printf("%s\n", buf.String()) + } + + return nil +} diff --git a/fsql_test.go b/fsql_test.go new file mode 100644 index 0000000..2433eb6 --- /dev/null +++ b/fsql_test.go @@ -0,0 +1,421 @@ +package fsql + +import ( + "bytes" + "crypto/sha1" + "encoding/hex" + "fmt" + "io" + "os" + "path/filepath" + "reflect" + "strings" + "testing" + "time" +) + +var files = map[string]*os.FileInfo{} + +func TestRun_All(t *testing.T) { + type Case struct { + query string + expected string + } + + cases := []Case{ + { + query: "SELECT all FROM ./testdata WHERE name = foo", + expected: fmt.Sprintf("%s\t-------\tfoo\n", + strings.Join(GetAttrs("foo", "mode", "size", "time"), "\t")), + }, + { + query: "SELECT all FROM ./testdata WHERE name LIKE waldo", + expected: fmt.Sprintf("%s\twaldo\n", + strings.Join(GetAttrs("foo/quuz/waldo", "mode", "size", "time", "hash"), "\t")), + }, + { + query: "SELECT all FROM ./testdata WHERE FORMAT(time, 'Jan 02 2006 15:04') > 'Jan 01 2999 00:00'", + expected: "", + }, + { + query: "SELECT all FROM ./testdata WHERE mode IS DIR", + expected: fmt.Sprintf( + strings.Repeat("%s\n", 8), + fmt.Sprintf( + "%s\t-------\t%-8s", + strings.Join(GetAttrs(".", "mode", "size", "time"), "\t"), + "testdata", + ), + fmt.Sprintf( + "%s\t-------\t%-8s", + strings.Join(GetAttrs("bar", "mode", "size", "time"), "\t"), + "bar", + ), + fmt.Sprintf( + "%s\t-------\t%-8s", + strings.Join(GetAttrs("bar/garply", "mode", "size", "time"), "\t"), + "garply", + ), + fmt.Sprintf( + "%s\t-------\t%-8s", + strings.Join(GetAttrs("bar/garply/xyzzy", "mode", "size", "time"), "\t"), + "xyzzy", + ), + fmt.Sprintf( + "%s\t-------\t%-8s", + strings.Join(GetAttrs("bar/garply/xyzzy/thud", "mode", "size", "time"), "\t"), + "thud", + ), + fmt.Sprintf( + "%s\t-------\t%-8s", + strings.Join(GetAttrs("foo", "mode", "size", "time"), "\t"), + "foo", + ), + fmt.Sprintf( + "%s\t-------\t%-8s", + strings.Join(GetAttrs("foo/quuz", "mode", "size", "time"), "\t"), + "quuz", + ), + fmt.Sprintf( + "%s\t-------\t%-8s", + strings.Join(GetAttrs("foo/quuz/fred", "mode", "size", "time"), "\t"), + "fred", + ), + ), + }, + } + + for _, c := range cases { + actual := DoRun(c.query) + if !reflect.DeepEqual(c.expected, actual) { + t.Fatalf("%s\nExpected:\n%v\nGot:\n%v", c.query, c.expected, actual) + } + } +} + +func TestRun_Multiple(t *testing.T) { + type Case struct { + query string + expected string + } + + cases := []Case{ + { + query: "SELECT name, size FROM ./testdata WHERE name = foo", + expected: fmt.Sprintf("foo\t%s\n", GetAttrs("foo", "size")[0]), + }, + { + query: "SELECT size, name FROM ./testdata WHERE name = foo", + expected: fmt.Sprintf("%s\tfoo\n", GetAttrs("foo", "size")[0]), + }, + { + query: "SELECT FULLPATH(name), size, time FROM ./testdata/foo", + expected: fmt.Sprintf( + strings.Repeat("%s\n", 7), + fmt.Sprintf( + "%-31s\t%s", + "testdata/foo", + strings.Join(GetAttrs("foo", "size", "time"), "\t"), + ), + fmt.Sprintf( + "%-31s\t%s", + "testdata/foo/quux", + strings.Join(GetAttrs("foo/quux", "size", "time"), "\t"), + ), + fmt.Sprintf( + "%-31s\t%s", + "testdata/foo/quuz", + strings.Join(GetAttrs("foo/quuz", "size", "time"), "\t"), + ), + fmt.Sprintf( + "%-31s\t%s", + "testdata/foo/quuz/fred", + strings.Join(GetAttrs("foo/quuz/fred", "size", "time"), "\t"), + ), + fmt.Sprintf( + "%-31s\t%s", + "testdata/foo/quuz/fred/.gitkeep", + strings.Join(GetAttrs("foo/quuz/fred/.gitkeep", "size", "time"), "\t"), + ), + fmt.Sprintf( + "%-31s\t%s", + "testdata/foo/quuz/waldo", + strings.Join(GetAttrs("foo/quuz/waldo", "size", "time"), "\t"), + ), + fmt.Sprintf( + "%-31s\t%s", + "testdata/foo/qux", + strings.Join(GetAttrs("foo/qux", "size", "time"), "\t"), + ), + ), + }, + } + + for _, c := range cases { + actual := DoRun(c.query) + if !reflect.DeepEqual(c.expected, actual) { + t.Fatalf("%s\nExpected:\n%v\nGot:\n%v", c.query, c.expected, actual) + } + } +} + +func TestRun_Name(t *testing.T) { + type Case struct { + query string + expected string + } + + cases := []Case{ + { + query: "SELECT name FROM ./testdata WHERE name REGEXP ^g.*", + expected: "garply\ngrault\n", + }, + { + query: "SELECT FULLPATH(name) FROM ./testdata WHERE name REGEXP ^b.*", + expected: "testdata/bar\ntestdata/baz\n", + }, + { + query: "SELECT UPPER(FULLPATH(name)) FROM ./testdata WHERE mode IS DIR", + expected: fmt.Sprintf( + strings.Repeat("%-30s\n", 8), + "TESTDATA", + "TESTDATA/BAR", + "TESTDATA/BAR/GARPLY", + "TESTDATA/BAR/GARPLY/XYZZY", + "TESTDATA/BAR/GARPLY/XYZZY/THUD", + "TESTDATA/FOO", + "TESTDATA/FOO/QUUZ", + "TESTDATA/FOO/QUUZ/FRED", + ), + }, + } + + for _, c := range cases { + actual := DoRun(c.query) + if !reflect.DeepEqual(c.expected, actual) { + t.Fatalf("%s\nExpected:\n%v\nGot:\n%v", c.query, c.expected, actual) + } + } +} + +func TestRun_Size(t *testing.T) { + type Case struct { + query string + expected string + } + + cases := []Case{ + { + query: "SELECT size FROM ./testdata WHERE name = foo", + expected: fmt.Sprintf("%s\n", GetAttrs("foo", "size")[0]), + }, + { + query: "SELECT FORMAT(size, KB) FROM ./testdata WHERE name = foo", + expected: fmt.Sprintf("%s\n", GetAttrs("foo", "size:kb")[0]), + }, + { + query: "SELECT FORMAT(size, MB) FROM ./testdata WHERE name = foo", + expected: fmt.Sprintf("%s\n", GetAttrs("foo", "size:mb")[0]), + }, + { + query: "SELECT FORMAT(size, GB) FROM ./testdata WHERE name = foo", + expected: fmt.Sprintf("%s\n", GetAttrs("foo", "size:gb")[0]), + }, + { + query: "SELECT size FROM ./testdata WHERE name LIKE qu", + expected: fmt.Sprintf( + strings.Repeat("%s\n", 3), + GetAttrs("foo/quux", "size")[0], + GetAttrs("foo/quuz", "size")[0], + GetAttrs("foo/qux", "size")[0], + ), + }, + } + + for _, c := range cases { + actual := DoRun(c.query) + if !reflect.DeepEqual(c.expected, actual) { + t.Fatalf("%s\nExpected:\n%v\nGot:\n%v", c.query, c.expected, actual) + } + } +} + +func TestRun_Time(t *testing.T) { + type Case struct { + query string + expected string + } + + cases := []Case{ + { + query: "SELECT time FROM ./testdata WHERE name = baz", + expected: fmt.Sprintf("%s\n", GetAttrs("baz", "time")[0]), + }, + { + query: "SELECT FORMAT(time, ISO) FROM ./testdata WHERE name = foo", + expected: fmt.Sprintf("%s\n", GetAttrs("foo", "time:iso")[0]), + }, + { + query: "SELECT FORMAT(time, 2006) FROM ./testdata WHERE NOT name LIKE .%", + expected: strings.Repeat(fmt.Sprintf("%s\n", GetAttrs(".", "time:year")[0]), 14), + }, + { + query: "SELECT time FROM ./testdata/foo/quuz", + expected: fmt.Sprintf( + strings.Repeat("%s\n", 4), + GetAttrs("foo/quuz", "time")[0], + GetAttrs("foo/quuz/fred", "time")[0], + GetAttrs("foo/quuz/fred/.gitkeep", "time")[0], + GetAttrs("foo/quuz/waldo", "time")[0], + ), + }, + } + + for _, c := range cases { + actual := DoRun(c.query) + if !reflect.DeepEqual(c.expected, actual) { + t.Fatalf("%s\nExpected:\n%v\nGot:\n%v", c.query, c.expected, actual) + } + } +} + +func TestRun_Mode(t *testing.T) { + type Case struct { + query string + expected string + } + + cases := []Case{ + { + query: "SELECT mode FROM ./testdata WHERE name = foo", + expected: fmt.Sprintf("%s\n", GetAttrs("foo", "mode")[0]), + }, + { + query: "SELECT mode FROM ./testdata WHERE name = baz", + expected: fmt.Sprintf("%s\n", GetAttrs("baz", "mode")[0]), + }, + { + query: "SELECT mode FROM ./testdata WHERE mode IS DIR", + expected: fmt.Sprintf(strings.Repeat("%s\n", 8), + GetAttrs("", "mode")[0], + GetAttrs("foo", "mode")[0], + GetAttrs("foo/quuz", "mode")[0], + GetAttrs("foo/quuz/fred", "mode")[0], + GetAttrs("bar", "mode")[0], + GetAttrs("bar/garply", "mode")[0], + GetAttrs("bar/garply/xyzzy", "mode")[0], + GetAttrs("bar/garply/xyzzy/thud", "mode")[0], + ), + }, + } + + for _, c := range cases { + actual := DoRun(c.query) + if !reflect.DeepEqual(c.expected, actual) { + t.Fatalf("%s\nExpected:\n\"%v\"\nGot:\n\"%v\"", c.query, c.expected, actual) + } + } +} + +func TestRun_Hash(t *testing.T) { + type Case struct { + query string + expected string + } + + // TODO + cases := []Case{} + + for _, c := range cases { + actual := DoRun(c.query) + if !reflect.DeepEqual(c.expected, actual) { + t.Fatalf("%s\nExpected:\n%v\nGot:\n%v", c.query, c.expected, actual) + } + } +} + +func GetAttrs(path string, attrs ...string) []string { + // If the files map is empty, walk ./testdata and populate it. + if len(files) == 0 { + if err := filepath.Walk( + "./testdata", + func(path string, info os.FileInfo, err error) error { + files[filepath.Clean(path)] = &info + return nil + }, + ); err != nil { + return []string{} + } + } + + path = filepath.Clean(fmt.Sprintf("testdata/%s", path)) + file, ok := files[path] + if !ok { + return []string{} + } + + result := make([]string, len(attrs)) + for i, attr := range attrs { + // Hard-coding modifiers works for the time being, but we might need a more + // elegant solution when we introduce new modifiers in the future. + switch attr { + case "hash": + b, err := os.ReadFile(path) + if err != nil { + return []string{} + } + h := sha1.New() + if _, err := h.Write(b); err != nil { + return []string{} + } + result[i] = hex.EncodeToString(h.Sum(nil))[:7] + case "mode": + result[i] = (*file).Mode().String() + case "size": + result[i] = fmt.Sprintf("%d", (*file).Size()) + case "size:kb", "size:mb", "size:gb": + size := (*file).Size() + switch attr[len(attr)-2:] { + case "kb": + result[i] = fmt.Sprintf("%fkb", float64(size)/(1<<10)) + case "mb": + result[i] = fmt.Sprintf("%fmb", float64(size)/(1<<20)) + case "gb": + result[i] = fmt.Sprintf("%fgb", float64(size)/(1<<30)) + } + case "time": + result[i] = (*file).ModTime().Format(time.Stamp) + case "time:iso": + result[i] = (*file).ModTime().Format(time.RFC3339) + case "time:year": + result[i] = (*file).ModTime().Format("2006") + } + } + return result +} + +// DoRun executes fsql.Run and returns the output. +func DoRun(query string) string { + stdout := os.Stdout + ch := make(chan string) + + r, w, err := os.Pipe() + if err != nil { + return "" + } + os.Stdout = w + + if err := Run(query); err != nil { + return "" + } + + go func() { + var buf bytes.Buffer + io.Copy(&buf, r) + ch <- buf.String() + }() + + w.Close() + os.Stdout = stdout + return <-ch +} diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..7be0568 --- /dev/null +++ b/go.mod @@ -0,0 +1,13 @@ +module github.com/kashav/fsql + +go 1.21 + +require ( + github.com/oleiade/lane v1.0.1 + golang.org/x/crypto v0.17.0 +) + +require ( + golang.org/x/sys v0.15.0 // indirect + golang.org/x/term v0.15.0 // indirect +) diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..46b3185 --- /dev/null +++ b/go.sum @@ -0,0 +1,8 @@ +github.com/oleiade/lane v1.0.1 h1:hXofkn7GEOubzTwNpeL9MaNy8WxolCYb9cInAIeqShU= +github.com/oleiade/lane v1.0.1/go.mod h1:IyTkraa4maLfjq/GmHR+Dxb4kCMtEGeb+qmhlrQ5Mk4= +golang.org/x/crypto v0.17.0 h1:r8bRNjWL3GshPW3gkd+RpvzWrZAwPS49OmTGZ/uhM4k= +golang.org/x/crypto v0.17.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4= +golang.org/x/sys v0.15.0 h1:h48lPFYpsTvQJZF4EKyI4aLHaev3CxivZmv7yZig9pc= +golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/term v0.15.0 h1:y/Oo/a/q3IXu26lQgl04j/gjuBDOBlx7X6Om1j2CPW4= +golang.org/x/term v0.15.0/go.mod h1:BDl952bC7+uMoWR75FIrCDx79TPU9oHkTZ9yRbYOrX0= diff --git a/media/fsql.gif b/media/fsql.gif new file mode 100644 index 0000000..18df529 Binary files /dev/null and b/media/fsql.gif differ diff --git a/meta/meta.go b/meta/meta.go new file mode 100644 index 0000000..659f7ce --- /dev/null +++ b/meta/meta.go @@ -0,0 +1,22 @@ +package meta + +import "fmt" + +// GITCOMMIT indicates which git hash the binary was built off of. +var GITCOMMIT string + +// VERSION indicates which version of the binary is running. +var VERSION string + +// Release holds the current release number, should match the value +// in $GOPATH/src/github.com/kashav/fsql/VERSION. +const Release = "0.5.2" + +// Meta returns the version/commit string. +func Meta() string { + version, commit := VERSION, GITCOMMIT + if commit == "" || version == "" { + version, commit = Release, "master" + } + return fmt.Sprintf("fsql version %v, built off %v", version, commit) +} diff --git a/parser/attribute.go b/parser/attribute.go new file mode 100644 index 0000000..895f3f5 --- /dev/null +++ b/parser/attribute.go @@ -0,0 +1,102 @@ +package parser + +import ( + "strings" + + "github.com/kashav/fsql/query" + "github.com/kashav/fsql/tokenizer" +) + +var allAttributes = []string{"mode", "size", "time", "hash", "name"} + +func isValidAttribute(attribute string) error { + for _, valid := range allAttributes { + if attribute == valid { + return nil + } + } + return &ErrUnknownToken{attribute} +} + +// parseAttrs parses the list of attributes passed to the SELECT clause. +func (p *parser) parseAttrs(attributes *[]string, modifiers *map[string][]query.Modifier) error { + for { + ident := p.expect(tokenizer.Identifier) + if ident == nil { + return p.currentError() + } + + if ident.Raw == "*" || ident.Raw == "all" { + *attributes = allAttributes + } else { + p.current = ident + + attrModifiers := make([]query.Modifier, 0) + attribute, err := p.parseAttr(&attrModifiers) + if err != nil { + return err + } + *attributes = append(*attributes, attribute.Raw) + (*modifiers)[attribute.Raw] = attrModifiers + } + + if p.expect(tokenizer.Comma) == nil { + break + } + } + return nil +} + +// parseAttr recursively parses an attribute's modifiers and returns the +// associated attribute. +func (p *parser) parseAttr(modifiers *[]query.Modifier) (*tokenizer.Token, error) { + ident := p.expect(tokenizer.Identifier) + if ident == nil { + return nil, p.currentError() + } + + // ident is a modifier name (e.g. `FORMAT`) iff the next token is an open + // paren, otherwise an attribute (e.g. `name`). + if token := p.expect(tokenizer.OpenParen); token == nil { + if err := isValidAttribute(ident.Raw); err != nil { + return nil, err + } + return ident, nil + } + + // In the case of chained modifiers, we want to recurse and parse each + // inner modifier first. parseAttribute returns the associated attribute that + // we're looking for. + attribute, err := p.parseAttr(modifiers) + if err != nil { + return nil, err + } + if attribute == nil { + return nil, p.currentError() + } + if err := isValidAttribute(attribute.Raw); err != nil { + return nil, err + } + + modifier := query.Modifier{ + Name: strings.ToUpper(ident.Raw), + Arguments: make([]string, 0), + } + + // Parse the modifier arguments. + for { + if token := p.expect(tokenizer.Identifier); token != nil { + modifier.Arguments = append(modifier.Arguments, token.Raw) + continue + } + + if token := p.expect(tokenizer.Comma); token != nil { + continue + } + + if token := p.expect(tokenizer.CloseParen); token != nil { + *modifiers = append(*modifiers, modifier) + return attribute, nil + } + } +} diff --git a/parser/attribute_test.go b/parser/attribute_test.go new file mode 100644 index 0000000..e42292c --- /dev/null +++ b/parser/attribute_test.go @@ -0,0 +1,256 @@ +package parser + +import ( + "io" + "reflect" + "testing" + + "github.com/kashav/fsql/query" + "github.com/kashav/fsql/tokenizer" +) + +func TestAttributeParser_ExpectCorrectAttributes(t *testing.T) { + type Expected struct { + attributes []string + err error + } + + type Case struct { + input string + expected Expected + } + + cases := []Case{ + { + input: "name", + expected: Expected{attributes: []string{"name"}, err: nil}, + }, + { + input: "name, size", + expected: Expected{ + attributes: []string{"name", "size"}, + err: nil, + }, + }, + { + input: "*", + expected: Expected{attributes: allAttributes, err: nil}, + }, + { + input: "all", + expected: Expected{attributes: allAttributes, err: nil}, + }, + { + input: "format(time, iso)", + expected: Expected{attributes: []string{"time"}, err: nil}, + }, + + { + input: "", + expected: Expected{err: io.ErrUnexpectedEOF}, + }, + { + input: "name,", + expected: Expected{err: io.ErrUnexpectedEOF}, + }, + { + input: "identifier", + expected: Expected{err: &ErrUnknownToken{"identifier"}}, + }, + } + + for _, c := range cases { + attributes := make([]string, 0) + modifiers := make(map[string][]query.Modifier) + + p := &parser{tokenizer: tokenizer.NewTokenizer(c.input)} + err := p.parseAttrs(&attributes, &modifiers) + + if c.expected.err == nil { + if err != nil { + t.Fatalf("\nExpected no error\n Got %v", err) + } + if !reflect.DeepEqual(c.expected.attributes, attributes) { + t.Fatalf("\nExpected %v\n Got %v", c.expected.attributes, attributes) + } + } else if !reflect.DeepEqual(c.expected.err, err) { + t.Fatalf("\nExpected %v\n Got %v", c.expected.err, err) + } + } +} + +func TestAttributeParser_ExpectCorrectModifiers(t *testing.T) { + type Expected struct { + modifiers map[string][]query.Modifier + err error + } + type Case struct { + input string + expected Expected + } + + cases := []Case{ + { + input: "name", + expected: Expected{ + modifiers: map[string][]query.Modifier{"name": {}}, + err: nil, + }, + }, + { + input: "upper(name)", + expected: Expected{ + modifiers: map[string][]query.Modifier{ + "name": { + { + Name: "UPPER", + Arguments: []string{}, + }, + }, + }, + err: nil, + }, + }, + { + input: "format(time, iso)", + expected: Expected{ + modifiers: map[string][]query.Modifier{ + "time": { + { + Name: "FORMAT", + Arguments: []string{"iso"}, + }, + }, + }, + err: nil, + }, + }, + { + input: "format(time, \"iso\")", + expected: Expected{ + modifiers: map[string][]query.Modifier{ + "time": { + { + Name: "FORMAT", + Arguments: []string{"iso"}, + }, + }, + }, + err: nil, + }, + }, + { + input: "lower(name), format(size, mb)", + expected: Expected{ + modifiers: map[string][]query.Modifier{ + "name": { + { + Name: "LOWER", + Arguments: []string{}, + }, + }, + "size": { + { + Name: "FORMAT", + Arguments: []string{"mb"}, + }, + }, + }, + err: nil, + }, + }, + { + input: "format(fullpath(name), lower)", + expected: Expected{ + modifiers: map[string][]query.Modifier{ + "name": { + { + Name: "FULLPATH", + Arguments: []string{}, + }, + { + Name: "FORMAT", + Arguments: []string{"lower"}, + }, + }, + }, + err: nil, + }, + }, + + // No function/parameter validation yet! + { + input: "foo(name)", + expected: Expected{ + modifiers: map[string][]query.Modifier{ + "name": {{ + Name: "FOO", + Arguments: []string{}, + }, + }, + }, + err: nil, + }, + }, + { + input: "format(size, tb)", + expected: Expected{ + modifiers: map[string][]query.Modifier{ + "size": { + { + Name: "FORMAT", + Arguments: []string{"tb"}, + }, + }, + }, + err: nil, + }, + }, + { + input: "format(size, kb, mb)", + expected: Expected{ + modifiers: map[string][]query.Modifier{ + "size": { + { + Name: "FORMAT", + Arguments: []string{"kb", "mb"}, + }, + }, + }, + err: nil, + }, + }, + + { + input: "", + expected: Expected{err: io.ErrUnexpectedEOF}, + }, + { + input: "lower(name),", + expected: Expected{err: io.ErrUnexpectedEOF}, + }, + { + input: "identifier", + expected: Expected{err: &ErrUnknownToken{"identifier"}}, + }, + } + + for _, c := range cases { + attributes := make([]string, 0) + modifiers := make(map[string][]query.Modifier) + + p := &parser{tokenizer: tokenizer.NewTokenizer(c.input)} + err := p.parseAttrs(&attributes, &modifiers) + + if c.expected.err == nil { + if err != nil { + t.Fatalf("\nExpected no error\n Got %v", err) + } + if !reflect.DeepEqual(c.expected.modifiers, modifiers) { + t.Fatalf("\nExpected %v\n Got %v", c.expected.modifiers, modifiers) + } + } else if !reflect.DeepEqual(c.expected.err, err) { + t.Fatalf("\nExpected %v\n Got %v", c.expected.err, err) + } + } +} diff --git a/parser/condition.go b/parser/condition.go new file mode 100644 index 0000000..9295626 --- /dev/null +++ b/parser/condition.go @@ -0,0 +1,210 @@ +package parser + +import ( + "errors" + "os" + + "github.com/oleiade/lane" + + "github.com/kashav/fsql/query" + "github.com/kashav/fsql/tokenizer" +) + +// parseConditionTree parses the condition tree passed to the WHERE clause. +func (p *parser) parseConditionTree() (*query.ConditionNode, error) { + stack := lane.NewStack() + errFailedToParse := errors.New("failed to parse conditions") + + for { + if p.current = p.tokenizer.Next(); p.current == nil { + break + } + + switch p.current.Type { + + case tokenizer.Not: + // TODO: Handle NOT (...), for the time being we proceed with the other + // tokens and handle the negation when parsing the condition. + fallthrough + + case tokenizer.Identifier: + condition, err := p.parseCondition() + if err != nil { + return nil, err + } + if condition == nil { + return nil, p.currentError() + } + + if condition.IsSubquery { + if err := p.parseSubquery(condition); err != nil { + return nil, err + } + } + + leafNode := &query.ConditionNode{Condition: condition} + if prevNode, ok := stack.Pop().(*query.ConditionNode); !ok { + stack.Push(leafNode) + } else if prevNode.Condition == nil { + prevNode.Right = leafNode + stack.Push(prevNode) + } else { + return nil, errFailedToParse + } + + case tokenizer.And, tokenizer.Or: + leftNode, ok := stack.Pop().(*query.ConditionNode) + if !ok { + return nil, errFailedToParse + } + + node := query.ConditionNode{ + Type: &p.current.Type, + Left: leftNode, + } + stack.Push(&node) + + case tokenizer.OpenParen: + stack.Push(nil) + + case tokenizer.CloseParen: + rightNode, ok := stack.Pop().(*query.ConditionNode) + if !ok { + return nil, errFailedToParse + } + + if rootNode, ok := stack.Pop().(*query.ConditionNode); !ok { + stack.Push(rightNode) + } else { + rootNode.Right = rightNode + stack.Push(rootNode) + } + + } + } + + if stack.Size() == 0 { + return nil, p.currentError() + } + + if stack.Size() > 1 { + return nil, errFailedToParse + } + + node, ok := stack.Pop().(*query.ConditionNode) + if !ok { + return nil, errFailedToParse + } + return node, nil +} + +// parseCondition parses and returns the next condition. +func (p *parser) parseCondition() (*query.Condition, error) { + cond := &query.Condition{} + + // If we find a NOT, negate the condition. + if p.expect(tokenizer.Not) != nil { + cond.Negate = true + } + + ident := p.expect(tokenizer.Identifier) + if ident == nil { + return nil, p.currentError() + } + p.current = ident + + var modifiers []query.Modifier + attr, err := p.parseAttr(&modifiers) + if err != nil { + return nil, err + } + cond.Attribute = attr.Raw + cond.AttributeModifiers = modifiers + + // If this condition has modifiers, then p.current was unset while parsing + // the modifier, se we set the current token manually. + if len(modifiers) > 0 { + p.current = p.tokenizer.Next() + } + if p.current == nil { + return nil, p.currentError() + } + cond.Operator = p.current.Type + p.current = nil + + // Parse subquery of format `(...)`. + if p.expect(tokenizer.OpenParen) != nil { + token := p.expect(tokenizer.Subquery) + if token == nil { + return nil, p.currentError() + } + cond.IsSubquery = true + cond.Value = token.Raw + if p.expect(tokenizer.CloseParen) == nil { + return nil, p.currentError() + } + return cond, nil + } + + // Parse list of values of format `[...]`. + if p.expect(tokenizer.OpenBracket) != nil { + values := make([]string, 0) + for { + if token := p.expect(tokenizer.Identifier); token != nil { + values = append(values, token.Raw) + } + if p.expect(tokenizer.Comma) != nil { + continue + } + if p.expect(tokenizer.CloseBracket) != nil { + break + } + } + cond.Value = values + return cond, nil + } + + // Not a list nor a subquery -> plain identifier! + token := p.expect(tokenizer.Identifier) + if token == nil { + return nil, p.currentError() + } + cond.Value = token.Raw + return cond, nil +} + +// parseSubquery parses a subquery by recursively evaluating it's condition(s). +// If the subquery contains references to aliases from the superquery, it's +// Subquery attribute is set. Otherwise, we evaluate it's Subquery and set +// it's Value to the result. +func (p *parser) parseSubquery(condition *query.Condition) error { + q, err := Run(condition.Value.(string)) + if err != nil { + return err + } + + // If the subquery has aliases, we'll have to parse the subquery against + // each file, so we don't do anything here. + if len(q.SourceAliases) > 0 { + condition.Subquery = q + return nil + } + + value := make(map[interface{}]bool, 0) + workFunc := func(path string, info os.FileInfo, res map[string]interface{}) { + for _, attr := range [...]string{"name", "size", "time", "mode"} { + if q.HasAttribute(attr) { + value[res[attr]] = true + return + } + } + } + + if err = q.Execute(workFunc); err != nil { + return err + } + + condition.Value = value + condition.IsSubquery = false + return nil +} diff --git a/parser/condition_test.go b/parser/condition_test.go new file mode 100644 index 0000000..31aa6ab --- /dev/null +++ b/parser/condition_test.go @@ -0,0 +1,408 @@ +package parser + +import ( + "errors" + "io" + "reflect" + "testing" + + "github.com/kashav/fsql/query" + "github.com/kashav/fsql/tokenizer" +) + +func TestConditionParser_ExpectCorrectCondition(t *testing.T) { + type Expected struct { + condition *query.Condition + err error + } + + type Case struct { + input string + expected Expected + } + + cases := []Case{ + { + input: "name LIKE foo%", + expected: Expected{ + condition: &query.Condition{ + Attribute: "name", + Operator: tokenizer.Like, + Value: "foo%", + }, + err: nil, + }, + }, + + { + input: "size = 10", + expected: Expected{ + condition: &query.Condition{ + Attribute: "size", + Operator: tokenizer.Equals, + Value: "10", + }, + err: nil, + }, + }, + + { + input: "mode IS dir", + expected: Expected{ + condition: &query.Condition{ + Attribute: "mode", + Operator: tokenizer.Is, + Value: "dir", + }, + err: nil, + }, + }, + + { + input: "format(time, iso) >= 2017-05-28T16:37:18Z", + expected: Expected{ + condition: &query.Condition{ + Attribute: "time", + AttributeModifiers: []query.Modifier{ + { + Name: "FORMAT", + Arguments: []string{"iso"}, + }, + }, + Operator: tokenizer.GreaterThanEquals, + Value: "2017-05-28T16:37:18Z", + }, + err: nil, + }, + }, + + { + input: "upper(name) != FOO", + expected: Expected{ + condition: &query.Condition{ + Attribute: "name", + AttributeModifiers: []query.Modifier{ + { + Name: "UPPER", + Arguments: []string{}, + }, + }, + Operator: tokenizer.NotEquals, + Value: "FOO", + }, + err: nil, + }, + }, + + { + input: "NOT name IN [foo,bar,baz]", + expected: Expected{ + condition: &query.Condition{ + Attribute: "name", + Operator: tokenizer.In, + Value: []string{"foo", "bar", "baz"}, + Negate: true, + }, + err: nil, + }, + }, + + // No attribute-operator validation yet (these 3 should /eventually/ throw + // some error)! + { + input: "time RLIKE '.*'", + expected: Expected{ + condition: &query.Condition{ + Attribute: "time", + Operator: tokenizer.RLike, + Value: ".*", + }, + err: nil, + }, + }, + + { + input: "size LIKE foo", + expected: Expected{ + condition: &query.Condition{ + Attribute: "size", + Operator: tokenizer.Like, + Value: "foo", + }, + err: nil, + }, + }, + + { + input: "time <> now", + expected: Expected{ + condition: &query.Condition{ + Attribute: "time", + Operator: tokenizer.NotEquals, + Value: "now", + }, + err: nil, + }, + }, + + { + input: "name =", + expected: Expected{err: io.ErrUnexpectedEOF}, + }, + + { + input: "file IS dir", + expected: Expected{err: &ErrUnknownToken{"file"}}, + }, + } + + for _, c := range cases { + p := &parser{tokenizer: tokenizer.NewTokenizer(c.input)} + actual, err := p.parseCondition() + + if c.expected.err == nil { + if err != nil { + t.Fatalf("\nExpected no error\n Got %v", err) + } + if !reflect.DeepEqual(c.expected.condition, actual) { + t.Fatalf("\nExpected %v\n Got %v", c.expected.condition, actual) + } + } else if !reflect.DeepEqual(c.expected.err, err) { + t.Fatalf("\nExpected %v\n Got %v", c.expected.err, err) + } + } +} + +func TestConditionParser_ExpectCorrectConditionTree(t *testing.T) { + type Expected struct { + node *query.ConditionNode + err error + } + + type Case struct { + input string + expected Expected + } + + // Not sure why, but compiler throws when attempting to take the memory + // address of any tokenizer.TokenType. + var ( + tmpAnd = tokenizer.And + tmpOr = tokenizer.Or + ) + + cases := []Case{ + { + input: "name LIKE foo%", + expected: Expected{ + node: &query.ConditionNode{ + Condition: &query.Condition{ + Attribute: "name", + Operator: tokenizer.Like, + Value: "foo%", + }, + }, + err: nil, + }, + }, + + { + input: "upper(name) = MAIN", + expected: Expected{ + node: &query.ConditionNode{ + Condition: &query.Condition{ + Attribute: "name", + AttributeModifiers: []query.Modifier{ + { + Name: "UPPER", + Arguments: []string{}, + }, + }, + Operator: tokenizer.Equals, + Value: "MAIN", + }, + }, + err: nil, + }, + }, + + { + input: "name LIKE %foo AND name <> bar.foo", + expected: Expected{ + node: &query.ConditionNode{ + Type: &tmpAnd, + Left: &query.ConditionNode{ + Condition: &query.Condition{ + Attribute: "name", + Operator: tokenizer.Like, + Value: "%foo", + }, + }, + Right: &query.ConditionNode{ + Condition: &query.Condition{ + Attribute: "name", + Operator: tokenizer.NotEquals, + Value: "bar.foo", + }, + }, + }, + err: nil, + }, + }, + + { + input: "size <= 10 OR NOT mode IS dir", + expected: Expected{ + node: &query.ConditionNode{ + Type: &tmpOr, + Left: &query.ConditionNode{ + Condition: &query.Condition{ + Attribute: "size", + Operator: tokenizer.LessThanEquals, + Value: "10", + }, + }, + Right: &query.ConditionNode{ + Condition: &query.Condition{ + Attribute: "mode", + Operator: tokenizer.Is, + Value: "dir", + Negate: true, + }, + }, + }, + err: nil, + }, + }, + + { + input: "size = 5 AND name = foo", + expected: Expected{ + node: &query.ConditionNode{ + Type: &tmpAnd, + Left: &query.ConditionNode{ + Condition: &query.Condition{ + Attribute: "size", + Operator: tokenizer.Equals, + Value: "5", + }, + }, + Right: &query.ConditionNode{ + Condition: &query.Condition{ + Attribute: "name", + Operator: tokenizer.Equals, + Value: "foo", + }, + }, + }, + err: nil, + }, + }, + + { + input: "format(size, mb) <= 2 AND (name = foo OR name = bar)", + expected: Expected{ + node: &query.ConditionNode{ + Type: &tmpAnd, + Left: &query.ConditionNode{ + Condition: &query.Condition{ + Attribute: "size", + AttributeModifiers: []query.Modifier{ + { + Name: "FORMAT", + Arguments: []string{"mb"}, + }, + }, + Operator: tokenizer.LessThanEquals, + Value: "2", + }, + }, + Right: &query.ConditionNode{ + Type: &tmpOr, + Left: &query.ConditionNode{ + Condition: &query.Condition{ + Attribute: "name", + Operator: tokenizer.Equals, + Value: "foo", + }, + }, + Right: &query.ConditionNode{ + Condition: &query.Condition{ + Attribute: "name", + Operator: tokenizer.Equals, + Value: "bar", + }, + }, + }, + }, + err: nil, + }, + }, + + { + input: "name = foo AND NOT (name = bar OR name = baz)", + expected: Expected{ + node: &query.ConditionNode{}, + err: &ErrUnexpectedToken{ + Expected: tokenizer.Identifier, + Actual: tokenizer.OpenParen, + }, + }, + }, + + { + input: "size = 5 AND ()", + expected: Expected{err: errors.New("failed to parse conditions")}, + }, + + // FIXME: The following case /should/ throw EOF (it doesn't right now). + // Case{input: "name = foo AND", expected: Expected{err: io.ErrUnexpectedEOF}}, + } + + for _, c := range cases { + p := &parser{tokenizer: tokenizer.NewTokenizer(c.input)} + actual, err := p.parseConditionTree() + + if c.expected.err == nil { + if err != nil { + t.Fatalf("\nExpected no error\n Got %v", err) + } + if !reflect.DeepEqual(c.expected.node, actual) { + t.Fatalf("\nExpected %v\n Got %v", c.expected.node, actual) + } + } else if !reflect.DeepEqual(c.expected.err, err) { + t.Fatalf("\nExpected %v\n Got %v", c.expected.err, err) + } + } +} + +func TestConditionParser_ExpectCorrectSubquery(t *testing.T) { + type Expected struct { + condition *query.Condition + err error + } + + type Case struct { + input *query.Condition + expected Expected + } + + // TODO: Complete these cases. This test relies on testdata fixtures. + cases := []Case{} + + for _, c := range cases { + p := &parser{tokenizer: tokenizer.NewTokenizer("")} + err := p.parseSubquery(c.input) + + if c.expected.err == nil { + if err != nil { + t.Fatalf("\nExpected no error\n Got %v", err) + } + if !reflect.DeepEqual(c.expected.condition, c.input) { + t.Fatalf("\nExpected %v\n Got %v", c.expected.condition, c.input) + } + } else if !reflect.DeepEqual(c.expected.err, err) { + t.Fatalf("\nExpected %v\n Got %v", c.expected.err, err) + } + } +} diff --git a/parser/error.go b/parser/error.go new file mode 100644 index 0000000..bd3ef8c --- /dev/null +++ b/parser/error.go @@ -0,0 +1,42 @@ +package parser + +import ( + "fmt" + "io" + + "github.com/kashav/fsql/tokenizer" +) + +// ErrUnexpectedToken represents an unexpected token error. +type ErrUnexpectedToken struct { + Actual tokenizer.TokenType + Expected tokenizer.TokenType +} + +func (e *ErrUnexpectedToken) Error() string { + return fmt.Sprintf("expected %s; got %s", e.Expected.String(), + e.Actual.String()) +} + +// ErrUnknownToken represents an unknown token error. +type ErrUnknownToken struct { + Raw string +} + +func (e *ErrUnknownToken) Error() string { + return fmt.Sprintf("unknown token: %s", e.Raw) +} + +// currentError returns the current error, based on the parser's current Token +// and the previously expected TokenType (set in parser.expect). +func (p *parser) currentError() error { + if p.current == nil { + return io.ErrUnexpectedEOF + } + + if p.current.Type == tokenizer.Unknown { + return &ErrUnknownToken{Raw: p.current.Raw} + } + + return &ErrUnexpectedToken{Actual: p.current.Type, Expected: p.expected} +} diff --git a/parser/error_test.go b/parser/error_test.go new file mode 100644 index 0000000..3448366 --- /dev/null +++ b/parser/error_test.go @@ -0,0 +1,28 @@ +package parser + +import ( + "testing" + + "github.com/kashav/fsql/tokenizer" +) + +func TestParser_ErrUnexpectedToken(t *testing.T) { + err := &ErrUnexpectedToken{ + Actual: tokenizer.Select, + Expected: tokenizer.Where, + } + expected := "expected where; got select" + actual := err.Error() + if expected != actual { + t.Fatalf("\nExpected: %s\n Got: %s", expected, actual) + } +} + +func TestParser_ErrUnknownTokent(t *testing.T) { + err := &ErrUnknownToken{"r"} + expected := "unknown token: r" + actual := err.Error() + if expected != actual { + t.Fatalf("\nExpected: %s\n Got: %s", expected, actual) + } +} diff --git a/parser/parser.go b/parser/parser.go new file mode 100644 index 0000000..a9af431 --- /dev/null +++ b/parser/parser.go @@ -0,0 +1,140 @@ +package parser + +import ( + "os/user" + "path/filepath" + "strings" + + "github.com/kashav/fsql/query" + "github.com/kashav/fsql/tokenizer" +) + +// Run parses the input string and returns the parsed AST (query). +func Run(input string) (*query.Query, error) { + return (&parser{}).parse(input) +} + +type parser struct { + tokenizer *tokenizer.Tokenizer + current *tokenizer.Token + expected tokenizer.TokenType +} + +// parse runs the respective parser function on each clause of the query. +func (p *parser) parse(input string) (*query.Query, error) { + q := query.NewQuery() + p.tokenizer = tokenizer.NewTokenizer(input) + if err := p.parseSelectClause(q); err != nil { + return nil, err + } + if err := p.parseFromClause(q); err != nil { + return nil, err + } + if err := p.parseWhereClause(q); err != nil { + return nil, err + } + return q, nil +} + +// parseSelectClause parses the SELECT clause of the query. +func (p *parser) parseSelectClause(q *query.Query) error { + // Determine if we should show all attributes. This is only true when + // no attributes are provided (regardless of if the SELECT keyword is + // provided or not). + var showAll = true + if p.expect(tokenizer.Select) == nil { + if p.current == nil || p.current.Type == tokenizer.Identifier { + showAll = false + } else if p.current.Type == tokenizer.From || p.current.Type == tokenizer.Where { + // No SELECT and next token is FROM/WHERE, show all! + showAll = true + } else { + // No SELECT and next token is not Identifier nor FROM/WHERE -> malformed + // input. + return p.currentError() + } + } else if current := p.expect(tokenizer.Identifier); current != nil { + p.current = current + showAll = false + } + + if showAll { + q.Attributes = allAttributes + } else if err := p.parseAttrs(&q.Attributes, &q.Modifiers); err != nil { + return err + } + + return nil +} + +// parseFromClause parses the FROM clause of the query. +func (p *parser) parseFromClause(q *query.Query) error { + if p.expect(tokenizer.From) == nil { + err := p.currentError() + if p.expect(tokenizer.Identifier) != nil { + // No FROM, but an identifier -> malformed query. + return err + } + + // No specified directory, so we default to the CWD. + q.Sources["include"] = append(q.Sources["include"], ".") + return nil + } + + if err := p.parseSourceList(&q.Sources, &q.SourceAliases); err != nil { + return err + } + + // Replace the tilde with the home directory in each source directory. This + // is only required when the query is wrapped in quotes, since the shell + // will automatically expand tildes otherwise. + u, err := user.Current() + if err != nil { + return err + } + for _, sourceType := range []string{"include", "exclude"} { + for i, src := range q.Sources[sourceType] { + if strings.Contains(src, "~") { + q.Sources[sourceType][i] = filepath.Join(u.HomeDir, src[1:]) + } + } + } + + return nil +} + +// parseWhereClause parses the WHERE clause of the query. +func (p *parser) parseWhereClause(q *query.Query) error { + if p.expect(tokenizer.Where) == nil { + err := p.currentError() + if p.expect(tokenizer.Identifier) == nil { + return nil + } + return err + } + root, err := p.parseConditionTree() + if err != nil { + return err + } + q.ConditionTree = root + + return nil +} + +// expect returns the next token if it matches the expectation t, and +// nil otherwise. +func (p *parser) expect(t tokenizer.TokenType) *tokenizer.Token { + p.expected = t + + if p.current == nil { + p.current = p.tokenizer.Next() + } + + if p.current != nil && p.current.Type == t { + tok := p.current + p.current = nil + return tok + } + + return nil +} diff --git a/parser/parser_test.go b/parser/parser_test.go new file mode 100644 index 0000000..fc05dd1 --- /dev/null +++ b/parser/parser_test.go @@ -0,0 +1,437 @@ +package parser + +import ( + "io" + "os/user" + "reflect" + "testing" + + "github.com/kashav/fsql/query" + "github.com/kashav/fsql/tokenizer" +) + +func TestParser_ParseSelect(t *testing.T) { + type Expected struct { + attributes []string + modifiers map[string][]query.Modifier + err error + } + + type Case struct { + input string + expected Expected + } + + cases := []Case{ + { + input: "all", + expected: Expected{ + attributes: allAttributes, + modifiers: map[string][]query.Modifier{}, + err: nil, + }, + }, + + { + input: "SELECT", + expected: Expected{ + attributes: allAttributes, + modifiers: map[string][]query.Modifier{}, + err: nil, + }, + }, + + { + input: "FROM", + expected: Expected{ + attributes: allAttributes, + modifiers: map[string][]query.Modifier{}, + err: nil, + }, + }, + + { + input: "SELECT name", + expected: Expected{ + attributes: []string{"name"}, + modifiers: map[string][]query.Modifier{"name": {}}, + err: nil, + }, + }, + + { + input: "SELECT format(size, kb)", + expected: Expected{ + attributes: []string{"size"}, + modifiers: map[string][]query.Modifier{ + "size": { + { + Name: "FORMAT", + Arguments: []string{"kb"}, + }, + }, + }, + err: nil, + }, + }, + + { + input: "", + expected: Expected{err: io.ErrUnexpectedEOF}, + }, + } + + for _, c := range cases { + q := query.NewQuery() + err := (&parser{tokenizer: tokenizer.NewTokenizer(c.input)}).parseSelectClause(q) + + if c.expected.err == nil { + if err != nil { + t.Fatalf("\nExpected no error\n Got %v", err) + } + if !reflect.DeepEqual(c.expected.attributes, q.Attributes) { + t.Fatalf("\nExpected %v\n Got %v", c.expected.attributes, q.Attributes) + } + if !reflect.DeepEqual(c.expected.modifiers, q.Modifiers) { + t.Fatalf("\nExpected %v\n Got %v", c.expected.modifiers, q.Modifiers) + } + } else if !reflect.DeepEqual(c.expected.err, err) { + t.Fatalf("\nExpected %v\n Got %v", c.expected.err, err) + } + } +} + +func TestParser_ParseFrom(t *testing.T) { + type Expected struct { + sources map[string][]string + aliases map[string]string + err error + } + + type Case struct { + input string + expected Expected + } + + u, err := user.Current() + if err != nil { + // TODO: If we can't get the current user, should we fatal or just return? + return + } + + cases := []Case{ + { + input: "WHERE", + expected: Expected{ + sources: map[string][]string{ + "include": {"."}, + "exclude": {}, + }, + aliases: map[string]string{}, + err: nil, + }, + }, + + { + input: "FROM .", + expected: Expected{ + sources: map[string][]string{ + "include": {"."}, + "exclude": {}, + }, + aliases: map[string]string{}, + err: nil, + }, + }, + + { + input: "FROM ~/foo, -./.git/", + expected: Expected{ + sources: map[string][]string{ + "include": {u.HomeDir + "/foo"}, + "exclude": {".git"}, + }, + aliases: map[string]string{}, + err: nil, + }, + }, + + { + input: "FROM ./foo/ AS foo", + expected: Expected{ + sources: map[string][]string{ + "include": {"foo"}, + "exclude": {}, + }, + aliases: map[string]string{"foo": "foo"}, + err: nil, + }, + }, + + { + input: "FROM", + expected: Expected{err: io.ErrUnexpectedEOF}, + }, + + { + input: "FROM WHERE", + expected: Expected{ + err: &ErrUnexpectedToken{ + Actual: tokenizer.Where, + Expected: tokenizer.Identifier, + }, + }, + }, + } + + for _, c := range cases { + q := query.NewQuery() + err := (&parser{tokenizer: tokenizer.NewTokenizer(c.input)}).parseFromClause(q) + + if c.expected.err == nil { + if err != nil { + t.Fatalf("\nExpected no error\n Got %v", err) + } + if !reflect.DeepEqual(c.expected.sources, q.Sources) { + t.Fatalf("\nExpected %v\n Got %v", c.expected.sources, q.Sources) + } + if !reflect.DeepEqual(c.expected.aliases, q.SourceAliases) { + t.Fatalf("\nExpected %v\n Got %v", c.expected.aliases, q.SourceAliases) + } + } else if !reflect.DeepEqual(c.expected.err, err) { + t.Fatalf("\nExpected %v\n Got %v", c.expected.err, err) + } + } +} + +func TestParser_ParseWhere(t *testing.T) { + type Expected struct { + tree *query.ConditionNode + err error + } + + type Case struct { + input string + expected Expected + } + + cases := []Case{ + { + input: "WHERE name LIKE foo", + expected: Expected{ + tree: &query.ConditionNode{ + Condition: &query.Condition{ + Attribute: "name", + Operator: tokenizer.Like, + Value: "foo", + }, + }, + err: nil, + }, + }, + + // Our tree is fully-zeroed in this case, so it's easier just to give it + // an empty Expected struct. + {input: "", expected: Expected{}}, + + {input: "WHERE", expected: Expected{err: io.ErrUnexpectedEOF}}, + + { + input: "name LIKE foo", + expected: Expected{ + err: &ErrUnexpectedToken{ + Expected: tokenizer.Where, + Actual: tokenizer.Identifier, + }, + }, + }, + } + + for _, c := range cases { + q := query.NewQuery() + err := (&parser{tokenizer: tokenizer.NewTokenizer(c.input)}).parseWhereClause(q) + + if c.expected.err == nil { + if err != nil { + t.Fatalf("\nExpected no error\n Got %v", err) + } + if !reflect.DeepEqual(c.expected.tree, q.ConditionTree) { + t.Fatalf("\nExpected %v\n Got %v", c.expected.tree, q.ConditionTree) + } + } else if !reflect.DeepEqual(c.expected.err, err) { + t.Fatalf("\nExpected %v\n Got %v", c.expected.err, err) + } + } +} + +func TestParser_Expect(t *testing.T) { + type Case struct { + param tokenizer.TokenType + expected *tokenizer.Token + } + + input := "SELECT all FROM . WHERE name = foo OR size <> 100" + p := &parser{tokenizer: tokenizer.NewTokenizer(input)} + + cases := []Case{ + { + param: tokenizer.Select, + expected: &tokenizer.Token{Type: tokenizer.Select, Raw: "SELECT"}, + }, + { + param: tokenizer.From, + expected: nil, + }, + { + param: tokenizer.Identifier, + expected: &tokenizer.Token{Type: tokenizer.Identifier, Raw: "all"}, + }, + { + param: tokenizer.Identifier, + expected: nil, + }, + { + param: tokenizer.From, + expected: &tokenizer.Token{Type: tokenizer.From, Raw: "FROM"}, + }, + { + param: tokenizer.Identifier, + expected: &tokenizer.Token{Type: tokenizer.Identifier, Raw: "."}, + }, + { + param: tokenizer.Identifier, + expected: nil, + }, + { + param: tokenizer.Where, + expected: &tokenizer.Token{Type: tokenizer.Where, Raw: "WHERE"}, + }, + { + param: tokenizer.Identifier, + expected: &tokenizer.Token{Type: tokenizer.Identifier, Raw: "name"}, + }, + { + param: tokenizer.Equals, + expected: &tokenizer.Token{Type: tokenizer.Equals, Raw: "="}, + }, + { + param: tokenizer.Identifier, + expected: &tokenizer.Token{Type: tokenizer.Identifier, Raw: "foo"}, + }, + { + param: tokenizer.Or, + expected: &tokenizer.Token{Type: tokenizer.Or, Raw: "OR"}, + }, + { + param: tokenizer.Identifier, + expected: &tokenizer.Token{Type: tokenizer.Identifier, Raw: "size"}, + }, + { + param: tokenizer.Identifier, + expected: nil, + }, + { + param: tokenizer.NotEquals, + expected: &tokenizer.Token{Type: tokenizer.NotEquals, Raw: "<>"}, + }, + { + param: tokenizer.Identifier, + expected: &tokenizer.Token{Type: tokenizer.Identifier, Raw: "100"}, + }, + } + + for _, c := range cases { + actual := p.expect(c.param) + if !reflect.DeepEqual(c.expected, actual) { + t.Fatalf("\nExpected %v\n Got %v", c.expected, actual) + } + } +} + +func TestParser_SelectAllVariations(t *testing.T) { + expected := &query.Query{ + Attributes: allAttributes, + Sources: map[string][]string{ + "include": {"."}, + "exclude": {}, + }, + ConditionTree: &query.ConditionNode{ + Condition: &query.Condition{ + Attribute: "name", + Operator: tokenizer.Like, + Value: "foo", + }, + }, + SourceAliases: map[string]string{}, + Modifiers: map[string][]query.Modifier{}, + } + + cases := []string{ + "FROM . WHERE name LIKE foo", + "all FROM . WHERE name LIKE foo", + "SELECT FROM . WHERE name LIKE foo", + "SELECT all FROM . WHERE name LIKE foo", + } + + for _, c := range cases { + actual, err := Run(c) + if err != nil { + t.Fatalf("\nExpected no error\n Got %v", err) + } + if !reflect.DeepEqual(expected, actual) { + t.Fatalf("\nExpected %v\n Got %v", expected, actual) + } + } +} + +func TestParser_Run(t *testing.T) { + type Expected struct { + q *query.Query + err error + } + + type Case struct { + input string + expected Expected + } + + // TODO: Add more cases. + cases := []Case{ + { + input: "SELECT all FROM . WHERE name LIKE foo", + expected: Expected{ + q: &query.Query{ + Attributes: allAttributes, + Sources: map[string][]string{ + "include": {"."}, + "exclude": {}, + }, + ConditionTree: &query.ConditionNode{ + Condition: &query.Condition{ + Attribute: "name", + Operator: tokenizer.Like, + Value: "foo", + }, + }, + SourceAliases: map[string]string{}, + Modifiers: map[string][]query.Modifier{}, + }, + err: nil, + }, + }, + } + + for _, c := range cases { + actual, err := Run(c.input) + if c.expected.err == nil { + if err != nil { + t.Fatalf("\nExpected no error\n Got %v", err) + } + if !reflect.DeepEqual(c.expected.q, actual) { + t.Fatalf("\nExpected %v\n Got %v", c.expected.q, actual) + } + } else if !reflect.DeepEqual(c.expected.err, err) { + t.Fatalf("\nExpected %v\n Got %v", c.expected.err, err) + } + } +} diff --git a/parser/source.go b/parser/source.go new file mode 100644 index 0000000..9c178c6 --- /dev/null +++ b/parser/source.go @@ -0,0 +1,45 @@ +package parser + +import ( + "fmt" + "path/filepath" + + "github.com/kashav/fsql/tokenizer" +) + +// parseSourceList parses the list of directories passed to the FROM clause. If +// a source is followed by the AS keyword, the following word is registered as +// an alias. +func (p *parser) parseSourceList(sources *map[string][]string, + aliases *map[string]string) error { + for { + // If the next token is a hypen, exclude this directory. + sourceType := "include" + if token := p.expect(tokenizer.Hyphen); token != nil { + sourceType = "exclude" + } + + source := p.expect(tokenizer.Identifier) + if source == nil { + return p.currentError() + } + source.Raw = filepath.Clean(source.Raw) + (*sources)[sourceType] = append((*sources)[sourceType], source.Raw) + + if token := p.expect(tokenizer.As); token != nil { + alias := p.expect(tokenizer.Identifier) + if alias == nil { + return p.currentError() + } + if sourceType == "exclude" { + return fmt.Errorf("cannot alias excluded directory %s", source.Raw) + } + (*aliases)[alias.Raw] = source.Raw + } + + if p.expect(tokenizer.Comma) == nil { + break + } + } + return nil +} diff --git a/parser/source_test.go b/parser/source_test.go new file mode 100644 index 0000000..161b079 --- /dev/null +++ b/parser/source_test.go @@ -0,0 +1,149 @@ +package parser + +import ( + "errors" + "io" + "reflect" + "testing" + + "github.com/kashav/fsql/tokenizer" +) + +func TestSourceParser_ExpectCorrectSources(t *testing.T) { + type Expected struct { + sources map[string][]string + err error + } + + type Case struct { + input string + expected Expected + } + + cases := []Case{ + { + input: ".", + expected: Expected{ + sources: map[string][]string{"include": {"."}}, + err: nil, + }, + }, + { + input: "., ~/foo", + expected: Expected{ + sources: map[string][]string{"include": {".", "~/foo"}}, + err: nil, + }, + }, + { + input: "., -.bar", + expected: Expected{ + sources: map[string][]string{ + "include": {"."}, + "exclude": {".bar"}, + }, + err: nil, + }, + }, + { + input: "-.bar, ., ~/foo AS foo", + expected: Expected{ + sources: map[string][]string{ + "include": {".", "~/foo"}, + "exclude": {".bar"}, + }, + err: nil, + }, + }, + + {input: "", expected: Expected{err: io.ErrUnexpectedEOF}}, + {input: "foo,", expected: Expected{err: io.ErrUnexpectedEOF}}, + } + + for _, c := range cases { + sources := make(map[string][]string, 0) + aliases := make(map[string]string, 0) + + p := &parser{tokenizer: tokenizer.NewTokenizer(c.input)} + err := p.parseSourceList(&sources, &aliases) + + if c.expected.err == nil { + if err != nil { + t.Fatalf("\nExpected no error\n Got %v", err) + } + if !reflect.DeepEqual(c.expected.sources, sources) { + t.Fatalf("\nExpected %v\n Got %v", c.expected.sources, sources) + } + } else if !reflect.DeepEqual(c.expected.err, err) { + t.Fatalf("\nExpected %v\n Got %v", c.expected.err, err) + } + } +} + +func TestSourceParser_ExpectCorrectAliases(t *testing.T) { + type Expected struct { + aliases map[string]string + err error + } + + type Case struct { + input string + expected Expected + } + + cases := []Case{ + { + input: ".", + expected: Expected{ + aliases: map[string]string{}, + err: nil, + }, + }, + { + input: ". AS cwd", + expected: Expected{ + aliases: map[string]string{"cwd": "."}, + err: nil, + }, + }, + { + input: "., -.bar, ~/foo AS foo", + expected: Expected{ + aliases: map[string]string{"foo": "~/foo"}, + err: nil, + }, + }, + + { + input: "-.bar AS bar", + expected: Expected{err: errors.New("cannot alias excluded directory .bar")}, + }, + { + input: "", + expected: Expected{err: io.ErrUnexpectedEOF}, + }, + { + input: "foo AS", + expected: Expected{err: io.ErrUnexpectedEOF}, + }, + } + + for _, c := range cases { + sources := make(map[string][]string, 0) + aliases := make(map[string]string, 0) + + p := &parser{tokenizer: tokenizer.NewTokenizer(c.input)} + err := p.parseSourceList(&sources, &aliases) + + if c.expected.err == nil { + if err != nil { + t.Fatalf("\nExpected no error\n Got %v", err) + } + if !reflect.DeepEqual(c.expected.aliases, aliases) { + t.Fatalf("\nExpected %v\n Got %v", c.expected.aliases, aliases) + } + } else if !reflect.DeepEqual(c.expected.err, err) { + t.Fatalf("\nExpected %v\n Got %v", c.expected.err, err) + } + } +} diff --git a/query/condition.go b/query/condition.go new file mode 100644 index 0000000..7340071 --- /dev/null +++ b/query/condition.go @@ -0,0 +1,137 @@ +package query + +import ( + "errors" + "fmt" + "os" + + "github.com/kashav/fsql/evaluate" + "github.com/kashav/fsql/tokenizer" + "github.com/kashav/fsql/transform" +) + +// ConditionNode represents a single node of a query's WHERE clause tree. +type ConditionNode struct { + Type *tokenizer.TokenType + Left *ConditionNode + Right *ConditionNode + Condition *Condition +} + +func (root *ConditionNode) String() string { + if root == nil { + return "" + } + + return fmt.Sprintf("{%v (%v %v) %v}", root.Type, root.Left, root.Right, + root.Condition) +} + +// evaluateTree runs pre-order traversal on the ConditionNode tree rooted at +// root and evaluates each conditional along the path with the provided compare +// method. +func (root *ConditionNode) evaluateTree(path string, info os.FileInfo) (bool, error) { + if root == nil { + return true, nil + } + + if root.Condition != nil { + if root.Condition.IsSubquery { + // Unevaluated subquery. + // TODO: Handle this case. + return false, errors.New("not implemented") + } + + if !root.Condition.Parsed { + if err := root.Condition.applyModifiers(); err != nil { + return false, err + } + } + + return root.Condition.evaluate(path, info) + } + + if *root.Type == tokenizer.And { + if ok, err := root.Left.evaluateTree(path, info); err != nil { + return false, err + } else if !ok { + return false, nil + } + return root.Right.evaluateTree(path, info) + } + + if *root.Type == tokenizer.Or { + if ok, err := root.Left.evaluateTree(path, info); err != nil { + return false, nil + } else if ok { + return true, nil + } + return root.Right.evaluateTree(path, info) + } + + return false, nil +} + +// Condition represents a WHERE condition. +type Condition struct { + Attribute string + AttributeModifiers []Modifier + Parsed bool + + Operator tokenizer.TokenType + Value interface{} + Negate bool + + Subquery *Query + IsSubquery bool +} + +// ApplyModifiers applies each modifier to the value of this Condition. +func (c *Condition) applyModifiers() error { + value := c.Value + + for _, m := range c.AttributeModifiers { + var err error + value, err = transform.Parse(&transform.ParseParams{ + Attribute: c.Attribute, + Value: value, + Name: m.Name, + Args: m.Arguments, + }) + if err != nil { + return err + } + } + + c.Value = value + c.Parsed = true + return nil +} + +// evaluate runs the respective evaluate function for this Condition. +func (c *Condition) evaluate(path string, file os.FileInfo) (bool, error) { + // FIXME: This is a bit of a hack. We can't pass c.AttributeModifiers, since + // that'll cause a import cycle, so we have to recreate the attribute + // modifiers slice using a separate type defined in evaluate. + modifiers := make([]evaluate.Modifier, len(c.AttributeModifiers)) + for i, m := range c.AttributeModifiers { + modifiers[i] = evaluate.Modifier{Name: m.Name, Arguments: m.Arguments} + } + + o := &evaluate.Opts{ + Path: path, + File: file, + Attribute: c.Attribute, + Modifiers: modifiers, + Operator: c.Operator, + Value: c.Value, + } + result, err := evaluate.Evaluate(o) + if err != nil { + return false, err + } + if c.Negate { + return !result, nil + } + return result, nil +} diff --git a/query/condition_test.go b/query/condition_test.go new file mode 100644 index 0000000..f6b34a8 --- /dev/null +++ b/query/condition_test.go @@ -0,0 +1 @@ +package query diff --git a/query/excluder.go b/query/excluder.go new file mode 100644 index 0000000..01105df --- /dev/null +++ b/query/excluder.go @@ -0,0 +1,43 @@ +package query + +import ( + "fmt" + "regexp" + "strings" +) + +// Excluder allows us to support different methods of excluding in the future. +type Excluder interface { + shouldExclude(path string) bool +} + +// regexpExclude uses regular expressions to tell if a file/path should be +// excluded. +type regexpExclude struct { + exclusions []string + regex *regexp.Regexp +} + +// ShouldExclude will return a boolean denoting whether or not the path should +// be excluded based on the given slice of exclusions. +func (r *regexpExclude) shouldExclude(path string) bool { + if r.regex == nil { + r.buildRegex() + } + if r.regex.String() == "" { + return false + } + return r.regex.MatchString(path) +} + +// buildRegex builds the regular expression for this RegexpExclude. +func (r *regexpExclude) buildRegex() { + exclusions := make([]string, len(r.exclusions)) + for i, exclusion := range r.exclusions { + // Wrap exclusion in ^ and (/.*)?$ AFTER trimming trailing slashes and + // escaping all dots. + exclusions[i] = fmt.Sprintf("^%s(/.*)?$", + strings.Replace(strings.TrimRight(exclusion, "/"), ".", "\\.", -1)) + } + r.regex = regexp.MustCompile(strings.Join(exclusions, "|")) +} diff --git a/query/excluder_test.go b/query/excluder_test.go new file mode 100644 index 0000000..18a931d --- /dev/null +++ b/query/excluder_test.go @@ -0,0 +1,51 @@ +package query + +import "testing" + +func TestShouldExclude_ExpectAllExcluded(t *testing.T) { + type Case struct { + input string + expected bool + } + + exclusions := []string{".git", ".gitignore"} + excluder := regexpExclude{exclusions: exclusions} + + cases := []Case{ + {input: ".git", expected: true}, + {input: ".git/", expected: true}, + {input: ".git/some/other/file", expected: true}, + {input: ".gitignore", expected: true}, + } + + for _, c := range cases { + actual := excluder.shouldExclude(c.input) + if actual != c.expected { + t.Fatalf("\nExpected %v\n Got %v", c.expected, actual) + } + } +} + +func TestShouldExclude_ExpectNotExcluded(t *testing.T) { + type Case struct { + input string + expected bool + } + + exclusions := []string{".git"} + excluder := regexpExclude{exclusions: exclusions} + + cases := []Case{ + {input: ".git", expected: true}, + {input: ".git/", expected: true}, + {input: ".git/some/other/file", expected: true}, + {input: ".gitignore", expected: false}, + } + + for _, c := range cases { + actual := excluder.shouldExclude(c.input) + if actual != c.expected { + t.Fatalf("\nExpected %v\n Got %v", c.expected, actual) + } + } +} diff --git a/query/modifier.go b/query/modifier.go new file mode 100644 index 0000000..428fa9f --- /dev/null +++ b/query/modifier.go @@ -0,0 +1,55 @@ +package query + +import ( + "fmt" + "os" + "strings" + + "github.com/kashav/fsql/transform" +) + +// Modifier represents an attribute modifier. +type Modifier struct { + Name string + Arguments []string +} + +func (m *Modifier) String() string { + return fmt.Sprintf("%s(%s)", m.Name, strings.Join(m.Arguments, ", ")) +} + +// applyModifiers iterates through each SELECT attribute for this query +// and applies the associated modifier to the attribute's output value. +func (q *Query) applyModifiers(path string, info os.FileInfo) (map[string]interface{}, error) { + results := make(map[string]interface{}, len(q.Attributes)) + + for _, attribute := range q.Attributes { + value, err := transform.DefaultFormatValue(attribute, path, info) + if err != nil { + return map[string]interface{}{}, err + } + + if _, ok := q.Modifiers[attribute]; !ok { + results[attribute] = value + continue + } + + for _, m := range q.Modifiers[attribute] { + value, err = transform.Format(&transform.FormatParams{ + Attribute: attribute, + Path: path, + Info: info, + Value: value, + Name: m.Name, + Args: m.Arguments, + }) + if err != nil { + return map[string]interface{}{}, err + } + } + + results[attribute] = value + } + + return results, nil +} diff --git a/query/modifier_test.go b/query/modifier_test.go new file mode 100644 index 0000000..fefdaf6 --- /dev/null +++ b/query/modifier_test.go @@ -0,0 +1,32 @@ +package query + +import "testing" + +func TestModifier_String(t *testing.T) { + type Case struct { + input Modifier + expected string + } + + cases := []Case{ + { + input: Modifier{Name: "upper", Arguments: []string{}}, + expected: "upper()", + }, + { + input: Modifier{Name: "format", Arguments: []string{"upper"}}, + expected: "format(upper)", + }, + } + + for _, c := range cases { + result := c.input.String() + if result != c.expected { + t.Fatalf("\nExpected: %s\n Got: %s", c.expected, result) + } + } +} + +func TestModifier_Apply(t *testing.T) { + // TODO +} diff --git a/query/query.go b/query/query.go new file mode 100644 index 0000000..4276359 --- /dev/null +++ b/query/query.go @@ -0,0 +1,119 @@ +package query + +import ( + "os" + "path/filepath" + "strings" +) + +// Query represents an input query. +type Query struct { + Attributes []string + Modifiers map[string][]Modifier + + Sources map[string][]string + SourceAliases map[string]string + + ConditionTree *ConditionNode +} + +// NewQuery returns a pointer to a Query. +func NewQuery() *Query { + return &Query{ + Attributes: make([]string, 0), + Modifiers: make(map[string][]Modifier), + Sources: map[string][]string{ + "include": make([]string, 0), + "exclude": make([]string, 0), + }, + SourceAliases: make(map[string]string), + ConditionTree: nil, + } +} + +// HasAttribute checks if this query contains any of the provided attributes. +func (q *Query) HasAttribute(attributes ...string) bool { + for _, attribute := range attributes { + for _, queryAttribute := range q.Attributes { + if attribute == queryAttribute { + return true + } + } + } + return false +} + +// Execute runs the query by walking the full path of each source and +// evaluating the condition tree for each file. This method calls workFunc on +// each "successful" file. +func (q *Query) Execute(workFunc interface{}) error { + seen := map[string]bool{} + excluder := ®expExclude{exclusions: q.Sources["exclude"]} + + for _, src := range q.Sources["include"] { + // TODO: Improve our method of detecting if src is a glob pattern. This + // currently doesn't support usage of square brackets, since the tokenizer + // doesn't recognize these as part of a directory. + // + // Pattern reference: https://golang.org/pkg/path/filepath/#Match. + if strings.ContainsAny(src, "*?") { + // If src does _resemble_ a glob pattern, we find all matches and + // evaluate the condition tree against each. + matches, err := filepath.Glob(src) + if err != nil { + return err + } + + for _, match := range matches { + if err = filepath.Walk(match, q.walkFunc(seen, excluder, workFunc)); err != nil { + return err + } + } + continue + } + + if err := filepath.Walk(src, q.walkFunc(seen, excluder, workFunc)); err != nil { + return err + } + } + + return nil +} + +// walkFunc returns a filepath.WalkFunc which evaluates the condition tree +// against the given file. +func (q *Query) walkFunc(seen map[string]bool, excluder Excluder, + workFunc interface{}) filepath.WalkFunc { + return func(path string, info os.FileInfo, err error) error { + if err != nil { + return err + } + + if path == "." { + return nil + } + + // Avoid walking a single directory more than once. + if _, ok := seen[path]; ok { + return nil + } + seen[path] = true + + if excluder.shouldExclude(path) { + return nil + } + + if ok, err := q.ConditionTree.evaluateTree(path, info); err != nil { + return err + } else if !ok { + return nil + } + + results, err := q.applyModifiers(path, info) + if err != nil { + return err + } + workFunc.(func(string, os.FileInfo, map[string]interface{}))(path, info, results) + return nil + } +} diff --git a/query/query_test.go b/query/query_test.go new file mode 100644 index 0000000..f6b34a8 --- /dev/null +++ b/query/query_test.go @@ -0,0 +1 @@ +package query diff --git a/terminal/pager/pager.go b/terminal/pager/pager.go new file mode 100644 index 0000000..3e78d33 --- /dev/null +++ b/terminal/pager/pager.go @@ -0,0 +1,31 @@ +package pager + +import ( + "bytes" + "os" + "os/exec" +) + +const cmd = "less" + +var opts = []string{"-S"} + +// CommandExists returns true iff cmd exists on the host machine. +func CommandExists() bool { + _, err := exec.LookPath(cmd) + return err == nil +} + +// New invokes cmd with in provided as stdin, if cmd is unavailable, return +// an error. +func New(in []byte) error { + path, err := exec.LookPath(cmd) + if err != nil { + return err + } + pager := exec.Command(path, opts...) + pager.Stdin = bytes.NewReader(in) + pager.Stdout = os.Stdout + pager.Stderr = os.Stderr + return pager.Run() +} diff --git a/terminal/terminal.go b/terminal/terminal.go new file mode 100644 index 0000000..7d6dfdb --- /dev/null +++ b/terminal/terminal.go @@ -0,0 +1,127 @@ +package terminal + +import ( + "bytes" + "errors" + "fmt" + "io" + "os" + "strings" + + "github.com/kashav/fsql" + "github.com/kashav/fsql/terminal/pager" + + "golang.org/x/crypto/ssh/terminal" +) + +var fd = int(os.Stdin.Fd()) +var query bytes.Buffer + +// Start listens for queries via stdin and invokes fsql.Run whenever a +// semicolon is read. +func Start() error { + if !terminal.IsTerminal(fd) { + return errors.New("not a terminal") + } + + state, err := terminal.MakeRaw(fd) + if err != nil { + return err + } + defer terminal.Restore(fd, state) + + prompt := ">>> " + term := terminal.NewTerminal(os.Stdin, prompt) + + // Listen for queries and invoke run whenever a semicolon is read. Continues + // until receiving an EOF (Ctrl-D) or _fatal_ error (i.e. anything not + // caused by the query itself). + for { + line, err := term.ReadLine() + if err == io.EOF { + fmt.Print("\r\nbye\r\n") + break + } + if err != nil { + return err + } + + if line == "exit" { + fmt.Print("bye\r\n") + break + } + + // TODO: If the previous character was a paren., bracket, or quote, we + // don't want to add a space here (although not necessary, since the + // tokenizer handles excess whitespace). + if query.Len() > 0 { + query.WriteString(" ") + } + query.WriteString(line) + + if strings.HasSuffix(line, ";") { + query.Truncate(query.Len() - 1) + + b := []byte{} + if out, err := run(query.String()); err != nil { + // This error likely corresponds to the query, so instead of exiting + // interactive mode, we simply write the error to stdout and proceed. + b = append(b, []byte(err.Error())...) + b = append(b, '\a', '\n') + term.Write(b) + } else if len(out) > 0 { + _, h, err := terminal.GetSize(fd) + if err != nil { + return err + } + + b = append(b, []byte(out)...) + + // Write to stdout if out is less than 3/4 of the height of the + // window OR the `less` command doesn't exist; otherwise, invoke the + // pager. + if float64(strings.Count(out, "\n")) <= 0.75*float64(h) || + !pager.CommandExists() { + term.Write(b) + } else if err = pager.New(b); err != nil { + return err + } + } + + query.Reset() + } + + prompt = "... " + if query.Len() == 0 { + prompt = ">>> " + } + term.SetPrompt(prompt) + } + return nil +} + +// run invokes fsql.Run with the provided query string. +func run(query string) (out string, err error) { + stdout := os.Stdout + r, w, err := os.Pipe() + if err != nil { + return "", err + } + os.Stdout = w + defer func() { os.Stdout = stdout }() + + ch := make(chan string) + go func() { + var buf bytes.Buffer + io.Copy(&buf, r) + ch <- buf.String() + }() + + err = fsql.Run(query) + // Must happen after the function call and before we try to read from ch. + if closeErr := w.Close(); closeErr != nil { + return "", closeErr + } + out = <-ch + return +} diff --git a/terminal/terminal_test.go b/terminal/terminal_test.go new file mode 100644 index 0000000..f11756a --- /dev/null +++ b/terminal/terminal_test.go @@ -0,0 +1,52 @@ +package terminal + +import ( + "errors" + "reflect" + "testing" +) + +func TestRun(t *testing.T) { + type Expected struct { + out string + err error + } + + type Case struct { + query string + expected Expected + } + + // We already test core functionality in the fsql package, so we can stick + // with _simple_ queries for the following cases. + cases := []Case{ + { + query: "select name, hash from ../testdata where name = baz", + expected: Expected{ + out: "baz\tda39a3e\n", + err: nil, + }, + }, + { + query: "select all from", + expected: Expected{ + out: "", + err: errors.New("unexpected EOF"), + }, + }, + } + + for _, c := range cases { + actual, err := run(c.query) + if c.expected.err == nil { + if err != nil { + t.Fatalf("\nExpected no error\n Got %v", err) + } + if !reflect.DeepEqual(c.expected.out, actual) { + t.Fatalf("\nExpected %v\n Got %v", c.expected.out, actual) + } + } else if !reflect.DeepEqual(c.expected.err, err) { + t.Fatalf("\nExpected %v\n Got %v", c.expected.err, err) + } + } +} diff --git a/testdata/bar/corge b/testdata/bar/corge new file mode 100644 index 0000000..e69de29 diff --git a/testdata/bar/garply/xyzzy/thud/.gitkeep b/testdata/bar/garply/xyzzy/thud/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/testdata/bar/grault b/testdata/bar/grault new file mode 100644 index 0000000..e69de29 diff --git a/testdata/baz b/testdata/baz new file mode 100755 index 0000000..e69de29 diff --git a/testdata/foo/quux b/testdata/foo/quux new file mode 100644 index 0000000..e69de29 diff --git a/testdata/foo/quuz/fred/.gitkeep b/testdata/foo/quuz/fred/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/testdata/foo/quuz/waldo b/testdata/foo/quuz/waldo new file mode 100644 index 0000000..e69de29 diff --git a/testdata/foo/qux b/testdata/foo/qux new file mode 100644 index 0000000..e69de29 diff --git a/tokenizer/token.go b/tokenizer/token.go new file mode 100644 index 0000000..6d25b4c --- /dev/null +++ b/tokenizer/token.go @@ -0,0 +1,127 @@ +package tokenizer + +import "fmt" + +// TokenType represents a Token's type. +type TokenType int8 + +// All TokenType constants. +const ( + Unknown TokenType = iota + + Identifier + Subquery + + Select + From + Where + + As + Or + And + Not + + In + Is + Like + RLike + + Equals + NotEquals + GreaterThanEquals + GreaterThan + LessThanEquals + LessThan + + Comma + Hyphen + ExclamationMark + OpenParen + CloseParen + OpenBracket + CloseBracket +) + +func (t TokenType) String() string { + switch t { + case Identifier: + return "identifier" + case Subquery: + return "subquery" + case Select: + return "select" + case From: + return "from" + case As: + return "as" + case Where: + return "where" + case Or: + return "or" + case And: + return "and" + case Not: + return "not" + case In: + return "in" + case Is: + return "is" + case Like: + return "like" + case RLike: + return "RLike" + case Equals: + return "equal" + case NotEquals: + return "not-equal" + case GreaterThanEquals: + return "greater-than-or-equal" + case GreaterThan: + return "greater-than" + case LessThanEquals: + return "less-than-or-equal" + case LessThan: + return "less-than" + case Comma: + return "comma" + case Hyphen: + return "hyphen" + case ExclamationMark: + return "exclamation-mark" + case OpenParen: + return "open-parentheses" + case CloseParen: + return "close-parentheses" + case OpenBracket: + return "open-bracket" + case CloseBracket: + return "close-bracket" + default: + return "unknown" + } +} + +// Token represents a single token. +type Token struct { + Type TokenType + Raw string +} + +func (t *Token) String() string { + return fmt.Sprintf("{type: %s, raw: \"%s\"}", + t.Type.String(), t.Raw) +} + +// Tokenizer represents a token worker. +type Tokenizer struct { + input []rune + tokens []*Token +} + +// NewTokenizer initializes a new Tokenizer. +func NewTokenizer(input string) *Tokenizer { + return &Tokenizer{ + input: []rune(input), + tokens: make([]*Token, 0), + } +} diff --git a/tokenizer/token_test.go b/tokenizer/token_test.go new file mode 100644 index 0000000..5ce5171 --- /dev/null +++ b/tokenizer/token_test.go @@ -0,0 +1,87 @@ +package tokenizer + +import "testing" + +func TestToken_TokenTypeString(t *testing.T) { + type Case struct { + tt TokenType + expected string + } + + cases := []Case{ + {tt: Identifier, expected: "identifier"}, + {tt: Subquery, expected: "subquery"}, + {tt: Select, expected: "select"}, + {tt: From, expected: "from"}, + {tt: As, expected: "as"}, + {tt: Where, expected: "where"}, + {tt: Or, expected: "or"}, + {tt: And, expected: "and"}, + {tt: Not, expected: "not"}, + {tt: In, expected: "in"}, + {tt: Is, expected: "is"}, + {tt: Like, expected: "like"}, + {tt: RLike, expected: "RLike"}, + {tt: Equals, expected: "equal"}, + {tt: NotEquals, expected: "not-equal"}, + {tt: GreaterThanEquals, expected: "greater-than-or-equal"}, + {tt: GreaterThan, expected: "greater-than"}, + {tt: LessThanEquals, expected: "less-than-or-equal"}, + {tt: LessThan, expected: "less-than"}, + {tt: Comma, expected: "comma"}, + {tt: Hyphen, expected: "hyphen"}, + {tt: ExclamationMark, expected: "exclamation-mark"}, + {tt: OpenParen, expected: "open-parentheses"}, + {tt: CloseParen, expected: "close-parentheses"}, + {tt: OpenBracket, expected: "open-bracket"}, + {tt: CloseBracket, expected: "close-bracket"}, + {tt: Unknown, expected: "unknown"}, + } + + for _, c := range cases { + actual := c.tt.String() + if c.expected != actual { + t.Fatalf("\nExpected %v\n Got %v", c.expected, actual) + } + } +} + +func TestToken_String(t *testing.T) { + type Case struct { + token Token + expected string + } + + cases := []Case{ + { + token: Token{Type: Identifier, Raw: "name"}, + expected: "{type: identifier, raw: \"name\"}", + }, + { + token: Token{Type: Comma, Raw: ","}, + expected: "{type: comma, raw: \",\"}", + }, + } + + for _, c := range cases { + actual := c.token.String() + if c.expected != actual { + t.Fatalf("\nExpected %v\n Got %v", c.expected, actual) + } + } +} + +func TestToken_NewTokenizer(t *testing.T) { + input := "SELECT all FROM ." + inputLength, tokenLength := len([]rune(input)), 0 + + tokenizer := NewTokenizer(input) + if len(tokenizer.input) != inputLength { + t.Fatalf("len(tokenizer.input)\nExpected %v\n Got %v", inputLength, + len(tokenizer.input)) + } + if len(tokenizer.tokens) != tokenLength { + t.Fatalf("len(tokenizer.tokens)\nExpected %v\n Got %v", tokenLength, + len(tokenizer.tokens)) + } +} diff --git a/tokenizer/tokenizer.go b/tokenizer/tokenizer.go new file mode 100644 index 0000000..6de4365 --- /dev/null +++ b/tokenizer/tokenizer.go @@ -0,0 +1,244 @@ +package tokenizer + +import ( + "fmt" + "strings" + "unicode" +) + +// All parses all tokens for this Tokenizer. +func (t *Tokenizer) All() []Token { + tokens := []Token{} + for tok := t.Next(); tok != nil; tok = t.Next() { + tokens = append(tokens, *tok) + } + return tokens +} + +// Next finds and returns the next Token in the input string. +func (t *Tokenizer) Next() *Token { + for unicode.IsSpace(t.current()) { + t.input = t.input[1:] + } + + current := t.current() + if current == -1 { + return nil + } + + switch current { + case '(': + t.input = t.input[1:] + return t.setToken(&Token{Type: OpenParen, Raw: "("}) + case ')': + t.input = t.input[1:] + return t.setToken(&Token{Type: CloseParen, Raw: ")"}) + case '[': + t.input = t.input[1:] + return t.setToken(&Token{Type: OpenBracket, Raw: "["}) + case ']': + t.input = t.input[1:] + return t.setToken(&Token{Type: CloseBracket, Raw: "]"}) + case ',': + t.input = t.input[1:] + return t.setToken(&Token{Type: Comma, Raw: ","}) + case '-': + t.input = t.input[1:] + return t.setToken(&Token{Type: Hyphen, Raw: "-"}) + case '!': + if t.getRuneAt(1) == '=' { + t.input = t.input[2:] + return t.setToken(&Token{Type: NotEquals, Raw: "!="}) + } + return t.setToken(&Token{Type: ExclamationMark, Raw: "!"}) + case '=': + t.input = t.input[1:] + return t.setToken(&Token{Type: Equals, Raw: "="}) + case '>': + if t.getRuneAt(1) == '=' { + t.input = t.input[2:] + return t.setToken(&Token{Type: GreaterThanEquals, Raw: ">="}) + } + t.input = t.input[1:] + return t.setToken(&Token{Type: GreaterThan, Raw: ">"}) + case '<': + if t.getRuneAt(1) == '=' { + t.input = t.input[2:] + return t.setToken(&Token{Type: LessThanEquals, Raw: "<="}) + } + if t.getRuneAt(1) == '>' { + t.input = t.input[2:] + return t.setToken(&Token{Type: NotEquals, Raw: "<>"}) + } + t.input = t.input[1:] + return t.setToken(&Token{Type: LessThan, Raw: "<"}) + } + + if !t.currentIs(-1, ',', '\'', '"', '`', '(', ')', '[', ']') { + word := t.readWord() + tok := &Token{Raw: word} + + switch strings.ToUpper(word) { + case "SELECT": + tok.Type = Select + case "FROM": + tok.Type = From + case "WHERE": + tok.Type = Where + case "AS": + tok.Type = As + case "OR": + tok.Type = Or + case "AND": + tok.Type = And + case "NOT": + tok.Type = Not + case "IN": + tok.Type = In + case "IS": + tok.Type = Is + case "LIKE": + tok.Type = Like + case "REGEXP", "RLIKE": + tok.Type = RLike + default: + tok.Type = Identifier + } + + if t.getPreviousToken() != nil && t.getPreviousToken().Type == OpenParen && + t.getTokenAt(1) != nil && t.getTokenAt(1).Type == In { + // The two previous tokens were: `IN` and `(`, so we're at a subquery. + tok.Type = Subquery + tok.Raw = fmt.Sprintf("%s %s", word, t.readQuery()) + } + + return t.setToken(tok) + } + + tok := &Token{Type: Unknown, Raw: string(current)} + + // If the current rune is a single/double quote or backtick, we want to keep + // reading until we reach the matching closing symbol. + if t.currentIs('\'', '"', '`') { + t.input = t.input[1:] + tok.Raw = t.readWord() + t.readUntil(current) + tok.Type = Identifier + } + + t.input = t.input[1:] + return t.setToken(tok) +} + +// setToken adds token to the list of this Tokenizer's tokens. +func (t *Tokenizer) setToken(token *Token) *Token { + t.tokens = append(t.tokens, token) + return token +} + +// getPreviousToken returns the token that was most-recently read. +func (t *Tokenizer) getPreviousToken() *Token { + return t.getTokenAt(0) +} + +// getTokenAt returns the token at index i from the end of the tokens slice. +func (t *Tokenizer) getTokenAt(i int) *Token { + j := len(t.tokens) - 1 - i + if j < 0 { + return nil + } + + return t.tokens[j] +} + +// current returns the run at the 0th index of the input. +func (t *Tokenizer) current() rune { + return t.getRuneAt(0) +} + +// getRuneAt returns the rune at the ith index of the input. +func (t *Tokenizer) getRuneAt(i int) rune { + if len(t.input) == i { + return -1 + } + + return t.input[i] +} + +// currentIs returns true iff the input's current rune (at index 0) is in runes. +func (t *Tokenizer) currentIs(runes ...rune) bool { + for _, r := range runes { + if r == t.current() { + return true + } + } + return false +} + +// readWord reads a single word from the input. Returns when the next rune is +// any of: nil (-1), empty space, comma, single/double quote, backtick, +// or opening/closing parenthesis/bracket. +func (t *Tokenizer) readWord() string { + word := []rune{} + + for { + if unicode.IsSpace(t.current()) || + t.currentIs(-1, ',', '\'', '"', '`', '(', ')', '[', ']') { + return string(word) + } + + word = append(word, t.current()) + t.input = t.input[1:] + } +} + +// readQuery reads a full string until reaching a closing parentheses. Counts +// opening parens to ensure that balance is maintained. +func (t *Tokenizer) readQuery() string { + var query string + + var count = 1 + for count > 0 { + for unicode.IsSpace(t.current()) { + t.input = t.input[1:] + } + + word := fmt.Sprintf("%s", t.readWord()) + + if t.current() == -1 { + break + } + + if t.current() == '(' { + count++ + word = "(" + } else if t.current() == ')' { + count-- + if count <= 0 { + query += word + break + } + word = ")" + } else if t.currentIs('\'', '`') { + word += string(t.current()) + } else { + word += " " + } + + query += word + t.input = t.input[1:] + } + + return query +} + +// readUnitl reads the input starting at start, until reaching a rune in runes. +func (t *Tokenizer) readUntil(runes ...rune) string { + var word string + for !t.currentIs(runes...) { + for unicode.IsSpace(t.current()) { + t.input = t.input[1:] + } + word = fmt.Sprintf("%s %s", word, t.readWord()) + } + return word +} diff --git a/tokenizer/tokenizer_test.go b/tokenizer/tokenizer_test.go new file mode 100644 index 0000000..0763453 --- /dev/null +++ b/tokenizer/tokenizer_test.go @@ -0,0 +1,202 @@ +package tokenizer + +import ( + "reflect" + "testing" +) + +func TestTokenizer_NextTokenType(t *testing.T) { + type Case struct { + input string + expected TokenType + } + + cases := []Case{ + {input: "SELECT", expected: Select}, + {input: "FROM", expected: From}, + {input: "WHERE", expected: Where}, + {input: "AS", expected: As}, + {input: "OR", expected: Or}, + {input: "AND", expected: And}, + {input: "NOT", expected: Not}, + {input: "IN", expected: In}, + {input: "IS", expected: Is}, + {input: "LIKE", expected: Like}, + {input: "RLIKE", expected: RLike}, + {input: "foo", expected: Identifier}, + {input: "(", expected: OpenParen}, + {input: ")", expected: CloseParen}, + {input: ",", expected: Comma}, + {input: "-", expected: Hyphen}, + {input: "=", expected: Equals}, + {input: "<>", expected: NotEquals}, + {input: "<", expected: LessThan}, + {input: "<=", expected: LessThanEquals}, + {input: ">", expected: GreaterThan}, + {input: ">=", expected: GreaterThanEquals}, + } + + for _, c := range cases { + actual := NewTokenizer(c.input).Next() + expected := &Token{Type: c.expected, Raw: c.input} + if !reflect.DeepEqual(actual, expected) { + t.Fatalf("\nExpected: %v\n Got: %v", expected, actual) + } + } +} + +func TestTokenizer_NextRaw(t *testing.T) { + type Case struct { + input string + expected string + } + + // TODO: Fix the last 2 cases, they're currently hanging. + cases := []Case{ + {input: "foo", expected: "foo"}, + {input: " foo ", expected: "foo"}, + {input: "\" foo \"", expected: " foo "}, + {input: "' foo '", expected: " foo "}, + {input: "` foo `", expected: " foo "}, + // Case{input: "\"foo'bar\"", expected: "foo'bar"}, + // Case{input: "\"()\"", expected: "()"}, + } + + for _, c := range cases { + actual := NewTokenizer(c.input).Next() + expected := &Token{Type: Identifier, Raw: c.expected} + if !reflect.DeepEqual(actual, expected) { + t.Fatalf("\nExpected: %v\n Got: %v", expected, actual) + } + } +} + +func TestTokenizer_AllSimple(t *testing.T) { + input := ` + SELECT + name, size + FROM + ~/Desktop + WHERE + name LIKE %go + ` + + actual := NewTokenizer(input).All() + expected := []Token{ + {Type: Select, Raw: "SELECT"}, + {Type: Identifier, Raw: "name"}, + {Type: Comma, Raw: ","}, + {Type: Identifier, Raw: "size"}, + {Type: From, Raw: "FROM"}, + {Type: Identifier, Raw: "~/Desktop"}, + {Type: Where, Raw: "WHERE"}, + {Type: Identifier, Raw: "name"}, + {Type: Like, Raw: "LIKE"}, + {Type: Identifier, Raw: "%go"}, + } + + for i := range expected { + if !reflect.DeepEqual(actual[i], expected[i]) { + t.Fatalf("\nExpected: %v\n Got: %v", expected[i], actual[i]) + } + } +} + +func TestTokenizer_AllSubquery(t *testing.T) { + input := ` + SELECT + name, size + FROM + ~/Desktop + WHERE + name LIKE %go OR + name IN ( + SELECT + name + FROM + $GOPATH/src/github.com + WHERE + name RLIKE .*_test\.go) + ` + + actual := NewTokenizer(input).All() + expected := []Token{ + {Type: Select, Raw: "SELECT"}, + {Type: Identifier, Raw: "name"}, + {Type: Comma, Raw: ","}, + {Type: Identifier, Raw: "size"}, + {Type: From, Raw: "FROM"}, + {Type: Identifier, Raw: "~/Desktop"}, + {Type: Where, Raw: "WHERE"}, + {Type: Identifier, Raw: "name"}, + {Type: Like, Raw: "LIKE"}, + {Type: Identifier, Raw: "%go"}, + {Type: Or, Raw: "OR"}, + {Type: Identifier, Raw: "name"}, + {Type: In, Raw: "IN"}, + {Type: OpenParen, Raw: "("}, + {Type: Subquery, Raw: "SELECT name FROM $GOPATH/src/github.com WHERE name RLIKE .*_test\\.go"}, + {Type: CloseParen, Raw: ")"}, + } + + for i := range expected { + if !reflect.DeepEqual(actual[i], expected[i]) { + t.Fatalf("\nExpected: %v\n Got: %v", expected[i], actual[i]) + } + } +} + +func TestTokenizer_ReadWord(t *testing.T) { + type Case struct { + input string + expected string + } + + cases := []Case{ + {input: "foo", expected: "foo"}, + {input: "foo bar", expected: "foo"}, + {input: "", expected: ""}, + } + + for _, c := range cases { + actual := NewTokenizer(c.input).readWord() + if !reflect.DeepEqual(actual, c.expected) { + t.Fatalf("\nExpected: %v\n Got: %v", c.expected, actual) + } + } +} + +func TestTokenizer_ReadQuery(t *testing.T) { + type Case struct { + input string + expected string + } + + // TODO: Complete these cases. + cases := []Case{} + + for _, c := range cases { + actual := NewTokenizer(c.input).readQuery() + if !reflect.DeepEqual(actual, c.expected) { + t.Fatalf("\nExpected: %v\n Got: %v", c.expected, actual) + } + } +} + +func TestTokenizer_ReadUntil(t *testing.T) { + type Case struct { + input string + until []rune + expected string + } + + // TODO: Complete these cases. + cases := []Case{} + + for _, c := range cases { + actual := NewTokenizer(c.input).readUntil(c.until...) + if !reflect.DeepEqual(actual, c.expected) { + t.Fatalf("\nExpected: %v\n Got: %v", c.expected, actual) + } + } +} diff --git a/transform/common.go b/transform/common.go new file mode 100644 index 0000000..fe13d0d --- /dev/null +++ b/transform/common.go @@ -0,0 +1,82 @@ +package transform + +import ( + "crypto/sha1" + "encoding/hex" + "hash" + "os" + "path/filepath" + "strings" +) + +// formatName runs the correct name format function based on the value of arg. +func formatName(arg, name string) interface{} { + switch strings.ToUpper(arg) { + case "UPPER": + return upper(name) + case "LOWER": + return lower(name) + } + return nil +} + +// upper returns the uppercased version of name. +func upper(name string) interface{} { + return strings.ToUpper(name) +} + +// lower returns the lowercase version of name. +func lower(name string) interface{} { + return strings.ToLower(name) +} + +// truncate returns the first n characters of str. If n is greater than the +// length of str or less than 0, return str. +func truncate(str string, n int) string { + if len(str) < n || n < 0 { + return str + } + + return str[0:n] +} + +// FindHash returns a func to create a new hash based on the provided name. +func FindHash(name string) func() hash.Hash { + switch strings.ToUpper(name) { + case "SHA1": + return sha1.New + } + return nil +} + +// ComputeHash applies the hash h to the file located at path. Returns a line +// of dashes for directories. +func ComputeHash(info os.FileInfo, path string, h hash.Hash) (interface{}, error) { + fallback := strings.Repeat("-", h.Size()*2) + + // If the current file is a symlink, attempt to evaluate the link and + // stat the resultant file. If either process fails, ignore the error and + // return the fallback. + if info.Mode()&os.ModeSymlink == os.ModeSymlink { + var err error + if path, err = filepath.EvalSymlinks(path); err != nil { + return fallback, nil + } + if info, err = os.Stat(path); err != nil { + return fallback, nil + } + } + + if info.IsDir() { + return fallback, nil + } + + b, err := os.ReadFile(path) + if err != nil { + return nil, err + } + if _, err := h.Write(b); err != nil { + return nil, err + } + return hex.EncodeToString(h.Sum(nil)), nil +} diff --git a/transform/common_test.go b/transform/common_test.go new file mode 100644 index 0000000..0c9c594 --- /dev/null +++ b/transform/common_test.go @@ -0,0 +1,144 @@ +package transform + +import ( + "crypto/sha1" + "hash" + "os" + "reflect" + "strings" + "testing" +) + +func TestCommon_FormatName(t *testing.T) { + type Case struct { + arg string + name string + expected string + } + + cases := []Case{ + {arg: "upper", name: "foo", expected: "FOO"}, + {arg: "upper", name: "FOO", expected: "FOO"}, + {arg: "lower", name: "foo", expected: "foo"}, + {arg: "lower", name: "FOO", expected: "foo"}, + } + + for _, c := range cases { + result := formatName(c.arg, c.name) + if result != c.expected { + t.Fatalf("\nExpected: %s\n Got: %s", c.expected, result) + } + } +} + +func TestCommon_Upper(t *testing.T) { + type Case struct { + name string + expected string + } + + cases := []Case{ + {name: "foo", expected: "FOO"}, + {name: "FOO", expected: "FOO"}, + } + + for _, c := range cases { + result := upper(c.name) + if result != c.expected { + t.Fatalf("\nExpected: %s\n Got: %s", c.expected, result) + } + } +} + +func TestCommon_Lower(t *testing.T) { + type Case struct { + name string + expected string + } + + cases := []Case{ + {name: "foo", expected: "foo"}, + {name: "FOO", expected: "foo"}, + } + + for _, c := range cases { + result := lower(c.name) + if result != c.expected { + t.Fatalf("\nExpected: %s\n Got: %s", c.expected, result) + } + } +} + +func TestCommon_Truncate(t *testing.T) { + input := "foo-bar-baz" + + type Case struct { + n int + expected string + } + + cases := []Case{ + {n: 3, expected: "foo"}, + {n: 7, expected: "foo-bar"}, + {n: 100, expected: input}, + {n: -1, expected: input}, + {n: len(input), expected: "foo-bar-baz"}, + {n: len(input) - 1, expected: "foo-bar-ba"}, + } + + for _, c := range cases { + actual := truncate(input, c.n) + if c.expected != actual { + t.Fatalf("\nExpected: %s\n Got: %s", c.expected, actual) + } + } +} + +func TestCommon_FindHash(t *testing.T) { + type Case struct { + name string + expected hash.Hash + } + + cases := []Case{ + {name: "SHA1", expected: sha1.New()}, + {name: "FOO", expected: nil}, + } + + for _, c := range cases { + actual := FindHash(c.name) + if actual == nil { + if c.expected != nil { + t.Fatalf("\nExpected: %s\n Got: nil", c.expected) + } + } else if h := actual(); !reflect.DeepEqual(c.expected, h) { + t.Fatalf("\nExpected: %s\n Got: %s", c.expected, h) + } + } +} + +func TestCommon_ComputeHash(t *testing.T) { + type Case struct { + path string + expected string + } + + cases := []Case{ + {path: "../testdata/foo", expected: strings.Repeat("-", 40)}, + {path: "../testdata/baz", expected: "da39a3ee5e6b4b0d3255bfef95601890afd80709"}, + } + + for _, c := range cases { + info, err := os.Stat(c.path) + if err != nil { + t.Fatalf("\nExpected no error\n Got: %s", err.Error()) + } + actual, err := ComputeHash(info, c.path, sha1.New()) + if err != nil { + t.Fatalf("\nExpected no error\n Got: %s", err.Error()) + } + if !reflect.DeepEqual(c.expected, actual) { + t.Fatalf("\nExpected: %s\n Got: %s", c.expected, actual) + } + } +} diff --git a/transform/error.go b/transform/error.go new file mode 100644 index 0000000..18188af --- /dev/null +++ b/transform/error.go @@ -0,0 +1,28 @@ +package transform + +import ( + "fmt" + "strings" +) + +// ErrNotImplemented used for non-implemented modifier functions. +type ErrNotImplemented struct { + Name string + Attribute string +} + +func (e *ErrNotImplemented) Error() string { + return fmt.Sprintf("function %s is not implemented for attribute %s", + strings.ToUpper(e.Name), e.Attribute) +} + +// ErrUnsupportedFormat used for unsupport arguments for FORMAT functions. +type ErrUnsupportedFormat struct { + Format string + Attribute string +} + +func (e *ErrUnsupportedFormat) Error() string { + return fmt.Sprintf("unsupported format type %s for attribute %s", + e.Format, e.Attribute) +} diff --git a/transform/error_test.go b/transform/error_test.go new file mode 100644 index 0000000..f8eef70 --- /dev/null +++ b/transform/error_test.go @@ -0,0 +1,22 @@ +package transform + +import "testing" + +func TestTransform_ErrNotImplemented(t *testing.T) { + err := &ErrNotImplemented{"n", "a"} + expected := "function N is not implemented for attribute a" + actual := err.Error() + if expected != actual { + t.Fatalf("\nExpected: %s\n Got: %s", expected, actual) + } +} + +func TestTransform_ErrUnsupportedFormat(t *testing.T) { + err := &ErrUnsupportedFormat{"f", "a"} + expected := "unsupported format type f for attribute a" + actual := err.Error() + if expected != actual { + t.Fatalf("\nExpected: %s\n Got: %s", expected, actual) + } + +} diff --git a/transform/format.go b/transform/format.go new file mode 100644 index 0000000..7718d5b --- /dev/null +++ b/transform/format.go @@ -0,0 +1,159 @@ +package transform + +import ( + "fmt" + "hash" + "os" + "strconv" + "strings" + "time" +) + +const defaultHashLength = 7 + +// FormatParams holds the params for a format-modifier function. +type FormatParams struct { + Attribute string + Path string + Info os.FileInfo + Value interface{} + + Name string + Args []string +} + +// Format runs the respective format function on the provided parameters. +func Format(p *FormatParams) (val interface{}, err error) { + switch strings.ToUpper(p.Name) { + case "FORMAT": + val, err = p.format() + case "UPPER": + val = upper(p.Value.(string)) + case "LOWER": + val = lower(p.Value.(string)) + case "FULLPATH": + val, err = p.fullPath() + case "SHORTPATH": + val, err = p.shortPath() + case "SHA1": + val, err = p.hash(FindHash(p.Name)()) + } + if err != nil { + return nil, err + } + if val == nil { + return nil, &ErrNotImplemented{p.Name, p.Attribute} + } + return val, nil +} + +// format runs a format function based on the value of the provided attribute. +func (p *FormatParams) format() (val interface{}, err error) { + switch p.Attribute { + case "name": + val = formatName(p.Args[0], p.Value.(string)) + case "size": + val, err = p.formatSize() + case "time": + val, err = p.formatTime() + } + if err != nil { + return nil, err + } + if val == nil { + return nil, &ErrUnsupportedFormat{p.Args[0], p.Attribute} + } + return val, nil +} + +// formatSize formats a size. Valid arguments include `KB`, `MB`, `GB` (case +// insensitive). +func (p *FormatParams) formatSize() (interface{}, error) { + size := p.Value.(int64) + switch strings.ToUpper(p.Args[0]) { + case "KB": + return fmt.Sprintf("%fkb", float64(size)/(1<<10)), nil + case "MB": + return fmt.Sprintf("%fmb", float64(size)/(1<<20)), nil + case "GB": + return fmt.Sprintf("%fgb", float64(size)/(1<<30)), nil + } + return nil, nil +} + +// formatTime formats a time. Valid arguments include `UNIX` and `ISO` (case +// insensitive), or a custom layout layout. If a custom layout is provided, it +// must be set according to 2006-01-02T15:04:05.999999-07:00. +func (p *FormatParams) formatTime() (interface{}, error) { + switch strings.ToUpper(p.Args[0]) { + case "ISO": + return p.Info.ModTime().Format(time.RFC3339), nil + case "UNIX": + return p.Info.ModTime().Format(time.UnixDate), nil + default: + return p.Info.ModTime().Format(p.Args[0]), nil + } +} + +// fullPath returns the full path of the current file. Only supports the +// `name` attribute. +func (p *FormatParams) fullPath() (interface{}, error) { + if p.Attribute != "name" { + return nil, nil + } + return p.Path, nil +} + +// shortPath returns the short path of the current file. Only supports the +// `name` attribute. +func (p *FormatParams) shortPath() (interface{}, error) { + if p.Attribute != "name" { + return nil, nil + } + return p.Info.Name(), nil +} + +// hash applies the provided hash algorithm h with ComputeHash. +func (p *FormatParams) hash(h hash.Hash) (interface{}, error) { + var ( + err error + n int + result interface{} + ) + + if len(p.Args) == 0 || p.Args[0] == "" { + n = defaultHashLength + } else if strings.ToUpper(p.Args[0]) == "FULL" { + n = -1 + } else if n, err = strconv.Atoi(p.Args[0]); err != nil { + return nil, err + } + + if result, err = ComputeHash(p.Info, p.Path, h); err != nil { + return nil, err + } + + return truncate(result.(string), n), nil +} + +// DefaultFormatValue returns the default format value for the provided +// attribute attr based on path and info. +func DefaultFormatValue(attr, path string, info os.FileInfo) (value interface{}, err error) { + switch attr { + case "mode": + value = info.Mode() + case "name": + value = info.Name() + case "size": + value = info.Size() + case "time": + value = info.ModTime().Format(time.Stamp) + case "hash": + if value, err = ComputeHash(info, path, FindHash("SHA1")()); value != nil { + value = truncate(value.(string), defaultHashLength) + } + default: + err = fmt.Errorf("unknown attribute %s", attr) + } + return value, err +} diff --git a/transform/format_test.go b/transform/format_test.go new file mode 100644 index 0000000..21cdd1e --- /dev/null +++ b/transform/format_test.go @@ -0,0 +1,94 @@ +package transform + +import ( + "fmt" + "reflect" + "testing" +) + +func TestTransform_Format(t *testing.T) { + type Expected struct { + val interface{} + err error + } + + type Case struct { + params *FormatParams + expected Expected + } + + // TODO: Add tests for the time and hash attribute. + cases := []Case{ + { + params: &FormatParams{ + Attribute: "size", + Path: "path", + Info: nil, + Value: int64(300), + Name: "format", + Args: []string{"kb"}, + }, + expected: Expected{ + val: fmt.Sprintf("%fkb", float64(300)/(1<<10)), + err: nil, + }, + }, + { + params: &FormatParams{ + Attribute: "size", + Path: "path", + Info: nil, + Value: int64(300), + Name: "format", + Args: []string{"kilobytes"}, + }, + expected: Expected{ + val: nil, + err: &ErrUnsupportedFormat{"kilobytes", "size"}, + }, + }, + { + params: &FormatParams{ + Attribute: "name", + Path: "path", + Info: nil, + Value: "VALUE", + Name: "format", + Args: []string{"lower"}, + }, + expected: Expected{val: "value", err: nil}, + }, + { + params: &FormatParams{ + Attribute: "name", + Path: "path", + Info: nil, + Value: "value", + Name: "upper", + Args: []string{}, + }, + expected: Expected{val: "VALUE", err: nil}, + }, + { + params: &FormatParams{ + Attribute: "name", + Path: "path", + Info: nil, + Value: "value", + Name: "fullpath", + Args: []string{}, + }, + expected: Expected{val: "path", err: nil}, + }, + } + + for _, c := range cases { + val, err := Format(c.params) + if !(reflect.DeepEqual(val, c.expected.val) && + reflect.DeepEqual(err, c.expected.err)) { + t.Fatalf("\nExpected: %v, %v\n Got: %v, %v", + c.expected.val, c.expected.err, + val, err) + } + } +} diff --git a/transform/parse.go b/transform/parse.go new file mode 100644 index 0000000..a3546ab --- /dev/null +++ b/transform/parse.go @@ -0,0 +1,148 @@ +package transform + +import ( + "hash" + "os" + "reflect" + "strconv" + "strings" + "time" +) + +// ParseParams holds the params for a parse-modifier function. +type ParseParams struct { + Attribute string + Value interface{} + + Name string + Args []string +} + +// Parse runs the associated modifier function for the provided parameters. +// Depending on the type of p.Value, we may recursively run this method +// on every element of the structure. +// +// We're using reflect _quite_ heavily for this, meaning it's kind of unsafe, +// it'd be great if we could find another solution while keeping it as +// abstract as it is. +func Parse(p *ParseParams) (val interface{}, err error) { + kind := reflect.TypeOf(p.Value).Kind() + + // If we have a slice/array, recursively run Parse on each element. + if kind == reflect.Slice || kind == reflect.Array { + s := reflect.ValueOf(p.Value) + for i := 0; i < s.Len(); i++ { + p.Value = s.Index(i).Interface() + if val, err = Parse(p); err != nil { + return nil, err + } + s.Index(i).Set(reflect.ValueOf(val)) + } + return s.Interface(), nil + } + + // If we have a map, recursively run Parse on each *key* and create a new + // map out of the return values. + if kind == reflect.Map { + result := reflect.MakeMap(reflect.TypeOf(p.Value)) + for _, key := range reflect.ValueOf(p.Value).MapKeys() { + p.Value = key.Interface() + if val, err = Parse(p); err != nil { + return nil, err + } + result.SetMapIndex(reflect.ValueOf(val), reflect.ValueOf(true)) + } + return result.Interface(), nil + } + + // Not a slice nor a map. + switch strings.ToUpper(p.Name) { + case "FORMAT": + val, err = p.format() + case "UPPER": + val = upper(p.Value.(string)) + case "LOWER": + val = lower(p.Value.(string)) + case "SHA1": + val, err = p.hash(FindHash(p.Name)()) + } + + if err != nil { + return nil, err + } + if val == nil { + return nil, &ErrNotImplemented{p.Name, p.Attribute} + } + return val, nil +} + +// format runs the correct format function based on the provided attribute. +func (p *ParseParams) format() (val interface{}, err error) { + switch p.Attribute { + case "name": + val = formatName(p.Args[0], p.Value.(string)) + case "size": + val, err = p.formatSize() + case "time": + val, err = p.formatTime() + } + if err != nil { + return nil, err + } + if val == nil { + return nil, &ErrUnsupportedFormat{p.Args[0], p.Attribute} + } + return val, nil +} + +// formatSize formats the size attribute. Valid arguments include `B`, `KB`, +// `MB`, and `GB` (case insensitive). +func (p *ParseParams) formatSize() (interface{}, error) { + size, err := strconv.ParseFloat(p.Value.(string), 64) + if err != nil { + return nil, err + } + switch strings.ToUpper(p.Args[0]) { + case "B": + size *= 1 + case "KB": + size *= 1 << 10 + case "MB": + size *= 1 << 20 + case "GB": + size *= 1 << 30 + default: + return nil, nil + } + return size, nil +} + +// formatTime formats the time attribute. Valid arguments include `ISO`, +// `UNIX`, (case insensitive) or a custom layout. If a custom layout is +// provided, it must be set according to 2006-01-02T15:04:05.999999-07:00. +func (p *ParseParams) formatTime() (interface{}, error) { + var ( + t time.Time + err error + ) + switch strings.ToUpper(p.Args[0]) { + case "ISO": + t, err = time.Parse(time.RFC3339, p.Value.(string)) + case "UNIX": + t, err = time.Parse(time.UnixDate, p.Value.(string)) + default: + t, err = time.Parse(p.Args[0], p.Value.(string)) + } + if err != nil { + return nil, err + } + return t, nil +} + +func (p *ParseParams) hash(h hash.Hash) (interface{}, error) { + info, err := os.Stat(p.Value.(string)) + if err != nil { + return nil, err + } + return ComputeHash(info, p.Value.(string), h) +} diff --git a/transform/parse_test.go b/transform/parse_test.go new file mode 100644 index 0000000..1afc416 --- /dev/null +++ b/transform/parse_test.go @@ -0,0 +1,31 @@ +package transform + +import ( + "reflect" + "testing" +) + +type ParseOutput struct { + val interface{} + err error +} + +type ParseCase struct { + params *ParseParams + expected ParseOutput +} + +func TestTransform_Parse(t *testing.T) { + // TODO: Complete this. + cases := []ParseCase{} + + for _, c := range cases { + val, err := Parse(c.params) + if !(reflect.DeepEqual(val, c.expected.val) && + reflect.DeepEqual(err, c.expected.err)) { + t.Fatalf("\nExpected: %v, %v\n Got: %v, %v", + c.expected.val, c.expected.err, + val, err) + } + } +}