wehub/triggerdotdev--trigger.dev
1
0
Fork 0
Code Issues Pull Requests Actions 2 Deployments Models Agent Notes Packages Projects Releases Wiki Activity

chore: import upstream snapshot with attribution

Browse Source
This commit is contained in:
wehub-resource-sync
2026-07-13 13:32:57 +08:00
commit cd420f9332
4811 changed files with 884702 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
internal-packages/tsql/.gitignore
+3
View File
@@ -0,0 +1,3 @@
# ANTLR intermediate build files
src/grammar/.antlr/
internal-packages/tsql/NOTICE.md
+22
View File
@@ -0,0 +1,22 @@
Portions of this package are derived from PostHog (MIT License).
Copyright (c) 2020-2025 PostHog Inc.
The original license is reproduced below:
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
internal-packages/tsql/README.md
+19
View File
@@ -0,0 +1,19 @@
# TSQL (TriggerSQL)
TriggerSQL is a DSL that is safely converted into ClickHouse SQL queries with protection against SQL injection and it's tenant-safe (users can only query their own data).
## Attribution
This package is derived from [PostHog's HogQL](https://github.com/PostHog/posthog/tree/master/posthog/hogql) (MIT License). See [NOTICE.md](./NOTICE.md) for the full copyright notice.
## ANTLR Grammar
The ANTLR grammar is heavily inspired by PostHog's HogQL.
These are found in [./src/grammar](./src/grammar) and are the `.g4` files.
## Generating the source code
```sh
pnpm run grammar:build
```
internal-packages/tsql/package.json
+23
View File
@@ -0,0 +1,23 @@
{
"name": "@internal/tsql",
"private": true,
"version": "0.0.1",
"main": "./src/index.ts",
"types": "./src/index.ts",
"type": "module",
"dependencies": {
"@trigger.dev/core": "workspace:*",
"antlr4ts": "0.5.0-alpha.4",
"zod": "3.25.76"
},
"scripts": {
"typecheck": "tsc --noEmit",
"grammar:build": "pnpm grammar:build:typescript",
"grammar:build:typescript": "cat src/grammar/TSQLLexer.typescript.g4 > src/grammar/TSQLLexer.g4 && tail -n +2 src/grammar/TSQLLexer.common.g4 |sed s/isOpeningTag/self.isOpeningTag/ >> src/grammar/TSQLLexer.g4 && antlr4ts src/grammar/TSQLLexer.g4 && rm src/grammar/TSQLLexer.g4 && antlr4ts -visitor -no-listener -Dlanguage=TypeScript src/grammar/TSQLParser.g4",
"test": "vitest --sequence.concurrent=false --no-file-parallelism",
"test:coverage": "vitest --sequence.concurrent=false --no-file-parallelism --coverage.enabled"
},
"devDependencies": {
"antlr4ts-cli": "0.5.0-alpha.4"
}
}
internal-packages/tsql/src/grammar/TSQLLexer.common.g4
+288
View File
@@ -0,0 +1,288 @@
lexer grammar TSQLLexer;
// NB! We cat TSQLLexter.typescript.g4 when generating the grammar.
// NOTE: don't forget to add new keywords to the parser rule "keyword"!
// Keywords
ALL: A L L;
AND: A N D;
ANTI: A N T I;
ANY: A N Y;
ARRAY: A R R A Y;
AS: A S;
ASCENDING: A S C | A S C E N D I N G;
ASOF: A S O F;
BETWEEN: B E T W E E N;
BOTH: B O T H;
BY: B Y;
CASE: C A S E;
CAST: C A S T;
CATCH: C A T C H;
COHORT: C O H O R T;
COLLATE: C O L L A T E;
CROSS: C R O S S;
CUBE: C U B E;
CURRENT: C U R R E N T;
DATE: D A T E;
DAY: D A Y;
DESC: D E S C;
DESCENDING: D E S C E N D I N G;
DISTINCT: D I S T I N C T;
ELSE: E L S E;
END: E N D;
EXCEPT: E X C E P T;
EXTRACT: E X T R A C T;
FINAL: F I N A L;
FINALLY: F I N A L L Y;
FIRST: F I R S T;
FN: F N;
FOLLOWING: F O L L O W I N G;
FOR: F O R;
FROM: F R O M;
FULL: F U L L;
FUN: F U N;
GROUP: G R O U P;
HAVING: H A V I N G;
HOUR: H O U R;
ID: I D;
IF: I F;
ILIKE: I L I K E;
IN: I N;
INF: I N F | I N F I N I T Y;
INNER: I N N E R;
INTERSECT: I N T E R S E C T;
INTERVAL: I N T E R V A L;
IS: I S;
JOIN: J O I N;
KEY: K E Y;
LAST: L A S T;
LEADING: L E A D I N G;
LEFT: L E F T;
LET: L E T;
LIKE: L I K E;
LIMIT: L I M I T;
MINUTE: M I N U T E;
MONTH: M O N T H;
NAN_SQL: N A N; // conflicts with macro NAN
NOT: N O T;
NULL_SQL: N U L L; // conflicts with macro NULL
NULLS: N U L L S;
OFFSET: O F F S E T;
ON: O N;
OR: O R;
ORDER: O R D E R;
OUTER: O U T E R;
OVER: O V E R;
PARTITION: P A R T I T I O N;
PRECEDING: P R E C E D I N G;
PREWHERE: P R E W H E R E;
QUARTER: Q U A R T E R;
RANGE: R A N G E;
RETURN: R E T U R N;
RIGHT: R I G H T;
ROLLUP: R O L L U P;
ROW: R O W;
ROWS: R O W S;
SAMPLE: S A M P L E;
SECOND: S E C O N D;
SELECT: S E L E C T;
SEMI: S E M I;
SETTINGS: S E T T I N G S;
SUBSTRING: S U B S T R I N G;
THEN: T H E N;
THROW: T H R O W;
TIES: T I E S;
TIMESTAMP: T I M E S T A M P;
TO: T O;
TOP: T O P;
TOTALS: T O T A L S;
TRAILING: T R A I L I N G;
TRIM: T R I M;
TRUNCATE: T R U N C A T E;
TRY: T R Y;
UNBOUNDED: U N B O U N D E D;
UNION: U N I O N;
USING: U S I N G;
WEEK: W E E K;
WHEN: W H E N;
WHERE: W H E R E;
WHILE: W H I L E;
WINDOW: W I N D O W;
WITH: W I T H;
YEAR: Y E A R | Y Y Y Y;
// Tokens
// copied from clickhouse_driver/util/escape.py
ESCAPE_CHAR_COMMON
: BACKSLASH B
| BACKSLASH F
| BACKSLASH R
| BACKSLASH N
| BACKSLASH T
| BACKSLASH '0'
| BACKSLASH A
| BACKSLASH V
| BACKSLASH BACKSLASH;
IDENTIFIER
: (LETTER | UNDERSCORE | DOLLAR) (LETTER | UNDERSCORE | DEC_DIGIT | DOLLAR)*
| BACKQUOTE ( ~([\\`]) | ESCAPE_CHAR_COMMON | BACKSLASH QUOTE_SINGLE | (BACKQUOTE BACKQUOTE) )* BACKQUOTE
| QUOTE_DOUBLE ( ~([\\"]) | ESCAPE_CHAR_COMMON | BACKSLASH QUOTE_DOUBLE | (QUOTE_DOUBLE QUOTE_DOUBLE) )* QUOTE_DOUBLE
;
FLOATING_LITERAL
: HEXADECIMAL_LITERAL DOT HEX_DIGIT* (P | E) (PLUS | DASH)? DEC_DIGIT+
| HEXADECIMAL_LITERAL (P | E) (PLUS | DASH)? DEC_DIGIT+
| DECIMAL_LITERAL DOT DEC_DIGIT* E (PLUS | DASH)? DEC_DIGIT+
| DOT DECIMAL_LITERAL E (PLUS | DASH)? DEC_DIGIT+
| DECIMAL_LITERAL E (PLUS | DASH)? DEC_DIGIT+
;
OCTAL_LITERAL: '0' OCT_DIGIT+;
DECIMAL_LITERAL: DEC_DIGIT+;
HEXADECIMAL_LITERAL: '0' X HEX_DIGIT+;
// It's important that quote-symbol is a single character.
STRING_LITERAL: QUOTE_SINGLE ( ~([\\']) | ESCAPE_CHAR_COMMON | BACKSLASH QUOTE_SINGLE | (QUOTE_SINGLE QUOTE_SINGLE) )* QUOTE_SINGLE;
// Alphabet and allowed symbols
fragment A: [aA];
fragment B: [bB];
fragment C: [cC];
fragment D: [dD];
fragment E: [eE];
fragment F: [fF];
fragment G: [gG];
fragment H: [hH];
fragment I: [iI];
fragment J: [jJ];
fragment K: [kK];
fragment L: [lL];
fragment M: [mM];
fragment N: [nN];
fragment O: [oO];
fragment P: [pP];
fragment Q: [qQ];
fragment R: [rR];
fragment S: [sS];
fragment T: [tT];
fragment U: [uU];
fragment V: [vV];
fragment W: [wW];
fragment X: [xX];
fragment Y: [yY];
fragment Z: [zZ];
fragment LETTER: [a-zA-Z];
fragment OCT_DIGIT: [0-7];
fragment DEC_DIGIT: [0-9];
fragment HEX_DIGIT: [0-9a-fA-F];
ARROW: '->';
ASTERISK: '*';
BACKQUOTE: '`';
BACKSLASH: '\\';
COLON: ':';
COMMA: ',';
CONCAT: '||';
DASH: '-';
DOLLAR: '$';
DOT: '.';
EQ_DOUBLE: '==';
EQ_SINGLE: '=';
GT_EQ: '>=';
GT: '>';
HASH: '#';
IREGEX_SINGLE: '~*';
IREGEX_DOUBLE: '=~*';
LBRACE: '{' -> pushMode(DEFAULT_MODE);
LBRACKET: '[';
LPAREN: '(';
LT_EQ: '<=';
TAG_LT_SLASH: '</' -> type(LT_SLASH), pushMode(TSQLX_TAG_CLOSE);
TAG_LT_OPEN: '<' {isOpeningTag()}? -> type(LT), pushMode(TSQLX_TAG_OPEN);
LT: '<';
LT_SLASH: '</';
NOT_EQ: '!=' | '<>';
NOT_IREGEX: '!~*';
NOT_REGEX: '!~';
NULL_PROPERTY: '?.';
NULLISH: '??';
PERCENT: '%';
PLUS: '+';
QUERY: '?';
QUOTE_DOUBLE: '"';
QUOTE_SINGLE_TEMPLATE: 'f\'' -> pushMode(IN_TEMPLATE_STRING); // start of regular f'' template strings
QUOTE_SINGLE_TEMPLATE_FULL: 'F\'' -> pushMode(IN_FULL_TEMPLATE_STRING); // magic F' symbol used to parse "full text" templates
QUOTE_SINGLE: '\'';
REGEX_SINGLE: '~';
REGEX_DOUBLE: '=~';
RBRACE: '}' -> popMode;
RBRACKET: ']';
RPAREN: ')';
SEMICOLON: ';';
SLASH: '/';
SLASH_GT: '/>';
UNDERSCORE: '_';
// Comments and whitespace
MULTI_LINE_COMMENT: '/*' .*? '*/' -> skip;
SINGLE_LINE_COMMENT: ('--' | '//') ~('\n'|'\r')* ('\n' | '\r' | EOF) -> skip;
// whitespace is hidden and not skipped so that it's preserved in ANTLR errors like "no viable alternative"
WHITESPACE: [ \u000B\u000C\t\r\n] -> channel(HIDDEN);
// ───────── f' TEMPLATE STRING MODE ─────────
mode IN_TEMPLATE_STRING;
STRING_TEXT: ((~([\\'{])) | ESCAPE_CHAR_COMMON | BACKSLASH QUOTE_SINGLE | (BACKSLASH LBRACE) | (QUOTE_SINGLE QUOTE_SINGLE))+;
STRING_ESCAPE_TRIGGER: LBRACE -> pushMode(DEFAULT_MODE);
STRING_QUOTE_SINGLE: QUOTE_SINGLE -> type(QUOTE_SINGLE), popMode;
// ───────── F' FULL TEMPLATE STRING MODE ─────────
// a magic F' takes us to "full template strings" mode, where we don't need to escape single quotes and parse until EOF
// this can't be used within a normal columnExpr, but has to be parsed for separately
mode IN_FULL_TEMPLATE_STRING;
FULL_STRING_TEXT: ((~([{])) | ESCAPE_CHAR_COMMON | (BACKSLASH LBRACE))+;
FULL_STRING_ESCAPE_TRIGGER: LBRACE -> pushMode(DEFAULT_MODE);
// ───────── TSQLX TAG MODE for opening/self-closing tags ─────────
mode TSQLX_TAG_OPEN;
TAG_SELF_CLOSE_GT : '/>' -> type(SLASH_GT), popMode; // <tag …/>
TAG_OPEN_GT : '>' -> type(GT), popMode, pushMode(TSQLX_TEXT); // <tag …>
// minimal token set; map everything back to the default token types
TAG_IDENT : [a-zA-Z_][a-zA-Z0-9_-]* -> type(IDENTIFIER);
TAG_EQ : '=' -> type(EQ_SINGLE);
TAG_STRING : STRING_LITERAL -> type(STRING_LITERAL);
TAG_WS : [ \t\r\n]+ -> channel(HIDDEN);
TAG_LBRACE : '{' -> type(LBRACE), pushMode(DEFAULT_MODE);
// ───────── TSQLX TAG MODE for closing tags ─────────
mode TSQLX_TAG_CLOSE;
TAGC_GT : '>' -> type(GT), popMode; // *** no TEXT push ***
TAGC_IDENT : [a-zA-Z_][a-zA-Z0-9_-]* -> type(IDENTIFIER);
TAGC_WS : [ \t\r\n]+ -> channel(HIDDEN);
// ───────── TSQLX TEXT MODE ─────────
mode TSQLX_TEXT;
TSQLX_TEXT_TEXT
: ~[<{]+ ; // everything except “{” or “<”
TSQLX_TEXT_LBRACE
: '{' -> type(LBRACE), pushMode(DEFAULT_MODE);
TSQLX_TEXT_LT_SLASH
: '</' -> type(LT_SLASH), popMode, pushMode(TSQLX_TAG_CLOSE);
TSQLX_TEXT_LT
: '<' -> type(LT), pushMode(TSQLX_TAG_OPEN);
TSQLX_TEXT_WS
: [ \t\r\n]+ -> channel(HIDDEN);
internal-packages/tsql/src/grammar/TSQLLexer.interp
+570
View File
File diff suppressed because one or more lines are too long
internal-packages/tsql/src/grammar/TSQLLexer.tokens
+208
View File
@@ -0,0 +1,208 @@
ALL=1
AND=2
ANTI=3
ANY=4
ARRAY=5
AS=6
ASCENDING=7
ASOF=8
BETWEEN=9
BOTH=10
BY=11
CASE=12
CAST=13
CATCH=14
COHORT=15
COLLATE=16
CROSS=17
CUBE=18
CURRENT=19
DATE=20
DAY=21
DESC=22
DESCENDING=23
DISTINCT=24
ELSE=25
END=26
EXCEPT=27
EXTRACT=28
FINAL=29
FINALLY=30
FIRST=31
FN=32
FOLLOWING=33
FOR=34
FROM=35
FULL=36
FUN=37
GROUP=38
HAVING=39
HOUR=40
ID=41
IF=42
ILIKE=43
IN=44
INF=45
INNER=46
INTERSECT=47
INTERVAL=48
IS=49
JOIN=50
KEY=51
LAST=52
LEADING=53
LEFT=54
LET=55
LIKE=56
LIMIT=57
MINUTE=58
MONTH=59
NAN_SQL=60
NOT=61
NULL_SQL=62
NULLS=63
OFFSET=64
ON=65
OR=66
ORDER=67
OUTER=68
OVER=69
PARTITION=70
PRECEDING=71
PREWHERE=72
QUARTER=73
RANGE=74
RETURN=75
RIGHT=76
ROLLUP=77
ROW=78
ROWS=79
SAMPLE=80
SECOND=81
SELECT=82
SEMI=83
SETTINGS=84
SUBSTRING=85
THEN=86
THROW=87
TIES=88
TIMESTAMP=89
TO=90
TOP=91
TOTALS=92
TRAILING=93
TRIM=94
TRUNCATE=95
TRY=96
UNBOUNDED=97
UNION=98
USING=99
WEEK=100
WHEN=101
WHERE=102
WHILE=103
WINDOW=104
WITH=105
YEAR=106
ESCAPE_CHAR_COMMON=107
IDENTIFIER=108
FLOATING_LITERAL=109
OCTAL_LITERAL=110
DECIMAL_LITERAL=111
HEXADECIMAL_LITERAL=112
STRING_LITERAL=113
ARROW=114
ASTERISK=115
BACKQUOTE=116
BACKSLASH=117
COLON=118
COMMA=119
CONCAT=120
DASH=121
DOLLAR=122
DOT=123
EQ_DOUBLE=124
EQ_SINGLE=125
GT_EQ=126
GT=127
HASH=128
IREGEX_SINGLE=129
IREGEX_DOUBLE=130
LBRACE=131
LBRACKET=132
LPAREN=133
LT_EQ=134
LT=135
LT_SLASH=136
NOT_EQ=137
NOT_IREGEX=138
NOT_REGEX=139
NULL_PROPERTY=140
NULLISH=141
PERCENT=142
PLUS=143
QUERY=144
QUOTE_DOUBLE=145
QUOTE_SINGLE_TEMPLATE=146
QUOTE_SINGLE_TEMPLATE_FULL=147
QUOTE_SINGLE=148
REGEX_SINGLE=149
REGEX_DOUBLE=150
RBRACE=151
RBRACKET=152
RPAREN=153
SEMICOLON=154
SLASH=155
SLASH_GT=156
UNDERSCORE=157
MULTI_LINE_COMMENT=158
SINGLE_LINE_COMMENT=159
WHITESPACE=160
STRING_TEXT=161
STRING_ESCAPE_TRIGGER=162
FULL_STRING_TEXT=163
FULL_STRING_ESCAPE_TRIGGER=164
TAG_WS=165
TAGC_WS=166
TSQLX_TEXT_TEXT=167
TSQLX_TEXT_WS=168
'->'=114
'*'=115
'`'=116
'\\'=117
':'=118
','=119
'||'=120
'-'=121
'$'=122
'.'=123
'=='=124
'>='=126
'#'=128
'~*'=129
'=~*'=130
'{'=131
'['=132
'('=133
'<='=134
'<'=135
'</'=136
'!~*'=138
'!~'=139
'?.'=140
'??'=141
'%'=142
'+'=143
'?'=144
'"'=145
'f\''=146
'F\''=147
'\''=148
'~'=149
'=~'=150
'}'=151
']'=152
')'=153
';'=154
'/'=155
'_'=157
internal-packages/tsql/src/grammar/TSQLLexer.ts
+1730
View File
File diff suppressed because it is too large Load Diff
internal-packages/tsql/src/grammar/TSQLLexer.typescript.g4
+81
View File
@@ -0,0 +1,81 @@
lexer grammar TSQLLexer;
@header {
// put any global imports you need here
}
@members {
private _peekChar(k: number): string {
// Return the k-th look-ahead as a *single-char string* or '\0' at EOF.
const c = this._input.LA(k); // int code point or IntStream.EOF (-1)
if (c < 0 || c > 0x10FFFF) { // EOF or out-of-range → sentinel
return '\0';
}
return String.fromCharCode(c);
}
private _skipWsAndComments(idx: number): number {
// Return the first index ≥ idx that is *not* whitespace / single-line comment.
while (true) {
const ch = this._peekChar(idx);
if (/\s/.test(ch)) { // spaces, newlines, tabs …
idx++;
continue;
}
// single-line comments
if (ch === '/' && this._peekChar(idx + 1) === '/') { // //
idx += 2;
} else if (ch === '-' && this._peekChar(idx + 1) === '-') { // --
idx += 2;
} else if (ch === '#') { // #
idx++;
} else {
break; // no ws / comment
}
// consume until EOL / EOF
while (!['\0', '\n', '\r'].includes(this._peekChar(idx))) {
idx++;
}
}
return idx;
}
// ───── opening tag test ─────
isOpeningTag(): boolean {
const ch1 = this._peekChar(1);
if (!(/[a-zA-Z]/.test(ch1) || ch1 === '_')) {
return false; // not a tag name start
}
// skip tag name
let i = 2;
while (true) {
const ch = this._peekChar(i);
if (/[a-zA-Z0-9]/.test(ch) || ch === '_' || ch === '-') {
i++;
} else {
break;
}
}
let ch = this._peekChar(i);
// immediate delimiter → tag
if (ch === '>' || ch === '/') {
return true;
}
// need to look beyond whitespace
if (/\s/.test(ch)) {
i = this._skipWsAndComments(i + 1);
ch = this._peekChar(i);
return ch === '>' || ch === '/' || /[a-zA-Z0-9]/.test(ch) || ch === '_';
}
// anything else → not a tag
return false;
}
}
internal-packages/tsql/src/grammar/TSQLParser.g4
+315
View File
@@ -0,0 +1,315 @@
parser grammar TSQLParser;
options {
tokenVocab = TSQLLexer;
}
program: declaration* EOF;
declaration: varDecl | statement ;
expression: columnExpr;
varDecl: LET identifier ( COLON EQ_SINGLE expression )? ;
identifierList: identifier (COMMA identifier)* COMMA?;
statement : returnStmt
| throwStmt
| tryCatchStmt
| ifStmt
| whileStmt
| forInStmt
| forStmt
| funcStmt
| varAssignment
| block
| exprStmt
| emptyStmt
;
returnStmt : RETURN expression? SEMICOLON?;
throwStmt : THROW expression? SEMICOLON?;
catchBlock : CATCH (LPAREN catchVar=identifier (COLON catchType=identifier)? RPAREN)? catchStmt=block;
tryCatchStmt : TRY tryStmt=block catchBlock* (FINALLY finallyStmt=block)?;
ifStmt : IF LPAREN expression RPAREN statement ( ELSE statement )? ;
whileStmt : WHILE LPAREN expression RPAREN statement SEMICOLON?;
forStmt : FOR LPAREN
(initializerVarDeclr=varDecl | initializerVarAssignment=varAssignment | initializerExpression=expression)? SEMICOLON
condition=expression? SEMICOLON
(incrementVarDeclr=varDecl | incrementVarAssignment=varAssignment | incrementExpression=expression)?
RPAREN statement SEMICOLON?;
forInStmt : FOR LPAREN LET identifier (COMMA identifier)? IN expression RPAREN statement SEMICOLON?;
funcStmt : (FN | FUN) identifier LPAREN identifierList? RPAREN block;
varAssignment : expression COLON EQ_SINGLE expression ;
exprStmt : expression SEMICOLON?;
emptyStmt : SEMICOLON ;
block : LBRACE declaration* RBRACE ;
kvPair: expression ':' expression ;
kvPairList: kvPair (COMMA kvPair)* COMMA?;
// SELECT statement
select: (selectSetStmt | selectStmt | tSQLxTagElement) SEMICOLON? EOF;
selectStmtWithParens: selectStmt | LPAREN selectSetStmt RPAREN | placeholder;
subsequentSelectSetClause: (EXCEPT | UNION ALL | UNION DISTINCT | INTERSECT | INTERSECT DISTINCT) selectStmtWithParens;
selectSetStmt: selectStmtWithParens (subsequentSelectSetClause)*;
selectStmt:
with=withClause?
SELECT DISTINCT? topClause?
columns=columnExprList
from=fromClause?
arrayJoinClause?
prewhereClause?
where=whereClause?
groupByClause? (WITH (CUBE | ROLLUP))? (WITH TOTALS)?
havingClause?
windowClause?
orderByClause?
limitByClause?
(limitAndOffsetClause | offsetOnlyClause)?
settingsClause?
;
withClause: WITH withExprList;
topClause: TOP DECIMAL_LITERAL (WITH TIES)?;
fromClause: FROM joinExpr;
arrayJoinClause: (LEFT | INNER)? ARRAY JOIN columnExprList;
windowClause: WINDOW identifier AS LPAREN windowExpr RPAREN (COMMA identifier AS LPAREN windowExpr RPAREN)*;
prewhereClause: PREWHERE columnExpr;
whereClause: WHERE columnExpr;
groupByClause: GROUP BY ((CUBE | ROLLUP) LPAREN columnExprList RPAREN | columnExprList);
havingClause: HAVING columnExpr;
orderByClause: ORDER BY orderExprList;
projectionOrderByClause: ORDER BY columnExprList;
limitByClause: LIMIT limitExpr BY columnExprList;
limitAndOffsetClause
: LIMIT columnExpr (COMMA columnExpr)? (WITH TIES)? // compact OFFSET-optional form
| LIMIT columnExpr (WITH TIES)? OFFSET columnExpr // verbose OFFSET-included form with WITH TIES
;
offsetOnlyClause: OFFSET columnExpr;
settingsClause: SETTINGS settingExprList;
joinExpr
: joinExpr joinOp? JOIN joinExpr joinConstraintClause # JoinExprOp
| joinExpr joinOpCross joinExpr # JoinExprCrossOp
| tableExpr FINAL? sampleClause? # JoinExprTable
| LPAREN joinExpr RPAREN # JoinExprParens
;
joinOp
: ((ALL | ANY | ASOF)? INNER | INNER (ALL | ANY | ASOF)? | (ALL | ANY | ASOF)) # JoinOpInner
| ( (SEMI | ALL | ANTI | ANY | ASOF)? (LEFT | RIGHT) OUTER?
| (LEFT | RIGHT) OUTER? (SEMI | ALL | ANTI | ANY | ASOF)?
) # JoinOpLeftRight
| ((ALL | ANY)? FULL OUTER? | FULL OUTER? (ALL | ANY)?) # JoinOpFull
;
joinOpCross
: CROSS JOIN
| COMMA
;
joinConstraintClause
: ON columnExprList
| USING LPAREN columnExprList RPAREN
| USING columnExprList
;
sampleClause: SAMPLE ratioExpr (OFFSET ratioExpr)?;
limitExpr: columnExpr ((COMMA | OFFSET) columnExpr)?;
orderExprList: orderExpr (COMMA orderExpr)*;
orderExpr: columnExpr (ASCENDING | DESCENDING | DESC)? (NULLS (FIRST | LAST))? (COLLATE STRING_LITERAL)?;
ratioExpr: placeholder | numberLiteral (SLASH numberLiteral)?;
settingExprList: settingExpr (COMMA settingExpr)*;
settingExpr: identifier EQ_SINGLE literal;
windowExpr: winPartitionByClause? winOrderByClause? winFrameClause?;
winPartitionByClause: PARTITION BY columnExprList;
winOrderByClause: ORDER BY orderExprList;
winFrameClause: (ROWS | RANGE) winFrameExtend;
winFrameExtend
: winFrameBound # frameStart
| BETWEEN winFrameBound AND winFrameBound # frameBetween
;
winFrameBound: (CURRENT ROW | UNBOUNDED PRECEDING | UNBOUNDED FOLLOWING | numberLiteral PRECEDING | numberLiteral FOLLOWING);
//rangeClause: RANGE LPAREN (MIN identifier MAX identifier | MAX identifier MIN identifier) RPAREN;
// Columns
expr: columnExpr EOF;
columnTypeExpr
: identifier # ColumnTypeExprSimple // UInt64
| identifier LPAREN identifier columnTypeExpr (COMMA identifier columnTypeExpr)* COMMA? RPAREN # ColumnTypeExprNested // Nested
| identifier LPAREN enumValue (COMMA enumValue)* COMMA? RPAREN # ColumnTypeExprEnum // Enum
| identifier LPAREN columnTypeExpr (COMMA columnTypeExpr)* COMMA? RPAREN # ColumnTypeExprComplex // Array, Tuple
| identifier LPAREN columnExprList? RPAREN # ColumnTypeExprParam // FixedString(N)
;
columnExprList: columnExpr (COMMA columnExpr)* COMMA?;
columnExpr
: CASE caseExpr=columnExpr? (WHEN whenExpr=columnExpr THEN thenExpr=columnExpr)+ (ELSE elseExpr=columnExpr)? END # ColumnExprCase
| CAST LPAREN columnExpr AS columnTypeExpr RPAREN # ColumnExprCast
| DATE STRING_LITERAL # ColumnExprDate
// | EXTRACT LPAREN interval FROM columnExpr RPAREN # ColumnExprExtract // Interferes with a function call
| INTERVAL STRING_LITERAL # ColumnExprIntervalString
| INTERVAL columnExpr interval # ColumnExprInterval
| SUBSTRING LPAREN columnExpr FROM columnExpr (FOR columnExpr)? RPAREN # ColumnExprSubstring
| TIMESTAMP STRING_LITERAL # ColumnExprTimestamp
| TRIM LPAREN (BOTH | LEADING | TRAILING) string FROM columnExpr RPAREN # ColumnExprTrim
| identifier (LPAREN columnExprs=columnExprList? RPAREN) (LPAREN DISTINCT? columnArgList=columnExprList? RPAREN)? OVER LPAREN windowExpr RPAREN # ColumnExprWinFunction
| identifier (LPAREN columnExprs=columnExprList? RPAREN) (LPAREN DISTINCT? columnArgList=columnExprList? RPAREN)? OVER identifier # ColumnExprWinFunctionTarget
| identifier (LPAREN columnExprs=columnExprList? RPAREN)? LPAREN DISTINCT? columnArgList=columnExprList? RPAREN # ColumnExprFunction
| columnExpr LPAREN selectSetStmt RPAREN # ColumnExprCallSelect
| columnExpr LPAREN columnExprList? RPAREN # ColumnExprCall
| tSQLxTagElement # ColumnExprTagElement
| templateString # ColumnExprTemplateString
| literal # ColumnExprLiteral
// FIXME(ilezhankin): this part looks very ugly, maybe there is another way to express it
| columnExpr LBRACKET columnExpr RBRACKET # ColumnExprArrayAccess
| columnExpr DOT DECIMAL_LITERAL # ColumnExprTupleAccess
| columnExpr DOT identifier # ColumnExprPropertyAccess
| columnExpr NULL_PROPERTY LBRACKET columnExpr RBRACKET # ColumnExprNullArrayAccess
| columnExpr NULL_PROPERTY DECIMAL_LITERAL # ColumnExprNullTupleAccess
| columnExpr NULL_PROPERTY identifier # ColumnExprNullPropertyAccess
| DASH columnExpr # ColumnExprNegate
| left=columnExpr ( operator=ASTERISK // *
| operator=SLASH // /
| operator=PERCENT // %
) right=columnExpr # ColumnExprPrecedence1
| left=columnExpr ( operator=PLUS // +
| operator=DASH // -
| operator=CONCAT // ||
) right=columnExpr # ColumnExprPrecedence2
| left=columnExpr ( operator=EQ_DOUBLE // =
| operator=EQ_SINGLE // ==
| operator=NOT_EQ // !=
| operator=LT_EQ // <=
| operator=LT // <
| operator=GT_EQ // >=
| operator=GT // >
| operator=NOT? IN COHORT? // in, not in; in cohort; not in cohort
| operator=NOT? (LIKE | ILIKE) // like, not like, ilike, not ilike
| operator=REGEX_SINGLE // ~
| operator=REGEX_DOUBLE // =~
| operator=NOT_REGEX // !~
| operator=IREGEX_SINGLE // ~*
| operator=IREGEX_DOUBLE // =~*
| operator=NOT_IREGEX // !~*
) right=columnExpr # ColumnExprPrecedence3
| columnExpr IS NOT? NULL_SQL # ColumnExprIsNull
| columnExpr NULLISH columnExpr # ColumnExprNullish
| NOT columnExpr # ColumnExprNot
| columnExpr AND columnExpr # ColumnExprAnd
| columnExpr OR columnExpr # ColumnExprOr
// TODO(ilezhankin): `BETWEEN a AND b AND c` is parsed in a wrong way: `BETWEEN (a AND b) AND c`
| columnExpr NOT? BETWEEN columnExpr AND columnExpr # ColumnExprBetween
| <assoc=right> columnExpr QUERY columnExpr COLON columnExpr # ColumnExprTernaryOp
| columnExpr (AS identifier | AS STRING_LITERAL) # ColumnExprAlias
| (tableIdentifier DOT)? ASTERISK # ColumnExprAsterisk // single-column only
| LPAREN selectSetStmt RPAREN # ColumnExprSubquery // single-column only
| LPAREN columnExpr RPAREN # ColumnExprParens // single-column only
| LPAREN columnExprList RPAREN # ColumnExprTuple
| LBRACKET columnExprList? RBRACKET # ColumnExprArray
| LBRACE (kvPairList)? RBRACE # ColumnExprDict
| columnLambdaExpr # ColumnExprLambda
| columnIdentifier # ColumnExprIdentifier
;
columnLambdaExpr:
( LPAREN identifier (COMMA identifier)* COMMA? RPAREN
| identifier (COMMA identifier)* COMMA?
| LPAREN RPAREN
)
ARROW (columnExpr | block)
;
tSQLxChildElement
: tSQLxTagElement
| TSQLX_TEXT_TEXT
| LBRACE columnExpr RBRACE;
tSQLxTagElement
: LT identifier tSQLxTagAttribute* SLASH_GT
| LT identifier tSQLxTagAttribute* GT tSQLxChildElement* LT_SLASH identifier GT
;
tSQLxTagAttribute
: identifier EQ_SINGLE string
| identifier EQ_SINGLE LBRACE columnExpr RBRACE
| identifier
;
withExprList: withExpr (COMMA withExpr)* COMMA?;
withExpr
: identifier AS LPAREN selectSetStmt RPAREN # WithExprSubquery
// NOTE: asterisk and subquery goes before |columnExpr| so that we can mark them as multi-column expressions.
| columnExpr AS identifier # WithExprColumn
;
// This is slightly different in TSQL compared to ClickHouse SQL
// TSQL allows unlimited ("*") nestedIdentifier-s "properties.b.a.a.w.a.s".
// We parse and convert "databaseIdentifier.tableIdentifier.columnIdentifier.nestedIdentifier.*"
// to just one ast.Field(chain=['a','b','columnIdentifier','on','and','on']).
columnIdentifier: placeholder | ((tableIdentifier DOT)? nestedIdentifier);
nestedIdentifier: identifier (DOT identifier)*;
tableExpr
: tableIdentifier # TableExprIdentifier
| tableFunctionExpr # TableExprFunction
| LPAREN selectSetStmt RPAREN # TableExprSubquery
| tableExpr (alias | AS identifier) # TableExprAlias
| tSQLxTagElement # TableExprTag
| placeholder # TableExprPlaceholder
;
tableFunctionExpr: identifier LPAREN tableArgList? RPAREN;
tableIdentifier: (databaseIdentifier DOT)? nestedIdentifier;
tableArgList: columnExpr (COMMA columnExpr)* COMMA?;
// Databases
databaseIdentifier: identifier;
// Basics
floatingLiteral
: FLOATING_LITERAL
| DOT (DECIMAL_LITERAL | OCTAL_LITERAL)
| DECIMAL_LITERAL DOT (DECIMAL_LITERAL | OCTAL_LITERAL)? // can't move this to the lexer or it will break nested tuple access: t.1.2
;
numberLiteral: (PLUS | DASH)? (floatingLiteral | OCTAL_LITERAL | DECIMAL_LITERAL | HEXADECIMAL_LITERAL | INF | NAN_SQL);
literal
: numberLiteral
| STRING_LITERAL
| NULL_SQL
;
interval: SECOND | MINUTE | HOUR | DAY | WEEK | MONTH | QUARTER | YEAR;
keyword
// except NULL_SQL, INF, NAN_SQL
: ALL | AND | ANTI | ANY | ARRAY | AS | ASCENDING | ASOF | BETWEEN | BOTH | BY | CASE
| CAST | COHORT | COLLATE | CROSS | CUBE | CURRENT | DATE | DESC | DESCENDING
| DISTINCT | ELSE | END | EXTRACT | FINAL | FIRST
| FOR | FOLLOWING | FROM | FULL | GROUP | HAVING | ID | IS
| IF | ILIKE | IN | INNER | INTERVAL | JOIN | KEY
| LAST | LEADING | LEFT | LIKE | LIMIT
| NOT | NULLS | OFFSET | ON | OR | ORDER | OUTER | OVER | PARTITION
| PRECEDING | PREWHERE | RANGE | RETURN | RIGHT | ROLLUP | ROW
| ROWS | SAMPLE | SELECT | SEMI | SETTINGS | SUBSTRING
| THEN | TIES | TIMESTAMP | TOTALS | TRAILING | TRIM | TRUNCATE | TO | TOP
| UNBOUNDED | UNION | USING | WHEN | WHERE | WINDOW | WITH
;
keywordForAlias
: DATE | FIRST | ID | KEY
;
alias: IDENTIFIER | keywordForAlias; // |interval| can't be an alias, otherwise 'INTERVAL 1 SOMETHING' becomes ambiguous.
identifier: IDENTIFIER | interval | keyword;
enumValue: string EQ_SINGLE numberLiteral;
placeholder: LBRACE columnExpr RBRACE;
string: STRING_LITERAL | templateString;
templateString : QUOTE_SINGLE_TEMPLATE stringContents* QUOTE_SINGLE ;
stringContents : STRING_ESCAPE_TRIGGER columnExpr RBRACE | STRING_TEXT;
// These are magic "full template strings", which are used to parse "full text field" templates without the surrounding SQL.
// We will need to add F' to the start of the string to change the lexer's mode.
fullTemplateString: QUOTE_SINGLE_TEMPLATE_FULL stringContentsFull* EOF ;
stringContentsFull : FULL_STRING_ESCAPE_TRIGGER columnExpr RBRACE | FULL_STRING_TEXT;
internal-packages/tsql/src/grammar/TSQLParser.interp
+437
View File
File diff suppressed because one or more lines are too long
internal-packages/tsql/src/grammar/TSQLParser.tokens
+208
View File
@@ -0,0 +1,208 @@
ALL=1
AND=2
ANTI=3
ANY=4
ARRAY=5
AS=6
ASCENDING=7
ASOF=8
BETWEEN=9
BOTH=10
BY=11
CASE=12
CAST=13
CATCH=14
COHORT=15
COLLATE=16
CROSS=17
CUBE=18
CURRENT=19
DATE=20
DAY=21
DESC=22
DESCENDING=23
DISTINCT=24
ELSE=25
END=26
EXCEPT=27
EXTRACT=28
FINAL=29
FINALLY=30
FIRST=31
FN=32
FOLLOWING=33
FOR=34
FROM=35
FULL=36
FUN=37
GROUP=38
HAVING=39
HOUR=40
ID=41
IF=42
ILIKE=43
IN=44
INF=45
INNER=46
INTERSECT=47
INTERVAL=48
IS=49
JOIN=50
KEY=51
LAST=52
LEADING=53
LEFT=54
LET=55
LIKE=56
LIMIT=57
MINUTE=58
MONTH=59
NAN_SQL=60
NOT=61
NULL_SQL=62
NULLS=63
OFFSET=64
ON=65
OR=66
ORDER=67
OUTER=68
OVER=69
PARTITION=70
PRECEDING=71
PREWHERE=72
QUARTER=73
RANGE=74
RETURN=75
RIGHT=76
ROLLUP=77
ROW=78
ROWS=79
SAMPLE=80
SECOND=81
SELECT=82
SEMI=83
SETTINGS=84
SUBSTRING=85
THEN=86
THROW=87
TIES=88
TIMESTAMP=89
TO=90
TOP=91
TOTALS=92
TRAILING=93
TRIM=94
TRUNCATE=95
TRY=96
UNBOUNDED=97
UNION=98
USING=99
WEEK=100
WHEN=101
WHERE=102
WHILE=103
WINDOW=104
WITH=105
YEAR=106
ESCAPE_CHAR_COMMON=107
IDENTIFIER=108
FLOATING_LITERAL=109
OCTAL_LITERAL=110
DECIMAL_LITERAL=111
HEXADECIMAL_LITERAL=112
STRING_LITERAL=113
ARROW=114
ASTERISK=115
BACKQUOTE=116
BACKSLASH=117
COLON=118
COMMA=119
CONCAT=120
DASH=121
DOLLAR=122
DOT=123
EQ_DOUBLE=124
EQ_SINGLE=125
GT_EQ=126
GT=127
HASH=128
IREGEX_SINGLE=129
IREGEX_DOUBLE=130
LBRACE=131
LBRACKET=132
LPAREN=133
LT_EQ=134
LT=135
LT_SLASH=136
NOT_EQ=137
NOT_IREGEX=138
NOT_REGEX=139
NULL_PROPERTY=140
NULLISH=141
PERCENT=142
PLUS=143
QUERY=144
QUOTE_DOUBLE=145
QUOTE_SINGLE_TEMPLATE=146
QUOTE_SINGLE_TEMPLATE_FULL=147
QUOTE_SINGLE=148
REGEX_SINGLE=149
REGEX_DOUBLE=150
RBRACE=151
RBRACKET=152
RPAREN=153
SEMICOLON=154
SLASH=155
SLASH_GT=156
UNDERSCORE=157
MULTI_LINE_COMMENT=158
SINGLE_LINE_COMMENT=159
WHITESPACE=160
STRING_TEXT=161
STRING_ESCAPE_TRIGGER=162
FULL_STRING_TEXT=163
FULL_STRING_ESCAPE_TRIGGER=164
TAG_WS=165
TAGC_WS=166
TSQLX_TEXT_TEXT=167
TSQLX_TEXT_WS=168
'->'=114
'*'=115
'`'=116
'\\'=117
':'=118
','=119
'||'=120
'-'=121
'$'=122
'.'=123
'=='=124
'>='=126
'#'=128
'~*'=129
'=~*'=130
'{'=131
'['=132
'('=133
'<='=134
'<'=135
'</'=136
'!~*'=138
'!~'=139
'?.'=140
'??'=141
'%'=142
'+'=143
'?'=144
'"'=145
'f\''=146
'F\''=147
'\''=148
'~'=149
'=~'=150
'}'=151
']'=152
')'=153
';'=154
'/'=155
'_'=157
internal-packages/tsql/src/grammar/TSQLParser.ts
+11732
View File
File diff suppressed because it is too large Load Diff
internal-packages/tsql/src/grammar/TSQLParserVisitor.ts
+1312
View File
File diff suppressed because it is too large Load Diff
internal-packages/tsql/src/grammar/parser.test.ts
+166
View File
@@ -0,0 +1,166 @@
import { describe, it, expect } from "vitest";
import { CharStreams, CommonTokenStream } from "antlr4ts";
import { TSQLLexer } from "./TSQLLexer.js";
import { TSQLParser } from "./TSQLParser.js";
describe("TSQLParser", () => {
function parse(input: string) {
const inputStream = CharStreams.fromString(input);
const lexer = new TSQLLexer(inputStream);
const tokenStream = new CommonTokenStream(lexer);
const parser = new TSQLParser(tokenStream);
return parser;
}
describe("select statements", () => {
it("should parse a simple SELECT statement", () => {
const parser = parse("SELECT * FROM users;");
const tree = parser.select();
expect(tree).toBeDefined();
// The select rule can return selectStmt, selectSetStmt, or tSQLxTagElement
// Most SELECT statements are wrapped in selectSetStmt
const selectSetStmt = tree.selectSetStmt();
expect(selectSetStmt).toBeDefined();
// Get the underlying selectStmt from selectSetStmt -> selectStmtWithParens
const selectStmtWithParens = selectSetStmt!.selectStmtWithParens();
const selectStmt = selectStmtWithParens.selectStmt();
expect(selectStmt).toBeDefined();
expect(selectStmt!.SELECT()).toBeDefined();
expect(selectStmt!.columnExprList()).toBeDefined();
expect(selectStmt!.fromClause()).toBeDefined();
});
it("should parse SELECT with WHERE clause", () => {
const parser = parse("SELECT id, name FROM users WHERE id = 1;");
const tree = parser.select();
expect(tree).toBeDefined();
const selectSetStmt = tree.selectSetStmt();
const selectStmt = selectSetStmt!.selectStmtWithParens().selectStmt()!;
expect(selectStmt.whereClause()).toBeDefined();
expect(selectStmt.whereClause()!.WHERE()).toBeDefined();
expect(selectStmt.whereClause()!.columnExpr()).toBeDefined();
});
it("should parse SELECT with multiple columns", () => {
const parser = parse("SELECT id, name, email FROM users;");
const tree = parser.select();
expect(tree).toBeDefined();
const selectSetStmt = tree.selectSetStmt();
const selectStmt = selectSetStmt!.selectStmtWithParens().selectStmt()!;
const columnList = selectStmt.columnExprList();
expect(columnList).toBeDefined();
// columnExprList contains comma-separated expressions
expect(columnList.columnExpr().length).toBeGreaterThanOrEqual(1);
});
it("should parse SELECT with DISTINCT", () => {
const parser = parse("SELECT DISTINCT id FROM users;");
const tree = parser.select();
expect(tree).toBeDefined();
const selectSetStmt = tree.selectSetStmt();
const selectStmt = selectSetStmt!.selectStmtWithParens().selectStmt()!;
expect(selectStmt.DISTINCT()).toBeDefined();
});
});
describe("expressions", () => {
it("should parse a simple addition expression", () => {
const parser = parse("1 + 2");
const tree = parser.expr();
expect(tree).toBeDefined();
const columnExpr = tree.columnExpr();
expect(columnExpr).toBeDefined();
// Check that the expression has children (the operands and operator)
expect(columnExpr.text).toBeDefined();
const text = columnExpr.text;
expect(text).toContain("1");
expect(text).toContain("2");
expect(text).toContain("+");
});
it("should parse arithmetic expressions with parentheses", () => {
const parser = parse("(1 + 2) * 3");
const tree = parser.expr();
expect(tree).toBeDefined();
const columnExpr = tree.columnExpr();
expect(columnExpr).toBeDefined();
// The expression should contain the operators
const text = columnExpr.text;
expect(text).toContain("+");
expect(text).toContain("*");
expect(text).toContain("1");
expect(text).toContain("2");
expect(text).toContain("3");
});
it("should parse string literals", () => {
const parser = parse("'hello world'");
const tree = parser.expr();
expect(tree).toBeDefined();
const columnExpr = tree.columnExpr();
expect(columnExpr).toBeDefined();
const text = columnExpr.text;
expect(text).toContain("hello");
expect(text).toContain("world");
});
it("should parse numeric literals", () => {
const parser = parse("42");
const tree = parser.expr();
expect(tree).toBeDefined();
const columnExpr = tree.columnExpr();
expect(columnExpr).toBeDefined();
expect(columnExpr.text).toContain("42");
});
});
describe("program", () => {
it("should parse an empty program", () => {
const parser = parse("");
const tree = parser.program();
expect(tree).toBeDefined();
expect(tree.EOF()).toBeDefined();
expect(tree.declaration().length).toBe(0);
});
it("should parse variable declarations", () => {
const parser = parse("let x := 1");
const tree = parser.program();
expect(tree).toBeDefined();
expect(tree.declaration().length).toBe(1);
const declaration = tree.declaration(0);
// Check if it's a varDecl
const varDecl = declaration.varDecl();
expect(varDecl).toBeDefined();
expect(varDecl!.LET()).toBeDefined();
expect(varDecl!.identifier()).toBeDefined();
expect(varDecl!.expression()).toBeDefined();
});
it("should parse multiple declarations", () => {
const parser = parse("let x := 1; let y := 2");
const tree = parser.program();
expect(tree).toBeDefined();
// Count only non-empty declarations (varDecl or non-empty statements)
const varDecls = tree.declaration().filter((d) => d.varDecl() !== undefined);
expect(varDecls.length).toBe(2);
});
});
});
internal-packages/tsql/src/index.test.ts
+810
View File
@@ -0,0 +1,810 @@
import { describe, it, expect } from "vitest";
import {
compileTSQL,
parseTSQLSelect,
isColumnReferencedInExpression,
createFallbackExpression,
injectFallbackConditions,
type WhereClauseCondition,
} from "./index.js";
import { column, type TableSchema } from "./query/schema.js";
/**
* Test table schema for enforcedWhereClause tests
*/
const taskRunsSchema: TableSchema = {
name: "task_runs",
clickhouseName: "trigger_dev.task_runs_v2",
columns: {
id: { name: "id", ...column("String") },
status: { name: "status", ...column("String") },
created_at: { name: "created_at", ...column("DateTime64") },
updated_at: { name: "updated_at", ...column("DateTime64") },
time: { name: "time", ...column("DateTime64") },
triggered_at: { name: "triggered_at", ...column("DateTime64") },
organization_id: { name: "organization_id", ...column("String") },
project_id: { name: "project_id", ...column("String") },
environment_id: { name: "environment_id", ...column("String") },
},
tenantColumns: {
organizationId: "organization_id",
projectId: "project_id",
environmentId: "environment_id",
},
};
/**
* Test table schema with tenant columns (lookup table with tenant isolation)
*/
const _lookupTableSchema: TableSchema = {
name: "lookup_table",
clickhouseName: "trigger_dev.lookup_table",
tenantColumns: {
organizationId: "organization_id",
projectId: "project_id",
environmentId: "environment_id",
},
columns: {
id: { name: "id", ...column("String") },
name: { name: "name", ...column("String") },
},
};
/**
* Test table schema WITHOUT tenant columns (e.g., global reference data)
*/
const nonTenantTableSchema: TableSchema = {
name: "reference_data",
clickhouseName: "trigger_dev.reference_data",
// No tenantColumns - this is a global table
columns: {
id: { name: "id", ...column("String") },
value: { name: "value", ...column("String") },
},
};
/**
* Base options with tenant isolation for tests
*/
const baseEnforcedWhereClause: Record<string, WhereClauseCondition> = {
organization_id: { op: "eq", value: "org_test123" },
project_id: { op: "eq", value: "proj_test456" },
environment_id: { op: "eq", value: "env_test789" },
};
describe("isColumnReferencedInExpression", () => {
it("should detect column in simple WHERE clause", () => {
const ast = parseTSQLSelect("SELECT * FROM task_runs WHERE time > '2024-01-01'");
if (ast.expression_type === "select_query") {
expect(isColumnReferencedInExpression(ast.where, "time")).toBe(true);
expect(isColumnReferencedInExpression(ast.where, "status")).toBe(false);
}
});
it("should detect column in AND expression", () => {
const ast = parseTSQLSelect(
"SELECT * FROM task_runs WHERE time > '2024-01-01' AND status = 'completed'"
);
if (ast.expression_type === "select_query") {
expect(isColumnReferencedInExpression(ast.where, "time")).toBe(true);
expect(isColumnReferencedInExpression(ast.where, "status")).toBe(true);
expect(isColumnReferencedInExpression(ast.where, "id")).toBe(false);
}
});
it("should detect column in OR expression", () => {
const ast = parseTSQLSelect(
"SELECT * FROM task_runs WHERE time > '2024-01-01' OR status = 'completed'"
);
if (ast.expression_type === "select_query") {
expect(isColumnReferencedInExpression(ast.where, "time")).toBe(true);
expect(isColumnReferencedInExpression(ast.where, "status")).toBe(true);
}
});
it("should detect column in BETWEEN expression", () => {
const ast = parseTSQLSelect(
"SELECT * FROM task_runs WHERE time BETWEEN '2024-01-01' AND '2024-12-31'"
);
if (ast.expression_type === "select_query") {
expect(isColumnReferencedInExpression(ast.where, "time")).toBe(true);
expect(isColumnReferencedInExpression(ast.where, "status")).toBe(false);
}
});
it("should detect column in qualified reference (table.column)", () => {
const ast = parseTSQLSelect("SELECT * FROM task_runs WHERE task_runs.time > '2024-01-01'");
if (ast.expression_type === "select_query") {
expect(isColumnReferencedInExpression(ast.where, "time")).toBe(true);
}
});
it("should return false for empty WHERE clause", () => {
const ast = parseTSQLSelect("SELECT * FROM task_runs");
if (ast.expression_type === "select_query") {
expect(isColumnReferencedInExpression(ast.where, "time")).toBe(false);
}
});
it("should detect column in nested NOT expression", () => {
const ast = parseTSQLSelect("SELECT * FROM task_runs WHERE NOT time > '2024-01-01'");
if (ast.expression_type === "select_query") {
expect(isColumnReferencedInExpression(ast.where, "time")).toBe(true);
}
});
});
describe("createFallbackExpression", () => {
it("should create a simple comparison expression (gt)", () => {
const expr = createFallbackExpression("time", { op: "gt", value: "2024-01-01" });
expect(expr.expression_type).toBe("compare_operation");
});
it("should create a simple comparison expression (eq)", () => {
const expr = createFallbackExpression("status", { op: "eq", value: "completed" });
expect(expr.expression_type).toBe("compare_operation");
});
it("should create a between expression", () => {
const expr = createFallbackExpression("time", {
op: "between",
low: "2024-01-01",
high: "2024-12-31",
});
expect(expr.expression_type).toBe("between_expr");
});
it("should convert Date values to ISO strings", () => {
const date = new Date("2024-06-15T12:00:00Z");
const expr = createFallbackExpression("time", { op: "gte", value: date });
expect(expr.expression_type).toBe("compare_operation");
});
});
describe("injectFallbackConditions", () => {
it("should inject fallback when column is not in WHERE", () => {
const ast = parseTSQLSelect("SELECT * FROM task_runs WHERE status = 'completed'");
const fallbacks: Record<string, WhereClauseCondition> = {
time: { op: "gte", value: "2024-01-01" },
};
const modified = injectFallbackConditions(ast, fallbacks);
if (modified.expression_type === "select_query") {
expect(modified.where).toBeDefined();
expect(isColumnReferencedInExpression(modified.where, "time")).toBe(true);
expect(isColumnReferencedInExpression(modified.where, "status")).toBe(true);
}
});
it("should NOT inject fallback when column is already in WHERE", () => {
const ast = parseTSQLSelect("SELECT * FROM task_runs WHERE time > '2024-06-01'");
const fallbacks: Record<string, WhereClauseCondition> = {
time: { op: "gte", value: "2024-01-01" },
};
const modified = injectFallbackConditions(ast, fallbacks);
// The WHERE clause should remain as is (no additional time condition)
if (modified.expression_type === "select_query" && modified.where) {
// It should not be an AND expression - it should be the original compare_operation
expect(modified.where.expression_type).toBe("compare_operation");
}
});
it("should inject fallback when query has no WHERE clause", () => {
const ast = parseTSQLSelect("SELECT * FROM task_runs LIMIT 10");
const fallbacks: Record<string, WhereClauseCondition> = {
time: { op: "gte", value: "2024-01-01" },
};
const modified = injectFallbackConditions(ast, fallbacks);
if (modified.expression_type === "select_query") {
expect(modified.where).toBeDefined();
expect(isColumnReferencedInExpression(modified.where, "time")).toBe(true);
}
});
it("should inject multiple fallbacks", () => {
const ast = parseTSQLSelect("SELECT * FROM task_runs LIMIT 10");
const fallbacks: Record<string, WhereClauseCondition> = {
time: { op: "gte", value: "2024-01-01" },
status: { op: "eq", value: "completed" },
};
const modified = injectFallbackConditions(ast, fallbacks);
if (modified.expression_type === "select_query") {
expect(modified.where).toBeDefined();
expect(isColumnReferencedInExpression(modified.where, "time")).toBe(true);
expect(isColumnReferencedInExpression(modified.where, "status")).toBe(true);
}
});
it("should only inject fallbacks for unreferenced columns", () => {
const ast = parseTSQLSelect("SELECT * FROM task_runs WHERE time > '2024-06-01'");
const fallbacks: Record<string, WhereClauseCondition> = {
time: { op: "gte", value: "2024-01-01" }, // Should NOT be injected
status: { op: "eq", value: "completed" }, // Should be injected
};
const modified = injectFallbackConditions(ast, fallbacks);
if (modified.expression_type === "select_query" && modified.where) {
// Should be an AND expression combining fallback status with original time
expect(modified.where.expression_type).toBe("and");
}
});
});
describe("compileTSQL with whereClauseFallback", () => {
const baseOptions = {
tableSchema: [taskRunsSchema],
enforcedWhereClause: baseEnforcedWhereClause,
};
describe("simple comparison fallbacks", () => {
it("should apply gt fallback when column not in WHERE", () => {
const { sql } = compileTSQL("SELECT * FROM task_runs WHERE status = 'completed'", {
...baseOptions,
whereClauseFallback: {
time: { op: "gt", value: "2024-01-01" },
},
});
// Should contain a time comparison
expect(sql).toContain("time");
expect(sql).toContain("greater(");
});
it("should apply gte fallback when column not in WHERE", () => {
const { sql } = compileTSQL("SELECT id FROM task_runs", {
...baseOptions,
whereClauseFallback: {
time: { op: "gte", value: "2024-01-01" },
},
});
expect(sql).toContain("greaterOrEquals(");
});
it("should apply lt fallback when column not in WHERE", () => {
const { sql } = compileTSQL("SELECT id FROM task_runs", {
...baseOptions,
whereClauseFallback: {
time: { op: "lt", value: "2024-12-31" },
},
});
expect(sql).toContain("less(");
});
it("should apply lte fallback when column not in WHERE", () => {
const { sql } = compileTSQL("SELECT id FROM task_runs", {
...baseOptions,
whereClauseFallback: {
time: { op: "lte", value: "2024-12-31" },
},
});
expect(sql).toContain("lessOrEquals(");
});
it("should apply eq fallback when column not in WHERE", () => {
const { sql } = compileTSQL("SELECT id FROM task_runs", {
...baseOptions,
whereClauseFallback: {
status: { op: "eq", value: "completed" },
},
});
expect(sql).toContain("equals(");
expect(sql).toContain("status");
});
it("should apply neq fallback when column not in WHERE", () => {
const { sql } = compileTSQL("SELECT id FROM task_runs", {
...baseOptions,
whereClauseFallback: {
status: { op: "neq", value: "failed" },
},
});
expect(sql).toContain("notEquals(");
expect(sql).toContain("status");
});
});
describe("between fallback", () => {
it("should apply BETWEEN fallback when column not in WHERE", () => {
const { sql } = compileTSQL("SELECT id FROM task_runs WHERE status = 'completed'", {
...baseOptions,
whereClauseFallback: {
time: { op: "between", low: "2024-01-01", high: "2024-12-31" },
},
});
expect(sql).toContain("time BETWEEN");
});
it("should apply BETWEEN fallback with Date values", () => {
const startDate = new Date("2024-01-01T00:00:00Z");
const endDate = new Date("2024-12-31T23:59:59Z");
const { sql, params } = compileTSQL("SELECT id FROM task_runs", {
...baseOptions,
whereClauseFallback: {
time: { op: "between", low: startDate, high: endDate },
},
});
expect(sql).toContain("time BETWEEN");
// The dates should be converted to ISO strings and parameterized
expect(Object.values(params).some((v) => typeof v === "string" && v.includes("2024"))).toBe(
true
);
});
});
describe("fallback NOT applied when column already filtered", () => {
it("should NOT apply fallback when column is in simple WHERE", () => {
const { sql } = compileTSQL("SELECT * FROM task_runs WHERE time > '2024-06-01'", {
...baseOptions,
whereClauseFallback: {
time: { op: "gte", value: "2024-01-01" },
},
});
// Should only have the user's time condition, not the fallback
// Count occurrences of time comparison - should be 1 (just the user's)
const timeMatches = sql.match(/greater\(.*?time/g) || [];
expect(timeMatches.length).toBe(1);
});
it("should NOT apply fallback when column is in BETWEEN expression", () => {
const { sql } = compileTSQL(
"SELECT * FROM task_runs WHERE time BETWEEN '2024-06-01' AND '2024-06-30'",
{
...baseOptions,
whereClauseFallback: {
time: { op: "gte", value: "2024-01-01" },
},
}
);
// Should have the user's BETWEEN, not the fallback gte
expect(sql).toContain("time BETWEEN");
expect(sql).not.toContain("greaterOrEquals(");
});
it("should NOT apply fallback when column is in AND expression", () => {
const { sql } = compileTSQL(
"SELECT * FROM task_runs WHERE status = 'completed' AND time > '2024-06-01'",
{
...baseOptions,
whereClauseFallback: {
time: { op: "gte", value: "2024-01-01" },
},
}
);
// Should only have one time comparison
const timeMatches = sql.match(/greater\(.*?time/g) || [];
expect(timeMatches.length).toBe(1);
});
it("should NOT apply fallback when column is in OR expression", () => {
const { sql } = compileTSQL(
"SELECT * FROM task_runs WHERE status = 'completed' OR time > '2024-06-01'",
{
...baseOptions,
whereClauseFallback: {
time: { op: "gte", value: "2024-01-01" },
},
}
);
// Fallback should not be applied since time is mentioned in OR
const timeGreaterMatches = sql.match(/greater\(.*?time/g) || [];
expect(timeGreaterMatches.length).toBe(1);
});
it("should NOT apply fallback when column is in qualified reference", () => {
const { sql } = compileTSQL("SELECT * FROM task_runs WHERE task_runs.time > '2024-06-01'", {
...baseOptions,
whereClauseFallback: {
time: { op: "gte", value: "2024-01-01" },
},
});
// Fallback should not be applied
const timeGreaterMatches = sql.match(/greater.*time/g) || [];
expect(timeGreaterMatches.length).toBe(1);
});
});
describe("multiple fallbacks", () => {
it("should apply multiple fallbacks for different unreferenced columns", () => {
const { sql } = compileTSQL("SELECT id FROM task_runs", {
...baseOptions,
whereClauseFallback: {
time: { op: "gte", value: "2024-01-01" },
status: { op: "eq", value: "completed" },
},
});
expect(sql).toContain("greaterOrEquals(");
expect(sql).toContain("time");
expect(sql).toContain("equals(");
expect(sql).toContain("status");
});
it("should only apply fallbacks for unreferenced columns when some are filtered", () => {
const { sql } = compileTSQL("SELECT id FROM task_runs WHERE time > '2024-06-01'", {
...baseOptions,
whereClauseFallback: {
time: { op: "gte", value: "2024-01-01" }, // Should NOT be applied
status: { op: "eq", value: "completed" }, // Should be applied
},
});
// Status fallback should be applied
expect(sql).toContain("equals(");
expect(sql).toContain("status");
// Time should only have user's condition
const timeGreaterMatches = sql.match(/greater\(.*?time/g) || [];
expect(timeGreaterMatches.length).toBe(1);
});
});
describe("edge cases", () => {
it("should handle empty whereClauseFallback", () => {
const { sql } = compileTSQL("SELECT id FROM task_runs", {
...baseOptions,
whereClauseFallback: {},
});
// Should just compile normally without errors
expect(sql).toContain("SELECT");
expect(sql).toContain("FROM");
});
it("should handle undefined whereClauseFallback", () => {
const { sql } = compileTSQL("SELECT id FROM task_runs", {
...baseOptions,
// whereClauseFallback is undefined
});
// Should just compile normally without errors
expect(sql).toContain("SELECT");
expect(sql).toContain("FROM");
});
it("should work with UNION queries", () => {
const { sql } = compileTSQL(
"SELECT id FROM task_runs WHERE status = 'completed' UNION ALL SELECT id FROM task_runs WHERE status = 'failed'",
{
...baseOptions,
whereClauseFallback: {
time: { op: "gte", value: "2024-01-01" },
},
}
);
// Both queries in the UNION should have the time fallback
const timeMatches = sql.match(/greaterOrEquals\(.*?time/g) || [];
expect(timeMatches.length).toBe(2);
});
it("should handle numeric values in fallback", () => {
const { sql } = compileTSQL("SELECT id FROM task_runs", {
...baseOptions,
whereClauseFallback: {
id: { op: "gt", value: 100 },
},
});
expect(sql).toContain("greater(");
expect(sql).toContain("100");
});
});
});
describe("compileTSQL with enforcedWhereClause", () => {
describe("validation tests", () => {
it("should throw error when required tenant column is missing", () => {
expect(() =>
compileTSQL("SELECT id FROM task_runs", {
tableSchema: [taskRunsSchema],
enforcedWhereClause: {}, // Missing organization_id
})
).toThrow("Table 'task_runs' requires 'organization_id' in enforcedWhereClause");
});
it("should throw error when organization_id is missing but other tenant columns are present", () => {
expect(() =>
compileTSQL("SELECT id FROM task_runs", {
tableSchema: [taskRunsSchema],
enforcedWhereClause: {
project_id: { op: "eq", value: "proj_123" },
environment_id: { op: "eq", value: "env_456" },
},
})
).toThrow("Table 'task_runs' requires 'organization_id' in enforcedWhereClause");
});
it("should work with non-tenant table and empty enforcedWhereClause", () => {
const { sql } = compileTSQL("SELECT id FROM reference_data", {
tableSchema: [nonTenantTableSchema],
enforcedWhereClause: {},
});
expect(sql).toContain("SELECT");
expect(sql).toContain("FROM");
});
it("should work with only organization_id (project and env are optional)", () => {
const { sql } = compileTSQL("SELECT id FROM task_runs", {
tableSchema: [taskRunsSchema],
enforcedWhereClause: {
organization_id: { op: "eq", value: "org_123" },
},
});
expect(sql).toContain("organization_id");
expect(sql).not.toContain("project_id");
expect(sql).not.toContain("environment_id");
});
});
describe("basic functionality", () => {
it("should apply single enforced condition", () => {
const { sql } = compileTSQL("SELECT id FROM task_runs", {
tableSchema: [taskRunsSchema],
enforcedWhereClause: {
organization_id: { op: "eq", value: "org_123" },
},
});
expect(sql).toContain("equals(");
expect(sql).toContain("organization_id");
});
it("should apply multiple enforced conditions", () => {
const { sql } = compileTSQL("SELECT id FROM task_runs", {
tableSchema: [taskRunsSchema],
enforcedWhereClause: {
organization_id: { op: "eq", value: "org_123" },
project_id: { op: "eq", value: "proj_456" },
environment_id: { op: "eq", value: "env_789" },
},
});
expect(sql).toContain("organization_id");
expect(sql).toContain("project_id");
expect(sql).toContain("environment_id");
});
it("should apply enforced condition even when user filters on same field", () => {
const { sql } = compileTSQL("SELECT id FROM task_runs WHERE triggered_at > '2025-01-01'", {
tableSchema: [taskRunsSchema],
enforcedWhereClause: {
organization_id: { op: "eq", value: "org_123" },
triggered_at: { op: "gte", value: "2024-01-01" },
},
});
// Should have BOTH the user's condition AND the enforced condition
// User's condition: greater(triggered_at, '2025-01-01')
// Enforced condition: greaterOrEquals(triggered_at, '2024-01-01')
const triggeredAtMatches = sql.match(/triggered_at/g) || [];
expect(triggeredAtMatches.length).toBeGreaterThanOrEqual(2);
});
it("should apply different comparison operators", () => {
const { sql: sqlGt } = compileTSQL("SELECT id FROM task_runs", {
tableSchema: [taskRunsSchema],
enforcedWhereClause: {
organization_id: { op: "eq", value: "org_123" },
time: { op: "gt", value: "2024-01-01" },
},
});
expect(sqlGt).toContain("greater(");
const { sql: sqlLt } = compileTSQL("SELECT id FROM task_runs", {
tableSchema: [taskRunsSchema],
enforcedWhereClause: {
organization_id: { op: "eq", value: "org_123" },
time: { op: "lt", value: "2024-12-31" },
},
});
expect(sqlLt).toContain("less(");
const { sql: sqlNeq } = compileTSQL("SELECT id FROM task_runs", {
tableSchema: [taskRunsSchema],
enforcedWhereClause: {
organization_id: { op: "eq", value: "org_123" },
status: { op: "neq", value: "deleted" },
},
});
expect(sqlNeq).toContain("notEquals(");
});
it("should apply BETWEEN condition", () => {
const { sql } = compileTSQL("SELECT id FROM task_runs", {
tableSchema: [taskRunsSchema],
enforcedWhereClause: {
organization_id: { op: "eq", value: "org_123" },
time: { op: "between", low: "2024-01-01", high: "2024-12-31" },
},
});
expect(sql).toContain("time BETWEEN");
});
it("should handle Date values in enforced conditions", () => {
const sevenDaysAgo = new Date(Date.now() - 7 * 24 * 60 * 60 * 1000);
const { sql, params: _params } = compileTSQL("SELECT id FROM task_runs", {
tableSchema: [taskRunsSchema],
enforcedWhereClause: {
organization_id: { op: "eq", value: "org_123" },
triggered_at: { op: "gte", value: sevenDaysAgo },
},
});
expect(sql).toContain("triggered_at");
expect(sql).toContain("toDateTime64");
});
});
describe("enforcedWhereClause + whereClauseFallback interaction", () => {
it("should apply both enforced and fallback conditions when user doesn't filter", () => {
const { sql } = compileTSQL("SELECT id FROM task_runs", {
tableSchema: [taskRunsSchema],
enforcedWhereClause: {
organization_id: { op: "eq", value: "org_123" },
triggered_at: { op: "gte", value: "2024-01-01" },
},
whereClauseFallback: {
status: { op: "eq", value: "completed" },
},
});
// Should have both enforced (triggered_at) and fallback (status)
expect(sql).toContain("triggered_at");
expect(sql).toContain("status");
});
it("should apply enforced but not fallback when user filters on fallback column", () => {
const { sql, params } = compileTSQL("SELECT id FROM task_runs WHERE status = 'failed'", {
tableSchema: [taskRunsSchema],
enforcedWhereClause: {
organization_id: { op: "eq", value: "org_123" },
triggered_at: { op: "gte", value: "2024-01-01" },
},
whereClauseFallback: {
status: { op: "eq", value: "completed" },
},
});
// Enforced triggered_at should be applied
expect(sql).toContain("triggered_at");
// User's status = 'failed' should be there (as a parameter)
expect(Object.values(params)).toContain("failed");
// The fallback 'completed' should NOT be applied since user filtered on status
expect(Object.values(params)).not.toContain("completed");
});
it("should apply both enforced and fallback on same field (enforced always, fallback only if not filtered)", () => {
// User doesn't filter on triggered_at, so BOTH enforced AND fallback apply
const { sql } = compileTSQL("SELECT id FROM task_runs WHERE status = 'completed'", {
tableSchema: [taskRunsSchema],
enforcedWhereClause: {
organization_id: { op: "eq", value: "org_123" },
triggered_at: { op: "gte", value: "2024-06-01" }, // Enforced: last 6 months
},
whereClauseFallback: {
triggered_at: { op: "gte", value: "2024-01-01" }, // Fallback: last year
},
});
// Both should be applied (enforced at printer level, fallback at AST level)
const triggeredAtMatches = sql.match(/triggered_at/g) || [];
expect(triggeredAtMatches.length).toBeGreaterThanOrEqual(2);
});
it("should skip fallback but keep enforced when user filters on same field", () => {
const { sql } = compileTSQL("SELECT id FROM task_runs WHERE triggered_at > '2025-01-01'", {
tableSchema: [taskRunsSchema],
enforcedWhereClause: {
organization_id: { op: "eq", value: "org_123" },
triggered_at: { op: "gte", value: "2024-06-01" }, // Enforced: always applied
},
whereClauseFallback: {
triggered_at: { op: "gte", value: "2024-01-01" }, // Fallback: skipped since user filtered
},
});
// User's condition + enforced should be present
// Fallback should NOT be applied since user filtered on triggered_at
// Count distinct triggered_at conditions
const triggeredAtMatches = sql.match(/triggered_at/g) || [];
// Should be 2: user's condition + enforced condition (NOT 3, no fallback)
expect(triggeredAtMatches.length).toBe(2);
});
});
describe("security tests", () => {
it("should apply enforced conditions to UNION queries", () => {
const { sql } = compileTSQL(
"SELECT id FROM task_runs WHERE status = 'completed' UNION ALL SELECT id FROM task_runs WHERE status = 'failed'",
{
tableSchema: [taskRunsSchema],
enforcedWhereClause: {
organization_id: { op: "eq", value: "org_123" },
triggered_at: { op: "gte", value: "2024-01-01" },
},
}
);
// Both parts of the UNION should have the enforced conditions
const orgMatches = sql.match(/organization_id/g) || [];
expect(orgMatches.length).toBe(2);
const triggeredAtMatches = sql.match(/triggered_at/g) || [];
expect(triggeredAtMatches.length).toBe(2);
});
it("should NOT be bypassable via OR clause", () => {
const { sql } = compileTSQL("SELECT id FROM task_runs WHERE status = 'completed' OR 1=1", {
tableSchema: [taskRunsSchema],
enforcedWhereClause: {
organization_id: { op: "eq", value: "org_123" },
triggered_at: { op: "gte", value: "2024-01-01" },
},
});
// The enforced conditions should be ANDed with the entire user WHERE clause
// So the structure should be: (enforced AND enforced AND ...) AND (user_where)
expect(sql).toContain("organization_id");
expect(sql).toContain("triggered_at");
// The 1=1 should be within the user's OR clause, not affecting enforced conditions
});
it("should skip enforced conditions for columns that don't exist in table", () => {
const { sql } = compileTSQL("SELECT id FROM task_runs", {
tableSchema: [taskRunsSchema],
enforcedWhereClause: {
organization_id: { op: "eq", value: "org_123" },
nonexistent_column: { op: "eq", value: "test" },
},
});
// Should not contain nonexistent_column
expect(sql).not.toContain("nonexistent_column");
// Should still have organization_id
expect(sql).toContain("organization_id");
});
});
describe("edge cases", () => {
it("should handle empty enforced conditions for non-tenant table", () => {
const { sql } = compileTSQL("SELECT id FROM reference_data", {
tableSchema: [nonTenantTableSchema],
enforcedWhereClause: {},
});
expect(sql).toContain("SELECT");
expect(sql).not.toContain("WHERE"); // No WHERE clause needed
});
it("should properly format numeric values", () => {
const { sql } = compileTSQL("SELECT id FROM task_runs", {
tableSchema: [taskRunsSchema],
enforcedWhereClause: {
organization_id: { op: "eq", value: "org_123" },
},
});
// org_123 should be parameterized, not inlined
expect(sql).toContain("tsql_val_");
});
});
});
internal-packages/tsql/src/index.ts
+606
View File
@@ -0,0 +1,606 @@
// TSQL - Type-Safe SQL Query Language for ClickHouse
// Originally derived from PostHog's HogQL (see NOTICE.md for attribution)
import type { ANTLRErrorListener, RecognitionException, Recognizer } from "antlr4ts";
import { CharStreams, CommonTokenStream } from "antlr4ts";
import type { Token } from "antlr4ts/Token";
import { TSQLLexer } from "./grammar/TSQLLexer.js";
import { TSQLParser } from "./grammar/TSQLParser.js";
import type {
And,
BetweenExpr,
Call,
CompareOperation,
Constant,
Expression,
Field,
Not,
Or,
SelectQuery,
SelectSetQuery,
Tuple,
} from "./query/ast.js";
import { CompareOperationOp } from "./query/ast.js";
import { SyntaxError as TSQLSyntaxError } from "./query/errors.js";
import { TSQLParseTreeConverter } from "./query/parser.js";
import { printToClickHouse, type PrintResult } from "./query/printer.js";
import {
createPrinterContext,
type QuerySettings,
type SimpleComparisonCondition,
type TimeRange,
type WhereClauseCondition,
} from "./query/printer_context.js";
import { createSchemaRegistry, type FieldMappings, type TableSchema } from "./query/schema.js";
/**
* Simple error listener that captures syntax errors
*/
class TSQLErrorListener implements ANTLRErrorListener<Token> {
public error: string | null = null;
syntaxError(
_recognizer: Recognizer<Token, any>,
_offendingSymbol: Token | undefined,
line: number,
charPositionInLine: number,
msg: string,
_e: RecognitionException | undefined
): void {
this.error = `Syntax error at line ${line}:${charPositionInLine}: ${msg}`;
}
}
// Re-export AST types
export * from "./query/ast.js";
// Re-export errors
export * from "./query/errors.js";
// Re-export escape utilities
export {
escapeClickHouseIdentifier,
escapeClickHouseString,
escapeTSQLIdentifier,
escapeTSQLString,
getClickHouseType,
} from "./query/escape.js";
// Re-export function definitions
export {
findTSQLAggregation,
findTSQLFunction,
getAllExposedFunctionNames,
TSQL_AGGREGATIONS,
TSQL_CLICKHOUSE_FUNCTIONS,
TSQL_COMPARISON_MAPPING,
type TSQLFunctionMeta,
} from "./query/functions.js";
// Re-export schema types and functions
export {
column,
createSchemaRegistry,
findColumn,
findTable,
getAllowedUserValues,
// Core column utilities
getCoreColumns,
getExternalValue,
getInternalValue,
getInternalValueFromMapping,
getInternalValueFromMappingCaseInsensitive,
// Value mapping utilities
getUserFriendlyValue,
getVirtualColumnExpression,
// Field mapping utilities (runtime dynamic mappings)
hasFieldMapping,
isValidUserValue,
// Virtual column utilities
isVirtualColumn,
// Error message sanitization
sanitizeErrorMessage,
validateFilterColumn,
validateGroupColumn,
validateSelectColumn,
validateSortColumn,
validateTable,
type ClickHouseType,
type ColumnSchema,
type FieldMappings,
type ColumnFormatType,
type OutputColumnMetadata,
type RequiredFilter,
type SchemaRegistry,
type TableSchema,
type TenantColumnConfig,
} from "./query/schema.js";
// Re-export printer context
export {
createPrinterContext,
DEFAULT_QUERY_SETTINGS,
PrinterContext,
type BetweenCondition,
type InCondition,
type PrinterContextOptions,
type QueryNotice,
type QuerySettings,
type SimpleComparisonCondition,
type TimeRange,
type WhereClauseCondition,
} from "./query/printer_context.js";
// Re-export time bucket utilities
export {
BUCKET_THRESHOLDS,
calculateTimeBucketInterval,
type BucketThreshold,
type TimeBucketInterval,
} from "./query/time_buckets.js";
// Re-export printer
export { ClickHousePrinter, printToClickHouse, type PrintResult } from "./query/printer.js";
// Re-export parser converter for advanced usage
export { TSQLParseTreeConverter } from "./query/parser.js";
// Re-export validator
export {
validateQuery,
type ValidationIssue,
type ValidationResult,
type ValidationSeverity,
} from "./query/validator.js";
// Re-export result transformation utilities
export {
createResultTransformer,
transformResults,
type TransformResultsOptions,
} from "./query/results.js";
/**
* Parse a TSQL SELECT query string into an AST
*
* @param query - The TSQL query string to parse
* @returns The parsed AST (SelectQuery or SelectSetQuery)
* @throws TSQLSyntaxError if the query is invalid
*
* @example
* ```typescript
* const ast = parseTSQLSelect("SELECT * FROM users WHERE id = 1");
* ```
*/
export function parseTSQLSelect(query: string): SelectQuery | SelectSetQuery {
const inputStream = CharStreams.fromString(query);
const lexer = new TSQLLexer(inputStream);
const tokenStream = new CommonTokenStream(lexer);
const parser = new TSQLParser(tokenStream);
// Remove default error listeners and add custom one
parser.removeErrorListeners();
const errorListener = new TSQLErrorListener();
parser.addErrorListener(errorListener);
const parseTree = parser.select();
if (errorListener.error) {
throw new TSQLSyntaxError(errorListener.error);
}
const converter = new TSQLParseTreeConverter();
const ast = converter.visit(parseTree);
// Validate the result is a select query
if (typeof ast === "string" || !("expression_type" in ast)) {
throw new TSQLSyntaxError("Failed to parse SELECT query");
}
if (ast.expression_type !== "select_query" && ast.expression_type !== "select_set_query") {
throw new TSQLSyntaxError(`Expected SELECT query, got ${ast.expression_type}`);
}
return ast as SelectQuery | SelectSetQuery;
}
/**
* Parse a TSQL expression string into an AST
*
* @param expr - The TSQL expression string to parse
* @returns The parsed expression AST
* @throws TSQLSyntaxError if the expression is invalid
*
* @example
* ```typescript
* const ast = parseTSQLExpr("id = 1 AND name = 'test'");
* ```
*/
export function parseTSQLExpr(expr: string): Expression {
const inputStream = CharStreams.fromString(expr);
const lexer = new TSQLLexer(inputStream);
const tokenStream = new CommonTokenStream(lexer);
const parser = new TSQLParser(tokenStream);
// Remove default error listeners and add custom one
parser.removeErrorListeners();
const errorListener = new TSQLErrorListener();
parser.addErrorListener(errorListener);
const parseTree = parser.columnExpr(0);
if (errorListener.error) {
throw new TSQLSyntaxError(errorListener.error);
}
const converter = new TSQLParseTreeConverter();
return converter.visit(parseTree) as Expression;
}
/**
* Check if a column is referenced in an expression (for WHERE clause detection).
* Recursively traverses And, Or, CompareOperation, BetweenExpr, and Field nodes.
*
* @param expr - The expression to search
* @param column - The column name to look for
* @returns true if the column is referenced in the expression
*/
export function isColumnReferencedInExpression(
expr: Expression | undefined,
column: string
): boolean {
if (!expr) return false;
const exprType = (expr as Expression).expression_type;
switch (exprType) {
case "and": {
const andExpr = expr as And;
return andExpr.exprs.some((e) => isColumnReferencedInExpression(e, column));
}
case "or": {
const orExpr = expr as Or;
return orExpr.exprs.some((e) => isColumnReferencedInExpression(e, column));
}
case "compare_operation": {
const compareExpr = expr as CompareOperation;
return (
isColumnReferencedInExpression(compareExpr.left, column) ||
isColumnReferencedInExpression(compareExpr.right, column)
);
}
case "between_expr": {
const betweenExpr = expr as BetweenExpr;
return isColumnReferencedInExpression(betweenExpr.expr, column);
}
case "field": {
const fieldExpr = expr as Field;
// Check if any part of the chain matches the column name
// Handles both unqualified (column) and qualified (table.column) references
return fieldExpr.chain.some((part) => part === column);
}
case "not": {
const notExpr = expr as Not;
return isColumnReferencedInExpression(notExpr.expr, column);
}
default:
return false;
}
}
/**
* Format a Date as a ClickHouse-compatible DateTime64 string.
* ClickHouse expects format: 'YYYY-MM-DD HH:MM:SS.mmm' (in UTC)
*/
function formatDateForClickHouse(date: Date): string {
const year = date.getUTCFullYear();
const month = String(date.getUTCMonth() + 1).padStart(2, "0");
const day = String(date.getUTCDate()).padStart(2, "0");
const hours = String(date.getUTCHours()).padStart(2, "0");
const minutes = String(date.getUTCMinutes()).padStart(2, "0");
const seconds = String(date.getUTCSeconds()).padStart(2, "0");
const ms = String(date.getUTCMilliseconds()).padStart(3, "0");
return `${year}-${month}-${day} ${hours}:${minutes}:${seconds}.${ms}`;
}
/**
* Create an AST expression for a fallback value.
* Date values are wrapped in toDateTime64() for ClickHouse compatibility.
*/
function createValueExpression(value: Date | string | number): Expression {
if (value instanceof Date) {
// Wrap Date in toDateTime64(formatted_string, 3) for ClickHouse DateTime64(3) columns
return {
expression_type: "call",
name: "toDateTime64",
args: [
{ expression_type: "constant", value: formatDateForClickHouse(value) } as Constant,
{ expression_type: "constant", value: 3 } as Constant,
],
} as Call;
}
return { expression_type: "constant", value } as Constant;
}
/**
* Map fallback operator to CompareOperationOp
*/
function mapFallbackOpToCompareOp(op: SimpleComparisonCondition["op"]): CompareOperationOp {
switch (op) {
case "eq":
return CompareOperationOp.Eq;
case "neq":
return CompareOperationOp.NotEq;
case "gt":
return CompareOperationOp.Gt;
case "gte":
return CompareOperationOp.GtEq;
case "lt":
return CompareOperationOp.Lt;
case "lte":
return CompareOperationOp.LtEq;
}
}
/**
* Create an AST expression from a fallback condition
*
* @param column - The column name
* @param fallback - The fallback condition
* @returns The AST expression for the fallback condition
*/
export function createFallbackExpression(
column: string,
fallback: WhereClauseCondition
): Expression {
const fieldExpr: Field = {
expression_type: "field",
chain: [column],
};
if (fallback.op === "between") {
const betweenExpr: BetweenExpr = {
expression_type: "between_expr",
expr: fieldExpr,
low: createValueExpression(fallback.low),
high: createValueExpression(fallback.high),
};
return betweenExpr;
}
if (fallback.op === "in") {
// Create a tuple of values for the IN clause
const tupleExpr: Tuple = {
expression_type: "tuple",
exprs: fallback.values.map((value) => createValueExpression(value)),
};
const inExpr: CompareOperation = {
expression_type: "compare_operation",
left: fieldExpr,
right: tupleExpr,
op: CompareOperationOp.In,
};
return inExpr;
}
// Simple comparison
const compareExpr: CompareOperation = {
expression_type: "compare_operation",
left: fieldExpr,
right: createValueExpression(fallback.value),
op: mapFallbackOpToCompareOp(fallback.op),
};
return compareExpr;
}
/**
* Inject fallback WHERE conditions into a parsed AST.
* Only adds fallback conditions for columns not already referenced in the WHERE clause.
*
* @param ast - The parsed SELECT query AST
* @param fallbacks - The fallback conditions to potentially inject
* @returns The modified AST with fallback conditions injected
*/
export function injectFallbackConditions(
ast: SelectQuery | SelectSetQuery,
fallbacks: Record<string, WhereClauseCondition>
): SelectQuery | SelectSetQuery {
// Handle SelectSetQuery (UNION, etc.) - apply to each query in the set
if (ast.expression_type === "select_set_query") {
const setQuery = ast as SelectSetQuery;
// Process the initial select query
const modifiedInitial = injectFallbackConditions(
setQuery.initial_select_query,
fallbacks
) as SelectQuery;
// Process subsequent queries
const modifiedSubsequent = setQuery.subsequent_select_queries.map((sq) => ({
...sq,
select_query: injectFallbackConditions(sq.select_query, fallbacks) as SelectQuery,
}));
return {
...setQuery,
initial_select_query: modifiedInitial,
subsequent_select_queries: modifiedSubsequent,
};
}
// Handle SelectQuery
const selectQuery = ast as SelectQuery;
const existingWhere = selectQuery.where;
// Collect fallback expressions for columns not already in WHERE
const fallbackExprs: Expression[] = [];
for (const [column, fallback] of Object.entries(fallbacks)) {
if (!isColumnReferencedInExpression(existingWhere, column)) {
fallbackExprs.push(createFallbackExpression(column, fallback));
}
}
// If no fallbacks to add, return original AST
if (fallbackExprs.length === 0) {
return ast;
}
// Combine fallbacks with existing WHERE using AND
let newWhere: Expression;
if (!existingWhere) {
// No existing WHERE - just use fallbacks
if (fallbackExprs.length === 1) {
newWhere = fallbackExprs[0];
} else {
newWhere = {
expression_type: "and",
exprs: fallbackExprs,
} as And;
}
} else {
// Combine existing WHERE with fallbacks
newWhere = {
expression_type: "and",
exprs: [...fallbackExprs, existingWhere],
} as And;
}
return {
...selectQuery,
where: newWhere,
};
}
/**
* Options for compiling a TSQL query to ClickHouse SQL
*/
export interface CompileTSQLOptions {
/** Schema definitions for allowed tables and columns */
tableSchema: TableSchema[];
/**
* REQUIRED: Conditions always applied at the table level.
* Must include tenant columns (e.g., organization_id) for multi-tenant tables.
* Applied to every table reference including subqueries, CTEs, and JOINs.
*
* @example
* ```typescript
* {
* // Tenant isolation
* organization_id: { op: "eq", value: "org_123" },
* project_id: { op: "eq", value: "proj_456" },
* environment_id: { op: "eq", value: "env_789" },
* // Plan-based time limit
* triggered_at: { op: "gte", value: new Date(Date.now() - 7 * 24 * 60 * 60 * 1000) }
* }
* ```
*/
enforcedWhereClause: Record<string, WhereClauseCondition | undefined>;
/** Optional query settings */
settings?: Partial<QuerySettings>;
/**
* Runtime field mappings for dynamic value translation.
* Maps internal ClickHouse values to external user-facing values.
*
* @example
* ```typescript
* {
* project: { "cm12345": "my-project-ref" },
* }
* ```
*/
fieldMappings?: FieldMappings;
/**
* Fallback WHERE conditions to apply when the user hasn't filtered on a column.
* Key is the column name, value is the fallback condition.
* These are applied at the AST level (top-level query only).
*
* @example
* ```typescript
* // Apply time > 7 days ago if user doesn't filter on time
* whereClauseFallback: {
* time: { op: 'gte', value: new Date(Date.now() - 7 * 24 * 60 * 60 * 1000) }
* }
*
* // Apply time BETWEEN two dates if user doesn't filter on time
* whereClauseFallback: {
* time: { op: 'between', low: startDate, high: endDate }
* }
* ```
*/
whereClauseFallback?: Record<string, WhereClauseCondition>;
/**
* Time range for `timeBucket()` interval calculation.
* When provided, `timeBucket()` uses this to determine the appropriate bucket size
* based on the span of the time range.
*
* @example
* ```typescript
* {
* from: new Date('2024-01-01'),
* to: new Date('2024-01-08'),
* }
* ```
*/
timeRange?: TimeRange;
}
/**
* Compile a TSQL query string to ClickHouse SQL with parameters
*
* This function:
* 1. Parses the TSQL query into an AST
* 2. Validates tables and columns against the schema
* 3. Injects enforced WHERE clauses (tenant isolation + plan limits) at printer level
* 4. Optionally injects fallback WHERE conditions at AST level
* 5. Generates parameterized ClickHouse SQL
*
* @param query - The TSQL query string to compile
* @param options - Compilation options including enforcedWhereClause and schema
* @returns The compiled SQL and parameters
* @throws TSQLSyntaxError if the query is invalid
* @throws QueryError if tables/columns are not allowed or required tenant columns are missing
*
* @example
* ```typescript
* const { sql, params } = compileTSQL(
* "SELECT * FROM task_runs WHERE status = 'completed' LIMIT 100",
* {
* tableSchema: [taskRunsSchema],
* enforcedWhereClause: {
* organization_id: { op: "eq", value: "org_123" },
* project_id: { op: "eq", value: "proj_456" },
* environment_id: { op: "eq", value: "env_789" },
* triggered_at: { op: "gte", value: new Date(Date.now() - 7 * 24 * 60 * 60 * 1000) },
* },
* }
* );
* ```
*/
export function compileTSQL(query: string, options: CompileTSQLOptions): PrintResult {
// 1. Parse the TSQL query
let ast = parseTSQLSelect(query);
// 2. Inject fallback WHERE conditions if provided (applied at AST level - top-level query only)
if (options.whereClauseFallback && Object.keys(options.whereClauseFallback).length > 0) {
ast = injectFallbackConditions(ast, options.whereClauseFallback);
}
// 3. Create schema registry from table schemas
const schemaRegistry = createSchemaRegistry(options.tableSchema);
// 4. Strip undefined values from enforcedWhereClause
const enforcedWhereClause = Object.fromEntries(
Object.entries(options.enforcedWhereClause).filter(([_, value]) => value !== undefined)
) as Record<string, WhereClauseCondition>;
// 5. Create printer context with enforced WHERE clause and field mappings
const context = createPrinterContext({
schema: schemaRegistry,
settings: options.settings,
fieldMappings: options.fieldMappings,
enforcedWhereClause,
timeRange: options.timeRange,
});
// 6. Print the AST to ClickHouse SQL (enforced conditions applied at printer level)
return printToClickHouse(ast, context);
}
internal-packages/tsql/src/query/ast.ts
+668
View File
@@ -0,0 +1,668 @@
// TypeScript translation of posthog/hogql/ast.py
import type { TSQLContext } from "./context";
import type {
DatabaseField,
FieldOrTable,
LazyJoin,
LazyTable,
Table,
UnknownDatabaseField,
VirtualTable,
} from "./models";
import type { ConstantDataType, TSQLQuerySettings } from "./constants";
// Base types
export interface AST {
start?: number;
end?: number;
accept?(visitor: any): any;
}
export interface Type extends AST {
get_child?(name: string, context: TSQLContext): Type;
has_child?(name: string, context: TSQLContext): boolean;
resolve_constant_type?(context: TSQLContext): ConstantType;
resolve_column_constant_type?(name: string, context: TSQLContext): ConstantType;
}
export interface Expr extends AST {
type?: Type;
}
export interface ConstantType extends Type {
data_type: ConstantDataType;
nullable?: boolean;
print_type?(): string;
}
export interface UnknownType extends ConstantType {
data_type: "unknown";
}
export type Expression =
| CTE
| Alias
| ArithmeticOperation
| And
| Or
| CompareOperation
| Not
| BetweenExpr
| OrderExpr
| ArrayAccess
| Array
| Dict
| TupleAccess
| Tuple
| Lambda
| Constant
| Field
| Placeholder
| Call
| ExprCall
| JoinConstraint
| JoinExpr
| WindowFrameExpr
| WindowExpr
| WindowFunction
| LimitByExpr
| SelectQuery
| SelectSetQuery
| RatioExpr
| SampleExpr
| TSQLXTag;
export interface CTE extends Expr {
expression_type: "cte";
name: string;
expr: Expression;
cte_type: "column" | "subquery";
}
// Type system
export type TableOrSelectType =
| BaseTableType
| SelectSetQueryType
| SelectQueryType
| SelectQueryAliasType;
export interface FieldAliasType extends Type {
alias: string;
type: Type;
}
export interface BaseTableType extends Type {
resolve_database_table?(context: TSQLContext): Table;
}
export interface TableType extends BaseTableType {
table: Table;
}
export interface LazyJoinType extends BaseTableType {
table_type: TableOrSelectType;
field: string;
lazy_join: LazyJoin;
}
export interface LazyTableType extends BaseTableType {
table: LazyTable;
}
export interface TableAliasType extends BaseTableType {
alias: string;
table_type: TableType | LazyTableType;
}
export interface VirtualTableType extends BaseTableType {
table_type: TableOrSelectType;
field: string;
virtual_table: VirtualTable;
}
export interface SelectQueryType extends Type {
aliases: Record<string, FieldAliasType>;
columns: Record<string, Type>;
tables: Record<string, TableOrSelectType>;
ctes: Record<string, CTE>;
anonymous_tables: (SelectQueryType | SelectSetQueryType)[];
parent?: SelectQueryType | SelectSetQueryType;
is_lambda_type?: boolean;
}
export interface SelectSetQueryType extends Type {
types: (SelectQueryType | SelectSetQueryType)[];
}
export interface SelectViewType extends BaseTableType {
view_name: string;
alias: string;
select_query_type: SelectQueryType | SelectSetQueryType;
}
export interface SelectQueryAliasType extends Type {
alias: string;
select_query_type: SelectQueryType | SelectSetQueryType;
}
export interface IntegerType extends ConstantType {
data_type: "int";
}
export interface DecimalType extends ConstantType {
data_type: "unknown";
}
export interface FloatType extends ConstantType {
data_type: "float";
}
export interface StringType extends ConstantType {
data_type: "str";
}
export interface StringJSONType extends StringType {}
export interface StringArrayType extends StringType {}
export interface BooleanType extends ConstantType {
data_type: "bool";
}
export interface DateType extends ConstantType {
data_type: "date";
}
export interface DateTimeType extends ConstantType {
data_type: "datetime";
}
export interface IntervalType extends ConstantType {
data_type: "unknown";
}
export interface UUIDType extends ConstantType {
data_type: "uuid";
}
export interface ArrayType extends ConstantType {
data_type: "array";
item_type: ConstantType;
}
export interface TupleType extends ConstantType {
data_type: "tuple";
item_types: ConstantType[];
repeat?: boolean;
}
export interface CallType extends Type {
name: string;
arg_types: ConstantType[];
param_types?: ConstantType[];
return_type: ConstantType;
}
export interface AsteriskType extends Type {
table_type: TableOrSelectType;
}
export interface FieldTraverserType extends Type {
chain: (string | number)[];
table_type: TableOrSelectType;
}
export interface ExpressionFieldType extends Type {
name: string;
expr: Expression;
table_type: TableOrSelectType;
isolate_scope?: boolean;
}
export interface FieldType extends Type {
name: string;
table_type: TableOrSelectType;
}
export interface UnresolvedFieldType extends Type {
name: string;
}
export interface PropertyType extends Type {
chain: (string | number)[];
field_type: FieldType;
joined_subquery?: SelectQueryAliasType;
joined_subquery_field_name?: string;
}
export interface LambdaArgumentType extends Type {
name: string;
}
// Enums
export enum ArithmeticOperationOp {
Add = "+",
Sub = "-",
Mult = "*",
Div = "/",
Mod = "%",
}
export enum CompareOperationOp {
Eq = "==",
NotEq = "!=",
Gt = ">",
GtEq = ">=",
Lt = "<",
LtEq = "<=",
Like = "like",
ILike = "ilike",
NotLike = "not like",
NotILike = "not ilike",
In = "in",
GlobalIn = "global in",
NotIn = "not in",
GlobalNotIn = "global not in",
InCohort = "in cohort",
NotInCohort = "not in cohort",
Regex = "=~",
IRegex = "=~*",
NotRegex = "!~",
NotIRegex = "!~*",
}
export const NEGATED_COMPARE_OPS: CompareOperationOp[] = [
CompareOperationOp.NotEq,
CompareOperationOp.NotLike,
CompareOperationOp.NotILike,
CompareOperationOp.NotIn,
CompareOperationOp.GlobalNotIn,
CompareOperationOp.NotInCohort,
CompareOperationOp.NotRegex,
CompareOperationOp.NotIRegex,
];
export type SetOperator =
| "UNION ALL"
| "UNION DISTINCT"
| "INTERSECT"
| "INTERSECT DISTINCT"
| "EXCEPT";
export type ParseResult = Expression | Declaration | string;
// Declaration and Statement types
export interface Declaration extends AST {}
export interface VariableAssignment extends Declaration {
left: Expression;
right: Expression;
}
export interface VariableDeclaration extends Declaration {
name: string;
expr?: Expression;
}
export interface Statement extends Declaration {}
export interface ExprStatement extends Statement {
expr?: Expression;
}
export interface ReturnStatement extends Statement {
expr?: Expression;
}
export interface ThrowStatement extends Statement {
expr?: Expression;
}
export interface TryCatchStatement extends Statement {
try_stmt: Statement;
catches: [string | null, string | null, Statement][];
finally_stmt?: Statement;
}
export interface IfStatement extends Statement {
expr: Expression;
then: Statement;
else_?: Statement;
}
export interface WhileStatement extends Statement {
expr: Expression;
body: Statement;
}
export interface ForStatement extends Statement {
initializer?: VariableDeclaration | VariableAssignment | Expression;
condition?: Expression;
increment?: VariableDeclaration;
body: Statement;
}
export interface ForInStatement extends Statement {
keyVar?: string;
valueVar: string;
expr: Expression;
body: Statement;
}
export interface Function extends Statement {
name: string;
params: string[];
body: Statement;
}
export interface Block extends Statement {
declarations: Declaration[];
}
export interface Program extends AST {
declarations: Declaration[];
}
// Expression types
export interface Alias extends Expr {
expression_type: "alias";
alias: string;
expr: Expression;
hidden?: boolean;
from_asterisk?: boolean;
}
export interface ArithmeticOperation extends Expr {
expression_type: "arithmetic_operation";
left: Expression;
right: Expression;
op: ArithmeticOperationOp;
}
export interface And extends Expr {
expression_type: "and";
type?: ConstantType;
exprs: Expression[];
}
export interface Or extends Expr {
expression_type: "or";
exprs: Expression[];
type?: ConstantType;
}
export interface CompareOperation extends Expr {
expression_type: "compare_operation";
left: Expression;
right: Expression;
op: CompareOperationOp;
type?: ConstantType;
}
export interface Not extends Expr {
expression_type: "not";
expr: Expression;
type?: ConstantType;
}
export interface BetweenExpr extends Expr {
expression_type: "between_expr";
expr: Expression;
low: Expression;
high: Expression;
negated?: boolean;
type?: ConstantType;
}
export interface OrderExpr extends Expr {
expression_type: "order_expr";
expr: Expression;
order?: "ASC" | "DESC";
}
export interface ArrayAccess extends Expr {
expression_type: "array_access";
array: Expression;
property: Expression;
nullish?: boolean;
}
export interface Array extends Expr {
expression_type: "array";
exprs: Expression[];
}
export interface Dict extends Expr {
expression_type: "dict";
items: [Expression, Expression][];
}
export interface TupleAccess extends Expr {
expression_type: "tuple_access";
tuple: Expression;
index: number;
nullish?: boolean;
}
export interface Tuple extends Expr {
expression_type: "tuple";
exprs: Expression[];
}
export interface Lambda extends Expr {
expression_type: "lambda";
args: string[];
expr: Expression | Block;
}
export interface Constant extends Expr {
expression_type: "constant";
value: any;
}
export interface Field extends Expr {
expression_type: "field";
chain: (string | number)[];
from_asterisk?: boolean;
}
export interface Placeholder extends Expr {
expression_type: "placeholder";
expr: Expression;
// Computed properties
chain?: (string | number)[] | null;
field?: string | null;
}
export interface Call extends Expr {
expression_type: "call";
name: string;
args: Expression[];
params?: Expression[];
distinct?: boolean;
}
export interface ExprCall extends Expr {
expression_type: "expr_call";
expr: Expression;
args: Expression[];
}
export interface JoinConstraint extends Expr {
expression_type: "join_constraint";
expr: Expression;
constraint_type: "ON" | "USING";
}
export interface JoinExpr extends Expr {
expression_type: "join_expr";
type?: TableOrSelectType;
join_type?: string;
table?: SelectQuery | SelectSetQuery | Placeholder | TSQLXTag | Field;
table_args?: Expression[];
alias?: string;
table_final?: boolean;
constraint?: JoinConstraint;
next_join?: JoinExpr;
sample?: SampleExpr;
}
export interface WindowFrameExpr extends Expr {
expression_type: "window_frame_expr";
frame_type?: "CURRENT ROW" | "PRECEDING" | "FOLLOWING";
frame_value?: number;
}
export interface WindowExpr extends Expr {
expression_type: "window_expr";
partition_by?: Expression[];
order_by?: OrderExpr[];
frame_method?: "ROWS" | "RANGE";
frame_start?: WindowFrameExpr;
frame_end?: WindowFrameExpr;
}
export interface WindowFunction extends Expr {
expression_type: "window_function";
name: string;
args?: Expression[];
exprs?: Expression[];
over_expr?: WindowExpr;
over_identifier?: string;
}
export interface LimitByExpr extends Expr {
expression_type: "limit_by_expr";
n: Expression;
exprs: Expression[];
offset_value?: Expression;
}
export interface SelectQuery extends Expr {
expression_type: "select_query";
type?: SelectQueryType;
ctes?: Record<string, CTE>;
select: Expression[];
distinct?: boolean;
select_from?: JoinExpr;
array_join_op?: string;
array_join_list?: Expression[];
window_exprs?: Record<string, WindowExpr>;
where?: Expression;
prewhere?: Expression;
having?: Expression;
group_by?: Expression[];
order_by?: OrderExpr[];
limit?: Expression;
limit_by?: LimitByExpr;
limit_with_ties?: boolean;
offset?: Expression;
settings?: TSQLQuerySettings;
view_name?: string;
}
export interface SelectSetNode extends AST {
select_query: SelectQuery | SelectSetQuery;
set_operator: SetOperator;
}
export interface SelectSetQuery extends Expr {
expression_type: "select_set_query";
type?: SelectSetQueryType;
initial_select_query: SelectQuery | SelectSetQuery;
subsequent_select_queries: SelectSetNode[];
// Equivalent to select_queries() method
select_queries?(): (SelectQuery | SelectSetQuery)[];
}
// Add static method equivalent for SelectSetQuery.create_from_queries()
export namespace SelectSetQuery {
export function createFromQueries(
queries: (SelectQuery | SelectSetQuery)[],
set_operator: SetOperator
): SelectQuery | SelectSetQuery {
return createSelectSetQueryFromQueries(queries, set_operator);
}
}
export interface RatioExpr extends Expr {
expression_type: "ratio_expr";
left: Constant;
right?: Constant;
}
export interface SampleExpr extends Expr {
expression_type: "sample_expr";
sample_value: RatioExpr;
offset_value?: RatioExpr;
}
export interface TSQLXAttribute extends AST {
name: string;
value: any;
}
export interface TSQLXTag extends Expr {
expression_type: "tsqlx_tag";
kind: string;
attributes: TSQLXAttribute[];
// Equivalent to to_dict() method
to_dict?(): Record<string, any>;
}
// Helper function to create empty SelectQuery (equivalent to SelectQuery.empty())
export function createEmptySelectQuery(columns?: Record<string, FieldOrTable>): SelectQuery {
if (!columns) {
columns = { _: { name: "_" } as UnknownDatabaseField };
}
return {
expression_type: "select_query",
select: Object.entries(columns).map(([column, field]) => ({
expression_type: "alias" as const,
alias: column,
expr: {
expression_type: "constant",
value: (field as DatabaseField).default_value?.() ?? null,
} as Constant,
})),
where: { expression_type: "constant", value: false } as Constant,
};
}
// Add static method equivalent for SelectQuery.empty()
export namespace SelectQuery {
export function empty(columns?: Record<string, FieldOrTable>): SelectQuery {
return createEmptySelectQuery(columns);
}
}
// Helper function for SelectSetQuery.select_queries()
export function selectQueries(query: SelectSetQuery): (SelectQuery | SelectSetQuery)[] {
return [
query.initial_select_query,
...query.subsequent_select_queries.map((node) => node.select_query),
];
}
// Helper function to create SelectSetQuery from multiple queries
export function createSelectSetQueryFromQueries(
queries: (SelectQuery | SelectSetQuery)[],
set_operator: SetOperator
): SelectQuery | SelectSetQuery {
if (queries.length === 0) {
throw new Error("Cannot create a SelectSetQuery from an empty list of queries");
} else if (queries.length === 1) {
return queries[0];
}
return {
expression_type: "select_set_query",
initial_select_query: queries[0],
subsequent_select_queries: queries.slice(1).map((query) => ({
select_query: query,
set_operator,
})) as SelectSetNode[],
} as SelectSetQuery;
}
internal-packages/tsql/src/query/constants.ts
+71
View File
@@ -0,0 +1,71 @@
// TypeScript translation of posthog/hogql/constants.py
export type ConstantDataType =
| "int"
| "float"
| "str"
| "bool"
| "array"
| "tuple"
| "date"
| "datetime"
| "uuid"
| "unknown";
export type ConstantSupportedPrimitive = number | string | boolean | Date | null;
export type ConstantSupportedData =
| ConstantSupportedPrimitive
| ConstantSupportedPrimitive[]
| [ConstantSupportedPrimitive, ...ConstantSupportedPrimitive[]];
export const KEYWORDS = ["true", "false", "null"] as const;
export const RESERVED_KEYWORDS = [...KEYWORDS, "team_id"] as const;
export const DEFAULT_RETURNED_ROWS = 100;
export const MAX_SELECT_RETURNED_ROWS = 50000;
export const MAX_SELECT_RETENTION_LIMIT = 100000;
export const MAX_SELECT_HEATMAPS_LIMIT = 1000000;
export const MAX_SELECT_COHORT_CALCULATION_LIMIT = 1000000000;
export const MAX_BYTES_BEFORE_EXTERNAL_GROUP_BY = 22 * 1024 * 1024 * 1024;
export const CSV_EXPORT_LIMIT = 300000;
export const CSV_EXPORT_BREAKDOWN_LIMIT_INITIAL = 512;
export const CSV_EXPORT_BREAKDOWN_LIMIT_LOW = 64;
export const BREAKDOWN_VALUES_LIMIT = 25;
export const BREAKDOWN_VALUES_LIMIT_FOR_COUNTRIES = 300;
export enum LimitContext {
QUERY = "query",
QUERY_ASYNC = "query_async",
EXPORT = "export",
COHORT_CALCULATION = "cohort_calculation",
HEATMAPS = "heatmaps",
SAVED_QUERY = "saved_query",
RETENTION = "retention",
}
// Settings applied at the SELECT level
export interface TSQLQuerySettings {
optimize_aggregation_in_order?: boolean;
date_time_output_format?: string;
date_time_input_format?: string;
join_algorithm?: string;
}
// Settings applied on top of all TSQL queries
export interface TSQLGlobalSettings extends TSQLQuerySettings {
readonly?: number;
max_execution_time?: number;
max_memory_usage?: number;
max_threads?: number;
allow_experimental_object_type?: boolean;
format_csv_allow_double_quotes?: boolean;
max_ast_elements?: number;
max_expanded_ast_elements?: number;
max_bytes_before_external_group_by?: number;
allow_experimental_analyzer?: boolean;
transform_null_in?: boolean;
optimize_min_equality_disjunction_chain_length?: number;
allow_experimental_join_condition?: boolean;
preferred_block_size_bytes?: number;
use_hive_partitioning?: number;
}
internal-packages/tsql/src/query/context.ts
+56
View File
@@ -0,0 +1,56 @@
// TypeScript translation of posthog/hogql/context.py
import type { LimitContext } from "./constants";
import type { Database } from "./database";
import type { PropertySwapper } from "./property_types";
import type { TSQLTimings } from "./timings";
export interface TSQLNotice {
start?: number;
end?: number;
message: string;
fix?: string;
}
export interface TSQLQueryModifiers {
optimizeJoinedFilters?: boolean;
debug?: boolean;
timings?: boolean;
useMaterializedViews?: boolean;
formatCsvAllowDoubleQuotes?: boolean;
convertToProjectTimezone?: boolean;
usePreaggregatedTableTransforms?: boolean;
optimizeProjections?: boolean;
}
export interface TSQLFieldAccess {
input: string[];
type?: "run";
field?: string;
sql: string;
}
export interface Team {
id: number;
project_id: number;
}
export interface TSQLContext {
team_id?: number;
team?: Team;
database?: Database;
values: Record<string, any>;
within_non_tsql_query?: boolean;
enable_select_queries?: boolean;
limit_top_select?: boolean;
limit_context?: LimitContext;
output_format?: string | null;
globals?: Record<string, any>;
warnings: TSQLNotice[];
notices: TSQLNotice[];
errors: TSQLNotice[];
timings: TSQLTimings;
modifiers: TSQLQueryModifiers;
debug?: boolean;
property_swapper?: PropertySwapper;
}
internal-packages/tsql/src/query/database.ts
+585
View File
@@ -0,0 +1,585 @@
// TypeScript translation of posthog/hogql/database/database.py
//
// NOTE: This implementation requires database/ORM access for:
// - serialize() method (needs DataWarehouseTable, DataWarehouseSavedQuery queries)
// - create_for() method (needs Team, DataWarehouseJoin, DataWarehouseSavedQuery queries)
// Adapt these methods to your database/ORM setup
import type { ConstantType } from "./ast";
import type { TSQLContext, TSQLQueryModifiers, Team } from "./context";
import type {
DatabaseField,
ExpressionField,
FieldOrTable,
FieldTraverser,
LazyJoin,
Table,
TableNode,
VirtualTable,
} from "./models";
import type { TSQLTimings } from "./timings";
import { QueryError, ResolutionError } from "./errors";
import { TSQLTimings as TSQLTimingsClass } from "./timings";
// Type definitions for schema serialization (adapt to your schema types)
export interface DatabaseSchemaTable {
fields: Record<string, DatabaseSchemaField>;
id: string;
name: string;
}
export interface DatabaseSchemaSystemTable extends DatabaseSchemaTable {
fields: Record<string, DatabaseSchemaField>;
id: string;
name: string;
}
export interface DatabaseSchemaDataWarehouseTable extends DatabaseSchemaTable {
fields: Record<string, DatabaseSchemaField>;
id: string;
name: string;
format?: string;
url_pattern?: string;
schema?: DatabaseSchemaSchema;
source?: DatabaseSchemaSource;
row_count?: number;
}
export interface DatabaseSchemaViewTable extends DatabaseSchemaTable {
fields: Record<string, DatabaseSchemaField>;
id: string;
name: string;
query: { query: string };
row_count?: number;
}
export interface DatabaseSchemaManagedViewTable extends DatabaseSchemaTable {
fields: Record<string, DatabaseSchemaField>;
id: string;
name: string;
kind: string;
source_id?: string;
query: { query: string };
}
export interface DatabaseSchemaEndpointTable extends DatabaseSchemaTable {
fields: Record<string, DatabaseSchemaField>;
id: string;
name: string;
query: { query: string };
row_count?: number;
status?: string;
}
export interface DatabaseSchemaField {
name: string;
tsql_value: string;
type: DatabaseSerializedFieldType;
schema_valid: boolean;
fields?: string[];
table?: string;
chain?: Array<string | number>;
id?: string;
}
export interface DatabaseSchemaSchema {
id: string;
name: string;
should_sync: boolean;
incremental: boolean;
status: string;
last_synced_at: string;
}
export interface DatabaseSchemaSource {
id: string;
status: string;
source_type: string;
prefix: string;
last_synced_at?: string | null;
}
export enum DatabaseSerializedFieldType {
STRING = "string",
INTEGER = "integer",
FLOAT = "float",
DECIMAL = "decimal",
BOOLEAN = "boolean",
DATE = "date",
DATETIME = "datetime",
UUID = "uuid",
ARRAY = "array",
JSON = "json",
TUPLE = "tuple",
UNKNOWN = "unknown",
EXPRESSION = "expression",
VIEW = "view",
LAZY_TABLE = "lazy_table",
VIRTUAL_TABLE = "virtual_table",
FIELD_TRAVERSER = "field_traverser",
}
export interface SerializedField {
key: string;
name: string;
type: DatabaseSerializedFieldType;
schema_valid: boolean;
fields?: string[];
table?: string;
chain?: Array<string | number>;
}
import { TableNodeImpl } from "./models";
export class Database {
// Users can query from the tables below
tables: TableNode;
private _warehouseTableNames: string[] = [];
private _warehouseSelfManagedTableNames: string[] = [];
private _viewTableNames: string[] = [];
private _coreTableNames: string[] = [];
private _timezone?: string | null;
private _weekStartDay?: string | null; // WeekStartDay enum
private _serializationErrors: Record<string, string> = {};
constructor(timezone?: string | null, weekStartDay?: string | null) {
// Initialize with root TableNode
this.tables = new TableNodeImpl("root");
this._timezone = timezone || null;
this._weekStartDay = weekStartDay || null;
}
getTimezone(): string {
return this._timezone || "UTC";
}
getWeekStartDay(): string {
return this._weekStartDay || "sunday"; // Adapt to your WeekStartDay enum
}
getSerializationErrors(): Record<string, string> {
/** Return any errors encountered during serialization. */
return { ...this._serializationErrors };
}
hasTable(tableName: string | string[]): boolean {
const path = typeof tableName === "string" ? tableName.split(".") : tableName;
return this.tables.has_child ? this.tables.has_child(path) : false;
}
getTableNode(tableName: string | string[]): TableNode {
let path: string[];
if (typeof tableName === "string") {
path = tableName.split(".");
} else {
path = tableName;
}
// Handle edge case where tableName is a list with a single string containing dots
if (path.length === 1 && path[0].includes(".")) {
path = path[0].split(".");
}
if (!this.tables.get_child) {
throw new ResolutionError(`TableNode.get_child not implemented`);
}
return this.tables.get_child(path);
}
getTable(tableName: string | string[]): Table {
try {
const node = this.getTableNode(tableName);
if (!node.get) {
throw new ResolutionError("TableNode.get not implemented");
}
const table = node.get();
if (!table || typeof table !== "object" || !("fields" in table)) {
throw new ResolutionError("Table is not set");
}
return table as Table;
} catch (e) {
const name = Array.isArray(tableName) ? tableName.join(".") : tableName;
if (e instanceof ResolutionError) {
throw new QueryError(`Unknown table \`${name}\`.`);
}
throw e;
}
}
getAllTableNames(): string[] {
const warehouseTableNames = this._warehouseTableNames.filter((x) => x.includes("."));
return [
...this._coreTableNames,
...warehouseTableNames,
...this._warehouseSelfManagedTableNames,
...this._viewTableNames,
];
}
// Core tables exposed via SQL editor autocomplete and data management
getCoreTableNames(): string[] {
return [...this._coreTableNames, ...this.getSystemTableNames()];
}
getSystemTableNames(): string[] {
const systemNode = this.tables.children["system"];
if (systemNode && systemNode.resolve_all_table_names) {
return ["query_log", ...systemNode.resolve_all_table_names()];
}
return ["query_log"];
}
getWarehouseTableNames(): string[] {
return [...this._warehouseTableNames, ...this._warehouseSelfManagedTableNames];
}
getViewNames(): string[] {
return this._viewTableNames;
}
addCoreTable(tableName: string, node: TableNode): void {
if (this.tables.add_child) {
this.tables.add_child(node);
}
this._coreTableNames.push(tableName);
}
private _addWarehouseTables(node: TableNode): void {
if (this.tables.merge_with) {
this.tables.merge_with(node);
}
if (node.resolve_all_table_names) {
const names = node.resolve_all_table_names();
this._warehouseTableNames.push(...names.sort());
}
}
private _addWarehouseSelfManagedTables(node: TableNode): void {
if (this.tables.merge_with) {
this.tables.merge_with(node);
}
if (node.resolve_all_table_names) {
const names = node.resolve_all_table_names();
this._warehouseSelfManagedTableNames.push(...names.sort());
}
}
private _addViews(node: TableNode): void {
if (this.tables.merge_with) {
this.tables.merge_with(node);
}
if (node.resolve_all_table_names) {
const names = node.resolve_all_table_names();
this._viewTableNames.push(...names.sort());
}
}
serialize(context: TSQLContext, includeOnly?: Set<string>): Record<string, DatabaseSchemaTable> {
// NOTE: This method requires database queries to fetch:
// - DataWarehouseTable objects
// - DataWarehouseSavedQuery objects
// - External data sources and schemas
//
// Adapt this to your database/ORM setup
const tables: Record<string, DatabaseSchemaTable> = {};
if (!context.team_id) {
throw new ResolutionError("Must provide team_id to serialize database");
}
// Core tables
const coreTableNames = this.getCoreTableNames();
for (const tableName of coreTableNames) {
if (includeOnly && !includeOnly.has(tableName)) {
continue;
}
let fieldInput: Record<string, FieldOrTable> = {};
const table = this.getTable(tableName);
if ("get_asterisk" in table && typeof table.get_asterisk === "function") {
fieldInput = table.get_asterisk() || {};
} else if ("fields" in table) {
fieldInput = table.fields;
}
const fields = serializeFields(fieldInput, context, tableName.split("."), undefined);
const fieldsDict: Record<string, DatabaseSchemaField> = {};
for (const field of fields) {
fieldsDict[field.name] = field;
}
tables[tableName] = {
fields: fieldsDict,
id: tableName,
name: tableName,
} as DatabaseSchemaTable;
}
// System tables
const systemTables = this.getSystemTableNames();
for (const tableKey of systemTables) {
if (includeOnly && !includeOnly.has(tableKey)) {
continue;
}
let systemFieldInput: Record<string, FieldOrTable> = {};
const table = this.getTable(tableKey);
if ("get_asterisk" in table && typeof table.get_asterisk === "function") {
systemFieldInput = table.get_asterisk() || {};
} else if ("fields" in table) {
systemFieldInput = table.fields;
}
const fields = serializeFields(systemFieldInput, context, tableKey.split("."), undefined);
const fieldsDict: Record<string, DatabaseSchemaField> = {};
for (const field of fields) {
fieldsDict[field.name] = field;
}
tables[tableKey] = {
fields: fieldsDict,
id: tableKey,
name: tableKey,
} as DatabaseSchemaSystemTable;
}
// NOTE: Data Warehouse Tables and Views processing requires database queries
// Implement based on your database/ORM setup:
// - Fetch DataWarehouseTable objects
// - Fetch DataWarehouseSavedQuery objects
// - Process and serialize them
return tables;
}
static createFor(
teamId?: number,
options?: {
team?: Team;
modifiers?: TSQLQueryModifiers;
timings?: TSQLTimings;
}
): Database {
// NOTE: This method requires extensive database/ORM access:
// - Team model queries
// - DataWarehouseTable queries
// - DataWarehouseSavedQuery queries
// - DataWarehouseJoin queries
// - GroupTypeMapping queries
// - Feature flag checks
//
// This is a skeleton structure - adapt to your setup
const timings = options?.timings || new TSQLTimingsClass();
const { team, modifiers: _modifiers } = options || {};
// Validate team/teamId
if (!teamId && !team) {
throw new Error("Either team_id or team must be provided");
}
if (team && teamId && team.id !== teamId) {
throw new Error("team_id and team must be the same");
}
// NOTE: Fetch team from database if not provided
// const fetchedTeam = team || await Team.findById(teamId);
// Create database instance
const database = timings.measure("database", () => {
// NOTE: Get timezone and week_start_day from team
// const timezone = fetchedTeam.timezone;
// const weekStartDay = fetchedTeam.week_start_day;
return new Database(undefined, undefined);
});
// NOTE: Apply modifiers, setup tables, etc.
// This requires extensive database access and table setup logic
return database;
}
}
// Helper functions
const TSQL_CHARACTERS_TO_BE_WRAPPED = ["@", "-", "!", "$", "+"];
function constantTypeToSerializedFieldType(
constantType: ConstantType
): DatabaseSerializedFieldType | null {
// Type checking for ConstantType subtypes
// NOTE: In TypeScript, we need to check properties rather than instanceof
// since these are interfaces, not classes
if ("data_type" in constantType) {
const dataType = constantType.data_type;
if (dataType === "str") {
return DatabaseSerializedFieldType.STRING;
}
if (dataType === "bool") {
return DatabaseSerializedFieldType.BOOLEAN;
}
if (dataType === "date") {
return DatabaseSerializedFieldType.DATE;
}
if (dataType === "datetime") {
return DatabaseSerializedFieldType.DATETIME;
}
if (dataType === "uuid") {
return DatabaseSerializedFieldType.STRING;
}
if (dataType === "array") {
return DatabaseSerializedFieldType.ARRAY;
}
if (dataType === "tuple") {
return DatabaseSerializedFieldType.JSON;
}
if (dataType === "int") {
return DatabaseSerializedFieldType.INTEGER;
}
if (dataType === "float") {
return DatabaseSerializedFieldType.FLOAT;
}
}
// Fallback: check print_type if available
if ("print_type" in constantType && typeof constantType.print_type === "function") {
const printed = constantType.print_type();
if (printed === "String" || printed === "JSON" || printed === "Array") {
return printed === "String"
? DatabaseSerializedFieldType.STRING
: printed === "JSON"
? DatabaseSerializedFieldType.JSON
: DatabaseSerializedFieldType.ARRAY;
}
if (printed === "Boolean") return DatabaseSerializedFieldType.BOOLEAN;
if (printed === "Date") return DatabaseSerializedFieldType.DATE;
if (printed === "DateTime") return DatabaseSerializedFieldType.DATETIME;
if (printed === "UUID") return DatabaseSerializedFieldType.STRING;
if (printed === "Integer") return DatabaseSerializedFieldType.INTEGER;
if (printed === "Float") return DatabaseSerializedFieldType.FLOAT;
if (printed === "Decimal") return DatabaseSerializedFieldType.DECIMAL;
}
return null;
}
export function serializeFields(
fieldInput: Record<string, FieldOrTable>,
context: TSQLContext,
tableChain: string[],
dbColumns?: Record<string, any> // DataWarehouseTableColumns
): DatabaseSchemaField[] {
// NOTE: This requires resolve_types_from_table from resolver
// Import as needed: import { resolveTypesFromTable } from '../resolver';
const fieldOutput: DatabaseSchemaField[] = [];
for (const [fieldKey, field] of Object.entries(fieldInput)) {
let schemaValid = true;
if (dbColumns) {
const column = dbColumns[fieldKey];
if (typeof column === "string") {
schemaValid = true;
} else if (column && typeof column === "object") {
schemaValid = column.valid !== false;
}
}
let tsqlValue: string;
if (TSQL_CHARACTERS_TO_BE_WRAPPED.some((char) => fieldKey.includes(char))) {
tsqlValue = `\`${fieldKey}\``;
} else {
tsqlValue = fieldKey;
}
if ("hidden" in field && field.hidden) {
continue;
}
if ("name" in field && "get_constant_type" in field) {
// DatabaseField
const dbField = field as DatabaseField;
let fieldType: DatabaseSerializedFieldType;
// Determine field type based on DatabaseField subclass
// NOTE: You'll need to check instanceof or use type guards
// For now, using a simplified approach
if (dbField.get_constant_type) {
const constantType = dbField.get_constant_type();
fieldType =
constantTypeToSerializedFieldType(constantType) || DatabaseSerializedFieldType.UNKNOWN;
} else {
fieldType = DatabaseSerializedFieldType.UNKNOWN;
}
fieldOutput.push({
name: fieldKey,
tsql_value: tsqlValue,
type: fieldType,
schema_valid: schemaValid,
});
} else if ("expr" in field) {
// ExpressionField
const _exprField = field as ExpressionField;
// NOTE: Requires resolve_types_from_table
// const resolvedExpr = resolveTypesFromTable(exprField.expr, tableChain, context, 'tsql');
// const constantType = resolvedExpr.type?.resolve_constant_type(context);
// const fieldType = constantTypeToSerializedFieldType(constantType) || DatabaseSerializedFieldType.EXPRESSION;
fieldOutput.push({
name: fieldKey,
tsql_value: tsqlValue,
type: DatabaseSerializedFieldType.EXPRESSION,
schema_valid: schemaValid,
});
} else if ("resolve_table" in field) {
// LazyJoin
const lazyJoin = field as LazyJoin;
if (lazyJoin.resolve_table) {
const resolvedTable = lazyJoin.resolve_table(context);
const type =
"id" in resolvedTable && resolvedTable.id
? DatabaseSerializedFieldType.VIEW
: DatabaseSerializedFieldType.LAZY_TABLE;
fieldOutput.push({
name: fieldKey,
tsql_value: tsqlValue,
type,
schema_valid: schemaValid,
table: resolvedTable.to_printed_tsql ? resolvedTable.to_printed_tsql() : fieldKey,
fields: "fields" in resolvedTable ? Object.keys(resolvedTable.fields) : [],
id: "id" in resolvedTable && resolvedTable.id ? String(resolvedTable.id) : fieldKey,
});
}
} else if ("fields" in field && !("resolve_table" in field)) {
// VirtualTable
const virtualTable = field as VirtualTable;
fieldOutput.push({
name: fieldKey,
tsql_value: tsqlValue,
type: DatabaseSerializedFieldType.VIRTUAL_TABLE,
schema_valid: schemaValid,
table: virtualTable.to_printed_tsql ? virtualTable.to_printed_tsql() : fieldKey,
fields: Object.keys(virtualTable.fields),
});
} else if ("chain" in field) {
// FieldTraverser
const traverser = field as FieldTraverser;
fieldOutput.push({
name: fieldKey,
tsql_value: tsqlValue,
type: DatabaseSerializedFieldType.FIELD_TRAVERSER,
schema_valid: schemaValid,
chain: traverser.chain,
});
}
}
return fieldOutput;
}
internal-packages/tsql/src/query/errors.ts
+61
View File
@@ -0,0 +1,61 @@
// TypeScript translation of posthog/hogql/errors.py
import type { Expr } from "./ast";
export class BaseTSQLError extends Error {
message: string;
start?: number;
end?: number;
constructor(
message: string,
options?: {
start?: number;
end?: number;
node?: Expr;
}
) {
super(message);
this.message = message;
if (options?.node && options.node.start !== undefined && options.node.end !== undefined) {
this.start = options.node.start;
this.end = options.node.end;
} else {
this.start = options?.start;
this.end = options?.end;
}
}
}
export class ExposedTSQLError extends BaseTSQLError {
/** An exception that can be exposed to the user. */
}
export class InternalTSQLError extends BaseTSQLError {
/** An internal exception in the TSQL engine. */
}
export class SyntaxError extends ExposedTSQLError {
/** The input does not conform to TSQL syntax. */
}
export class QueryError extends ExposedTSQLError {
/** The query is invalid, though correct syntactically. */
}
export class NotImplementedError extends InternalTSQLError {
/** This feature isn't implemented in TSQL (yet). */
}
export class ParsingError extends InternalTSQLError {
/** Parsing failed. */
}
export class ImpossibleASTError extends InternalTSQLError {
/** Parsing or resolution resulted in an impossible AST. */
}
export class ResolutionError extends InternalTSQLError {
/** Resolution of a table/field/expression failed. */
}
internal-packages/tsql/src/query/escape.test.ts
+241
View File
@@ -0,0 +1,241 @@
import { describe, it, expect } from "vitest";
import {
escapeClickHouseIdentifier,
escapeTSQLIdentifier,
escapeClickHouseString,
escapeTSQLString,
getClickHouseType,
SQLValueEscaper,
safeIdentifier,
} from "./escape.js";
import { QueryError } from "./errors.js";
describe("escapeClickHouseIdentifier", () => {
it("should pass through simple identifiers", () => {
expect(escapeClickHouseIdentifier("id")).toBe("id");
expect(escapeClickHouseIdentifier("user_name")).toBe("user_name");
expect(escapeClickHouseIdentifier("Column1")).toBe("Column1");
expect(escapeClickHouseIdentifier("_private")).toBe("_private");
});
it("should escape identifiers with special characters", () => {
expect(escapeClickHouseIdentifier("my column")).toBe("`my column`");
expect(escapeClickHouseIdentifier("table-name")).toBe("`table-name`");
expect(escapeClickHouseIdentifier("column.with.dots")).toBe("`column.with.dots`");
});
it("should escape identifiers starting with numbers", () => {
expect(escapeClickHouseIdentifier("1column")).toBe("`1column`");
expect(escapeClickHouseIdentifier("123")).toBe("`123`");
});
it("should escape backticks in identifiers", () => {
expect(escapeClickHouseIdentifier("column`name")).toBe("`column\\`name`");
});
it("should escape control characters", () => {
expect(escapeClickHouseIdentifier("col\nname")).toBe("`col\\nname`");
expect(escapeClickHouseIdentifier("col\tname")).toBe("`col\\tname`");
});
it("should throw for identifiers containing %", () => {
expect(() => escapeClickHouseIdentifier("column%name")).toThrow(QueryError);
});
});
describe("escapeTSQLIdentifier", () => {
it("should pass through simple identifiers", () => {
expect(escapeTSQLIdentifier("id")).toBe("id");
expect(escapeTSQLIdentifier("user_name")).toBe("user_name");
});
it("should allow dollar signs in identifiers", () => {
expect(escapeTSQLIdentifier("$property")).toBe("$property");
expect(escapeTSQLIdentifier("property$value")).toBe("property$value");
});
it("should handle numeric identifiers", () => {
expect(escapeTSQLIdentifier(0)).toBe("0");
expect(escapeTSQLIdentifier(123)).toBe("123");
});
it("should throw for identifiers containing %", () => {
expect(() => escapeTSQLIdentifier("column%name")).toThrow(QueryError);
});
});
describe("SQLValueEscaper", () => {
describe("ClickHouse dialect", () => {
const escaper = new SQLValueEscaper({ dialect: "clickhouse" });
it("should escape null", () => {
expect(escaper.visit(null)).toBe("NULL");
expect(escaper.visit(undefined)).toBe("NULL");
});
it("should escape booleans as numbers", () => {
expect(escaper.visit(true)).toBe("1");
expect(escaper.visit(false)).toBe("0");
});
it("should escape integers", () => {
expect(escaper.visit(0)).toBe("0");
expect(escaper.visit(42)).toBe("42");
expect(escaper.visit(-100)).toBe("-100");
});
it("should escape floats", () => {
expect(escaper.visit(3.14)).toBe("3.14");
expect(escaper.visit(-0.5)).toBe("-0.5");
});
it("should escape special floats", () => {
expect(escaper.visit(NaN)).toBe("NaN");
expect(escaper.visit(Infinity)).toBe("Inf");
expect(escaper.visit(-Infinity)).toBe("-Inf");
});
it("should escape strings with quotes", () => {
expect(escaper.visit("hello")).toBe("'hello'");
expect(escaper.visit("hello'world")).toBe("'hello\\'world'");
});
it("should escape strings with control characters", () => {
expect(escaper.visit("line1\nline2")).toBe("'line1\\nline2'");
expect(escaper.visit("col1\tcol2")).toBe("'col1\\tcol2'");
});
it("should escape arrays", () => {
expect(escaper.visit([1, 2, 3])).toBe("[1, 2, 3]");
expect(escaper.visit(["a", "b"])).toBe("['a', 'b']");
expect(escaper.visit(["hello", "world"])).toBe("['hello', 'world']");
});
it("should escape nested arrays", () => {
expect(
escaper.visit([
[1, 2],
[3, 4],
])
).toBe("[[1, 2], [3, 4]]");
});
it("should escape dates with toDateTime64 and default UTC timezone", () => {
const date = new Date("2024-01-15T10:30:00.500Z");
const result = escaper.visit(date);
expect(result).toBe("toDateTime64('2024-01-15 10:30:00.500000', 6, 'UTC')");
});
it("should escape dates with custom timezone", () => {
const escaperWithTz = new SQLValueEscaper({
dialect: "clickhouse",
timezone: "America/New_York",
});
const date = new Date("2024-01-15T10:30:00.500Z");
const result = escaperWithTz.visit(date);
expect(result).toBe("toDateTime64('2024-01-15 10:30:00.500000', 6, 'America/New_York')");
});
});
describe("TSQL dialect", () => {
const escaper = new SQLValueEscaper({ dialect: "tsql" });
it("should escape booleans as keywords", () => {
expect(escaper.visit(true)).toBe("true");
expect(escaper.visit(false)).toBe("false");
});
it("should escape dates with toDateTime and default UTC timezone", () => {
const date = new Date("2024-01-15T10:30:00.500Z");
const result = escaper.visit(date);
expect(result).toBe("toDateTime('2024-01-15 10:30:00.500000', 'UTC')");
});
it("should escape dates with custom timezone", () => {
const escaperWithTz = new SQLValueEscaper({
dialect: "tsql",
timezone: "Europe/London",
});
const date = new Date("2024-01-15T10:30:00.500Z");
const result = escaperWithTz.visit(date);
expect(result).toBe("toDateTime('2024-01-15 10:30:00.500000', 'Europe/London')");
});
});
});
describe("escapeClickHouseString", () => {
it("should escape string values", () => {
expect(escapeClickHouseString("test")).toBe("'test'");
});
it("should handle null", () => {
expect(escapeClickHouseString(null)).toBe("NULL");
});
it("should handle numbers", () => {
expect(escapeClickHouseString(42)).toBe("42");
});
});
describe("escapeTSQLString", () => {
it("should escape string values", () => {
expect(escapeTSQLString("test")).toBe("'test'");
});
it("should handle booleans differently from ClickHouse", () => {
expect(escapeTSQLString(true)).toBe("true");
expect(escapeTSQLString(false)).toBe("false");
});
});
describe("getClickHouseType", () => {
it("should return String for strings", () => {
expect(getClickHouseType("hello")).toBe("String");
});
it("should return UInt8 for booleans", () => {
expect(getClickHouseType(true)).toBe("UInt8");
expect(getClickHouseType(false)).toBe("UInt8");
});
it("should return Int32 for small integers", () => {
expect(getClickHouseType(42)).toBe("Int32");
expect(getClickHouseType(-100)).toBe("Int32");
});
it("should return Int64 for large integers", () => {
expect(getClickHouseType(3000000000)).toBe("Int64");
expect(getClickHouseType(-3000000000)).toBe("Int64");
});
it("should return Float64 for floats", () => {
expect(getClickHouseType(3.14)).toBe("Float64");
});
it("should return DateTime64(6) for dates", () => {
expect(getClickHouseType(new Date())).toBe("DateTime64(6)");
});
it("should return Array type for arrays", () => {
expect(getClickHouseType(["a", "b"])).toBe("Array(String)");
expect(getClickHouseType([1, 2])).toBe("Array(Int32)");
expect(getClickHouseType([])).toBe("Array(String)");
});
it("should return Nullable(String) for null", () => {
expect(getClickHouseType(null)).toBe("Nullable(String)");
expect(getClickHouseType(undefined)).toBe("Nullable(String)");
});
});
describe("safeIdentifier", () => {
it("should return identifier unchanged if no %", () => {
expect(safeIdentifier("column")).toBe("column");
expect(safeIdentifier("table_name")).toBe("table_name");
});
it("should remove % characters", () => {
expect(safeIdentifier("column%name")).toBe("columnname");
expect(safeIdentifier("%%test%%")).toBe("test");
});
});
internal-packages/tsql/src/query/escape.ts
+265
View File
@@ -0,0 +1,265 @@
// TypeScript port of posthog/hogql/escape_sql.py
// Keep this file in sync with the Python version
import { QueryError } from "./errors";
/**
* Character escape maps for ClickHouse string escaping
* Copied from clickhouse_driver.util.escape
*
* Note: In JavaScript, \a and \v are not recognized escape sequences like in Python.
* We use the actual ASCII codes: \x07 for bell (Python's \a) and \x0B for vertical tab.
*/
const escapeCharsMap: Record<string, string> = {
"\b": "\\b",
"\f": "\\f",
"\r": "\\r",
"\n": "\\n",
"\t": "\\t",
"\0": "\\0",
"\x07": "\\a", // Bell character (ASCII 7) - Python's \a
"\x0B": "\\v", // Vertical tab (ASCII 11) - use explicit code since JS \v may not work in all contexts
"\\": "\\\\",
};
const singlequoteEscapeCharsMap: Record<string, string> = {
...escapeCharsMap,
"'": "\\'",
};
const backquoteEscapeCharsMap: Record<string, string> = {
...escapeCharsMap,
"`": "\\`",
};
/**
* Sanitize an identifier by removing % characters
*/
export function safeIdentifier(identifier: string): string {
if (identifier.includes("%")) {
return identifier.replace(/%/g, "");
}
return identifier;
}
/**
* Escape a string value for use as a parameter in ClickHouse
* Copied from clickhouse_driver.util.escape_param
*/
export function escapeParamClickhouse(value: string): string {
const escaped = value
.split("")
.map((c) => singlequoteEscapeCharsMap[c] || c)
.join("");
return `'${escaped}'`;
}
/**
* Escape an identifier for use in TSQL/HogQL queries
* Adapted from clickhouse_driver.util.escape with support for $ in identifiers
*/
export function escapeTSQLIdentifier(identifier: string | number): string {
if (typeof identifier === "number") {
// In TSQL we allow integers as identifiers to access array elements
return String(identifier);
}
if (identifier.includes("%")) {
throw new QueryError(
`The TSQL identifier "${identifier}" is not permitted as it contains the "%" character`
);
}
// TSQL allows dollars in the identifier (same regex as frontend escapePropertyAsTSQLIdentifier)
if (/^[A-Za-z_$][A-Za-z0-9_$]*$/.test(identifier)) {
return identifier;
}
const escaped = identifier
.split("")
.map((c) => backquoteEscapeCharsMap[c] || c)
.join("");
return `\`${escaped}\``;
}
/**
* Escape an identifier for use in ClickHouse queries
* Copied from clickhouse_driver.util.escape, adapted from single quotes to backquotes
*/
export function escapeClickHouseIdentifier(identifier: string): string {
if (identifier.includes("%")) {
throw new QueryError(
`The identifier "${identifier}" is not permitted as it contains the "%" character`
);
}
if (/^[A-Za-z_][A-Za-z0-9_]*$/.test(identifier)) {
return identifier;
}
const escaped = identifier
.split("")
.map((c) => backquoteEscapeCharsMap[c] || c)
.join("");
return `\`${escaped}\``;
}
/**
* Type for values that can be escaped as SQL strings
*/
export type EscapableValue =
| null
| undefined
| string
| number
| boolean
| Date
| EscapableValue[]
| [EscapableValue, ...EscapableValue[]];
/**
* SQL Value Escaper class that handles different types of values
* Port of SQLValueEscaper from escape_sql.py
*/
export class SQLValueEscaper {
private timezone: string;
private dialect: "tsql" | "clickhouse";
constructor(options: { timezone?: string; dialect?: "tsql" | "clickhouse" } = {}) {
this.timezone = options.timezone || "UTC";
this.dialect = options.dialect || "clickhouse";
}
visit(value: EscapableValue): string {
if (value === null || value === undefined) {
return this.visitNull();
}
if (typeof value === "string") {
return this.visitString(value);
}
if (typeof value === "boolean") {
return this.visitBoolean(value);
}
if (typeof value === "number") {
if (Number.isInteger(value)) {
return this.visitInt(value);
}
return this.visitFloat(value);
}
if (value instanceof Date) {
return this.visitDateTime(value);
}
if (Array.isArray(value)) {
return this.visitArray(value);
}
throw new QueryError(`SQLValueEscaper cannot handle value of type ${typeof value}`);
}
private visitNull(): string {
return "NULL";
}
private visitString(value: string): string {
return escapeParamClickhouse(value);
}
private visitBoolean(value: boolean): string {
if (this.dialect === "clickhouse") {
return value ? "1" : "0";
}
return value ? "true" : "false";
}
private visitInt(value: number): string {
return String(value);
}
private visitFloat(value: number): string {
if (Number.isNaN(value)) {
return "NaN";
}
if (!Number.isFinite(value)) {
return value < 0 ? "-Inf" : "Inf";
}
return String(value);
}
private visitDateTime(value: Date): string {
// Format: YYYY-MM-DD HH:MM:SS.ffffff
const pad = (n: number, len: number = 2) => String(n).padStart(len, "0");
const year = value.getUTCFullYear();
const month = pad(value.getUTCMonth() + 1);
const day = pad(value.getUTCDate());
const hours = pad(value.getUTCHours());
const minutes = pad(value.getUTCMinutes());
const seconds = pad(value.getUTCSeconds());
const ms = pad(value.getUTCMilliseconds(), 3);
const datetimeString = `${year}-${month}-${day} ${hours}:${minutes}:${seconds}.${ms}000`;
if (this.dialect === "tsql") {
return `toDateTime(${this.visitString(datetimeString)}, ${this.visitString(this.timezone)})`;
}
return `toDateTime64(${this.visitString(datetimeString)}, 6, ${this.visitString(
this.timezone
)})`;
}
private visitArray(value: EscapableValue[]): string {
return `[${value.map((x) => this.visit(x)).join(", ")}]`;
}
}
/**
* Escape a value for use in a TSQL/HogQL query string
*/
export function escapeTSQLString(value: EscapableValue, timezone?: string): string {
return new SQLValueEscaper({ timezone, dialect: "tsql" }).visit(value);
}
/**
* Escape a value for use in a ClickHouse query string
*/
export function escapeClickHouseString(value: EscapableValue, timezone?: string): string {
return new SQLValueEscaper({ timezone, dialect: "clickhouse" }).visit(value);
}
/**
* Get the ClickHouse type string for a value
* Used when creating parameterized query placeholders like {param: Type}
*/
export function getClickHouseType(value: unknown): string {
if (value === null || value === undefined) {
return "Nullable(String)";
}
if (typeof value === "string") {
return "String";
}
if (typeof value === "boolean") {
return "UInt8";
}
if (typeof value === "number") {
if (Number.isInteger(value)) {
// Use Int64 for large integers, Int32 for smaller ones
if (value > 2147483647 || value < -2147483648) {
return "Int64";
}
return "Int32";
}
return "Float64";
}
if (value instanceof Date) {
return "DateTime64(6)";
}
if (Array.isArray(value)) {
if (value.length === 0) {
return "Array(String)";
}
const itemType = getClickHouseType(value[0]);
return `Array(${itemType})`;
}
// Default to String for unknown types
return "String";
}
internal-packages/tsql/src/query/functions.ts
+789
View File
@@ -0,0 +1,789 @@
// TypeScript port of posthog/hogql/functions/mapping.py and aggregations.py
// Keep this file in sync with the Python version
import { CompareOperationOp } from "./ast";
/**
* Metadata for a TSQL function
*/
export interface TSQLFunctionMeta {
/** The ClickHouse function name to use */
clickhouseName: string;
/** Minimum number of arguments */
minArgs: number;
/** Maximum number of arguments (undefined means unlimited) */
maxArgs?: number;
/** Minimum number of parameters (for parametric functions) */
minParams?: number;
/** Maximum number of parameters */
maxParams?: number;
/** Whether this is an aggregate function */
aggregate?: boolean;
/** Whether function is case-sensitive */
caseSensitive?: boolean;
/** Whether function is timezone-aware (will append timezone as last arg) */
tzAware?: boolean;
/** Whether the function uses placeholder arguments like {} */
usingPlaceholderArguments?: boolean;
}
/**
* Comparison function mappings from function names to CompareOperationOp
*/
export const TSQL_COMPARISON_MAPPING: Record<string, CompareOperationOp> = {
equals: CompareOperationOp.Eq,
notEquals: CompareOperationOp.NotEq,
less: CompareOperationOp.Lt,
greater: CompareOperationOp.Gt,
lessOrEquals: CompareOperationOp.LtEq,
greaterOrEquals: CompareOperationOp.GtEq,
like: CompareOperationOp.Like,
ilike: CompareOperationOp.ILike,
notLike: CompareOperationOp.NotLike,
notILike: CompareOperationOp.NotILike,
in: CompareOperationOp.In,
notIn: CompareOperationOp.NotIn,
};
/**
* ClickHouse functions available in TSQL
* Port of HOGQL_CLICKHOUSE_FUNCTIONS from mapping.py
*/
export const TSQL_CLICKHOUSE_FUNCTIONS: Record<string, TSQLFunctionMeta> = {
// Comparison
equals: { clickhouseName: "equals", minArgs: 2, maxArgs: 2 },
notEquals: { clickhouseName: "notEquals", minArgs: 2, maxArgs: 2 },
less: { clickhouseName: "less", minArgs: 2, maxArgs: 2 },
greater: { clickhouseName: "greater", minArgs: 2, maxArgs: 2 },
lessOrEquals: { clickhouseName: "lessOrEquals", minArgs: 2, maxArgs: 2 },
greaterOrEquals: { clickhouseName: "greaterOrEquals", minArgs: 2, maxArgs: 2 },
// Logical
and: { clickhouseName: "and", minArgs: 2 },
or: { clickhouseName: "or", minArgs: 2 },
xor: { clickhouseName: "xor", minArgs: 2 },
not: { clickhouseName: "not", minArgs: 1, maxArgs: 1, caseSensitive: false },
// Conditional
if: { clickhouseName: "if", minArgs: 3, maxArgs: 3, caseSensitive: false },
multiIf: { clickhouseName: "multiIf", minArgs: 3 },
// In
in: { clickhouseName: "in", minArgs: 2, maxArgs: 2 },
notIn: { clickhouseName: "notIn", minArgs: 2, maxArgs: 2 },
// Arithmetic
plus: { clickhouseName: "plus", minArgs: 2, maxArgs: 2 },
minus: { clickhouseName: "minus", minArgs: 2, maxArgs: 2 },
multiply: { clickhouseName: "multiply", minArgs: 2, maxArgs: 2 },
divide: { clickhouseName: "divide", minArgs: 2, maxArgs: 2 },
intDiv: { clickhouseName: "intDiv", minArgs: 2, maxArgs: 2 },
intDivOrZero: { clickhouseName: "intDivOrZero", minArgs: 2, maxArgs: 2 },
modulo: { clickhouseName: "modulo", minArgs: 2, maxArgs: 2 },
moduloOrZero: { clickhouseName: "moduloOrZero", minArgs: 2, maxArgs: 2 },
positiveModulo: { clickhouseName: "positiveModulo", minArgs: 2, maxArgs: 2 },
negate: { clickhouseName: "negate", minArgs: 1, maxArgs: 1 },
abs: { clickhouseName: "abs", minArgs: 1, maxArgs: 1 },
gcd: { clickhouseName: "gcd", minArgs: 2, maxArgs: 2 },
lcm: { clickhouseName: "lcm", minArgs: 2, maxArgs: 2 },
// Mathematical
exp: { clickhouseName: "exp", minArgs: 1, maxArgs: 1 },
log: { clickhouseName: "log", minArgs: 1, maxArgs: 1 },
ln: { clickhouseName: "log", minArgs: 1, maxArgs: 1 },
exp2: { clickhouseName: "exp2", minArgs: 1, maxArgs: 1 },
log2: { clickhouseName: "log2", minArgs: 1, maxArgs: 1 },
exp10: { clickhouseName: "exp10", minArgs: 1, maxArgs: 1 },
log10: { clickhouseName: "log10", minArgs: 1, maxArgs: 1 },
sqrt: { clickhouseName: "sqrt", minArgs: 1, maxArgs: 1 },
cbrt: { clickhouseName: "cbrt", minArgs: 1, maxArgs: 1 },
erf: { clickhouseName: "erf", minArgs: 1, maxArgs: 1 },
erfc: { clickhouseName: "erfc", minArgs: 1, maxArgs: 1 },
lgamma: { clickhouseName: "lgamma", minArgs: 1, maxArgs: 1 },
tgamma: { clickhouseName: "tgamma", minArgs: 1, maxArgs: 1 },
sin: { clickhouseName: "sin", minArgs: 1, maxArgs: 1 },
cos: { clickhouseName: "cos", minArgs: 1, maxArgs: 1 },
tan: { clickhouseName: "tan", minArgs: 1, maxArgs: 1 },
asin: { clickhouseName: "asin", minArgs: 1, maxArgs: 1 },
acos: { clickhouseName: "acos", minArgs: 1, maxArgs: 1 },
atan: { clickhouseName: "atan", minArgs: 1, maxArgs: 1 },
pow: { clickhouseName: "pow", minArgs: 2, maxArgs: 2 },
power: { clickhouseName: "power", minArgs: 2, maxArgs: 2 },
round: { clickhouseName: "round", minArgs: 1, maxArgs: 2 },
floor: { clickhouseName: "floor", minArgs: 1, maxArgs: 2 },
ceil: { clickhouseName: "ceil", minArgs: 1, maxArgs: 2 },
ceiling: { clickhouseName: "ceiling", minArgs: 1, maxArgs: 2 },
trunc: { clickhouseName: "trunc", minArgs: 1, maxArgs: 2 },
truncate: { clickhouseName: "truncate", minArgs: 1, maxArgs: 2 },
sign: { clickhouseName: "sign", minArgs: 1, maxArgs: 1 },
// String functions
empty: { clickhouseName: "empty", minArgs: 1, maxArgs: 1 },
notEmpty: { clickhouseName: "notEmpty", minArgs: 1, maxArgs: 1 },
length: { clickhouseName: "length", minArgs: 1, maxArgs: 1 },
lengthUTF8: { clickhouseName: "lengthUTF8", minArgs: 1, maxArgs: 1 },
char_length: { clickhouseName: "char_length", minArgs: 1, maxArgs: 1 },
character_length: { clickhouseName: "character_length", minArgs: 1, maxArgs: 1 },
lower: { clickhouseName: "lower", minArgs: 1, maxArgs: 1 },
upper: { clickhouseName: "upper", minArgs: 1, maxArgs: 1 },
lowerUTF8: { clickhouseName: "lowerUTF8", minArgs: 1, maxArgs: 1 },
upperUTF8: { clickhouseName: "upperUTF8", minArgs: 1, maxArgs: 1 },
reverse: { clickhouseName: "reverse", minArgs: 1, maxArgs: 1 },
reverseUTF8: { clickhouseName: "reverseUTF8", minArgs: 1, maxArgs: 1 },
concat: { clickhouseName: "concat", minArgs: 1 },
concatAssumeInjective: { clickhouseName: "concatAssumeInjective", minArgs: 1 },
substring: { clickhouseName: "substring", minArgs: 2, maxArgs: 3 },
substr: { clickhouseName: "substring", minArgs: 2, maxArgs: 3 },
mid: { clickhouseName: "substring", minArgs: 2, maxArgs: 3 },
substringUTF8: { clickhouseName: "substringUTF8", minArgs: 2, maxArgs: 3 },
appendTrailingCharIfAbsent: {
clickhouseName: "appendTrailingCharIfAbsent",
minArgs: 2,
maxArgs: 2,
},
convertCharset: { clickhouseName: "convertCharset", minArgs: 3, maxArgs: 3 },
base58Encode: { clickhouseName: "base58Encode", minArgs: 1, maxArgs: 1 },
base58Decode: { clickhouseName: "base58Decode", minArgs: 1, maxArgs: 1 },
base64Encode: { clickhouseName: "base64Encode", minArgs: 1, maxArgs: 1 },
base64Decode: { clickhouseName: "base64Decode", minArgs: 1, maxArgs: 1 },
tryBase64Decode: { clickhouseName: "tryBase64Decode", minArgs: 1, maxArgs: 1 },
endsWith: { clickhouseName: "endsWith", minArgs: 2, maxArgs: 2 },
startsWith: { clickhouseName: "startsWith", minArgs: 2, maxArgs: 2 },
trim: { clickhouseName: "trim", minArgs: 1, maxArgs: 2 },
trimLeft: { clickhouseName: "trimLeft", minArgs: 1, maxArgs: 2 },
trimRight: { clickhouseName: "trimRight", minArgs: 1, maxArgs: 2 },
ltrim: { clickhouseName: "trimLeft", minArgs: 1, maxArgs: 1 },
rtrim: { clickhouseName: "trimRight", minArgs: 1, maxArgs: 1 },
leftPad: { clickhouseName: "leftPad", minArgs: 2, maxArgs: 3 },
rightPad: { clickhouseName: "rightPad", minArgs: 2, maxArgs: 3 },
leftPadUTF8: { clickhouseName: "leftPadUTF8", minArgs: 2, maxArgs: 3 },
rightPadUTF8: { clickhouseName: "rightPadUTF8", minArgs: 2, maxArgs: 3 },
left: { clickhouseName: "left", minArgs: 2, maxArgs: 2 },
right: { clickhouseName: "right", minArgs: 2, maxArgs: 2 },
repeat: { clickhouseName: "repeat", minArgs: 2, maxArgs: 2 },
space: { clickhouseName: "space", minArgs: 1, maxArgs: 1 },
replace: { clickhouseName: "replace", minArgs: 3, maxArgs: 3 },
replaceOne: { clickhouseName: "replaceOne", minArgs: 3, maxArgs: 3 },
replaceAll: { clickhouseName: "replaceAll", minArgs: 3, maxArgs: 3 },
replaceRegexpOne: { clickhouseName: "replaceRegexpOne", minArgs: 3, maxArgs: 3 },
replaceRegexpAll: { clickhouseName: "replaceRegexpAll", minArgs: 3, maxArgs: 3 },
position: { clickhouseName: "position", minArgs: 2, maxArgs: 2 },
positionCaseInsensitive: { clickhouseName: "positionCaseInsensitive", minArgs: 2, maxArgs: 2 },
positionUTF8: { clickhouseName: "positionUTF8", minArgs: 2, maxArgs: 2 },
positionCaseInsensitiveUTF8: {
clickhouseName: "positionCaseInsensitiveUTF8",
minArgs: 2,
maxArgs: 2,
},
locate: { clickhouseName: "locate", minArgs: 2, maxArgs: 2 },
match: { clickhouseName: "match", minArgs: 2, maxArgs: 2 },
multiMatchAny: { clickhouseName: "multiMatchAny", minArgs: 2, maxArgs: 2 },
multiMatchAnyIndex: { clickhouseName: "multiMatchAnyIndex", minArgs: 2, maxArgs: 2 },
multiMatchAllIndices: { clickhouseName: "multiMatchAllIndices", minArgs: 2, maxArgs: 2 },
multiSearchFirstPosition: { clickhouseName: "multiSearchFirstPosition", minArgs: 2, maxArgs: 2 },
multiSearchFirstIndex: { clickhouseName: "multiSearchFirstIndex", minArgs: 2, maxArgs: 2 },
multiSearchAny: { clickhouseName: "multiSearchAny", minArgs: 2, maxArgs: 2 },
extract: { clickhouseName: "extract", minArgs: 2, maxArgs: 2 },
extractAll: { clickhouseName: "extractAll", minArgs: 2, maxArgs: 2 },
extractAllGroupsHorizontal: {
clickhouseName: "extractAllGroupsHorizontal",
minArgs: 2,
maxArgs: 2,
},
extractAllGroupsVertical: { clickhouseName: "extractAllGroupsVertical", minArgs: 2, maxArgs: 2 },
like: { clickhouseName: "like", minArgs: 2, maxArgs: 2 },
ilike: { clickhouseName: "ilike", minArgs: 2, maxArgs: 2 },
notLike: { clickhouseName: "notLike", minArgs: 2, maxArgs: 2 },
notILike: { clickhouseName: "notILike", minArgs: 2, maxArgs: 2 },
splitByChar: { clickhouseName: "splitByChar", minArgs: 2, maxArgs: 3 },
splitByString: { clickhouseName: "splitByString", minArgs: 2, maxArgs: 3 },
splitByRegexp: { clickhouseName: "splitByRegexp", minArgs: 2, maxArgs: 3 },
arrayStringConcat: { clickhouseName: "arrayStringConcat", minArgs: 1, maxArgs: 2 },
format: { clickhouseName: "format", minArgs: 1 },
coalesce: { clickhouseName: "coalesce", minArgs: 1 },
ifNull: { clickhouseName: "ifNull", minArgs: 2, maxArgs: 2 },
nullIf: { clickhouseName: "nullIf", minArgs: 2, maxArgs: 2 },
assumeNotNull: { clickhouseName: "assumeNotNull", minArgs: 1, maxArgs: 1 },
toNullable: { clickhouseName: "toNullable", minArgs: 1, maxArgs: 1 },
isNull: { clickhouseName: "isNull", minArgs: 1, maxArgs: 1 },
isNotNull: { clickhouseName: "isNotNull", minArgs: 1, maxArgs: 1 },
// Type conversions
toString: { clickhouseName: "toString", minArgs: 1, maxArgs: 1 },
toFixedString: { clickhouseName: "toFixedString", minArgs: 2, maxArgs: 2 },
toUInt8: { clickhouseName: "toUInt8", minArgs: 1, maxArgs: 1 },
toUInt16: { clickhouseName: "toUInt16", minArgs: 1, maxArgs: 1 },
toUInt32: { clickhouseName: "toUInt32", minArgs: 1, maxArgs: 1 },
toUInt64: { clickhouseName: "toUInt64", minArgs: 1, maxArgs: 1 },
toInt8: { clickhouseName: "toInt8", minArgs: 1, maxArgs: 1 },
toInt16: { clickhouseName: "toInt16", minArgs: 1, maxArgs: 1 },
toInt32: { clickhouseName: "toInt32", minArgs: 1, maxArgs: 1 },
toInt64: { clickhouseName: "toInt64", minArgs: 1, maxArgs: 1 },
toInt128: { clickhouseName: "toInt128", minArgs: 1, maxArgs: 1 },
toInt256: { clickhouseName: "toInt256", minArgs: 1, maxArgs: 1 },
toUInt128: { clickhouseName: "toUInt128", minArgs: 1, maxArgs: 1 },
toUInt256: { clickhouseName: "toUInt256", minArgs: 1, maxArgs: 1 },
toFloat32: { clickhouseName: "toFloat32", minArgs: 1, maxArgs: 1 },
toFloat64: { clickhouseName: "toFloat64", minArgs: 1, maxArgs: 1 },
toDecimal32: { clickhouseName: "toDecimal32", minArgs: 2, maxArgs: 2 },
toDecimal64: { clickhouseName: "toDecimal64", minArgs: 2, maxArgs: 2 },
toDecimal128: { clickhouseName: "toDecimal128", minArgs: 2, maxArgs: 2 },
toDecimal256: { clickhouseName: "toDecimal256", minArgs: 2, maxArgs: 2 },
toDate: { clickhouseName: "toDate", minArgs: 1, maxArgs: 2 },
toDateOrNull: { clickhouseName: "toDateOrNull", minArgs: 1, maxArgs: 2 },
toDateOrZero: { clickhouseName: "toDateOrZero", minArgs: 1, maxArgs: 2 },
toDate32: { clickhouseName: "toDate32", minArgs: 1, maxArgs: 2 },
toDate32OrNull: { clickhouseName: "toDate32OrNull", minArgs: 1, maxArgs: 2 },
toDate32OrZero: { clickhouseName: "toDate32OrZero", minArgs: 1, maxArgs: 2 },
toDateTime: { clickhouseName: "toDateTime", minArgs: 1, maxArgs: 2 },
toDateTimeOrNull: { clickhouseName: "toDateTimeOrNull", minArgs: 1, maxArgs: 2 },
toDateTimeOrZero: { clickhouseName: "toDateTimeOrZero", minArgs: 1, maxArgs: 2 },
toDateTime64: { clickhouseName: "toDateTime64", minArgs: 1, maxArgs: 3 },
toDateTime64OrNull: { clickhouseName: "toDateTime64OrNull", minArgs: 1, maxArgs: 3 },
toDateTime64OrZero: { clickhouseName: "toDateTime64OrZero", minArgs: 1, maxArgs: 3 },
toUUID: { clickhouseName: "toUUID", minArgs: 1, maxArgs: 1 },
toUUIDOrNull: { clickhouseName: "toUUIDOrNull", minArgs: 1, maxArgs: 1 },
toUUIDOrZero: { clickhouseName: "toUUIDOrZero", minArgs: 1, maxArgs: 1 },
toTypeName: { clickhouseName: "toTypeName", minArgs: 1, maxArgs: 1 },
// Date/time functions
now: { clickhouseName: "now", minArgs: 0, maxArgs: 1, tzAware: true },
now64: { clickhouseName: "now64", minArgs: 0, maxArgs: 2, tzAware: true },
today: { clickhouseName: "today", minArgs: 0, maxArgs: 0 },
yesterday: { clickhouseName: "yesterday", minArgs: 0, maxArgs: 0 },
toYear: { clickhouseName: "toYear", minArgs: 1, maxArgs: 1 },
toQuarter: { clickhouseName: "toQuarter", minArgs: 1, maxArgs: 1 },
toMonth: { clickhouseName: "toMonth", minArgs: 1, maxArgs: 1 },
toDayOfYear: { clickhouseName: "toDayOfYear", minArgs: 1, maxArgs: 1 },
toDayOfMonth: { clickhouseName: "toDayOfMonth", minArgs: 1, maxArgs: 1 },
toDayOfWeek: { clickhouseName: "toDayOfWeek", minArgs: 1, maxArgs: 3 },
toHour: { clickhouseName: "toHour", minArgs: 1, maxArgs: 1 },
toMinute: { clickhouseName: "toMinute", minArgs: 1, maxArgs: 1 },
toSecond: { clickhouseName: "toSecond", minArgs: 1, maxArgs: 1 },
toUnixTimestamp: { clickhouseName: "toUnixTimestamp", minArgs: 1, maxArgs: 2 },
toStartOfYear: { clickhouseName: "toStartOfYear", minArgs: 1, maxArgs: 1 },
toStartOfQuarter: { clickhouseName: "toStartOfQuarter", minArgs: 1, maxArgs: 1 },
toStartOfMonth: { clickhouseName: "toStartOfMonth", minArgs: 1, maxArgs: 1 },
toMonday: { clickhouseName: "toMonday", minArgs: 1, maxArgs: 1 },
toStartOfWeek: { clickhouseName: "toStartOfWeek", minArgs: 1, maxArgs: 2 },
toStartOfDay: { clickhouseName: "toStartOfDay", minArgs: 1, maxArgs: 1 },
toStartOfHour: { clickhouseName: "toStartOfHour", minArgs: 1, maxArgs: 1 },
toStartOfMinute: { clickhouseName: "toStartOfMinute", minArgs: 1, maxArgs: 1 },
toStartOfSecond: { clickhouseName: "toStartOfSecond", minArgs: 1, maxArgs: 1 },
toStartOfFiveMinutes: { clickhouseName: "toStartOfFiveMinutes", minArgs: 1, maxArgs: 1 },
toStartOfTenMinutes: { clickhouseName: "toStartOfTenMinutes", minArgs: 1, maxArgs: 1 },
toStartOfFifteenMinutes: { clickhouseName: "toStartOfFifteenMinutes", minArgs: 1, maxArgs: 1 },
toStartOfInterval: { clickhouseName: "toStartOfInterval", minArgs: 2, maxArgs: 4 },
toTime: { clickhouseName: "toTime", minArgs: 1, maxArgs: 2 },
toISOYear: { clickhouseName: "toISOYear", minArgs: 1, maxArgs: 1 },
toISOWeek: { clickhouseName: "toISOWeek", minArgs: 1, maxArgs: 1 },
toWeek: { clickhouseName: "toWeek", minArgs: 1, maxArgs: 3 },
toYearWeek: { clickhouseName: "toYearWeek", minArgs: 1, maxArgs: 3 },
date_add: { clickhouseName: "date_add", minArgs: 3, maxArgs: 3 },
date_diff: { clickhouseName: "date_diff", minArgs: 3, maxArgs: 4 },
date_sub: { clickhouseName: "date_sub", minArgs: 3, maxArgs: 3 },
date_trunc: { clickhouseName: "date_trunc", minArgs: 2, maxArgs: 3 },
dateDiff: { clickhouseName: "dateDiff", minArgs: 3, maxArgs: 4 },
dateAdd: { clickhouseName: "dateAdd", minArgs: 3, maxArgs: 3 },
dateSub: { clickhouseName: "dateSub", minArgs: 3, maxArgs: 3 },
dateTrunc: { clickhouseName: "dateTrunc", minArgs: 2, maxArgs: 3 },
addSeconds: { clickhouseName: "addSeconds", minArgs: 2, maxArgs: 2 },
addMinutes: { clickhouseName: "addMinutes", minArgs: 2, maxArgs: 2 },
addHours: { clickhouseName: "addHours", minArgs: 2, maxArgs: 2 },
addDays: { clickhouseName: "addDays", minArgs: 2, maxArgs: 2 },
addWeeks: { clickhouseName: "addWeeks", minArgs: 2, maxArgs: 2 },
addMonths: { clickhouseName: "addMonths", minArgs: 2, maxArgs: 2 },
addQuarters: { clickhouseName: "addQuarters", minArgs: 2, maxArgs: 2 },
addYears: { clickhouseName: "addYears", minArgs: 2, maxArgs: 2 },
subtractSeconds: { clickhouseName: "subtractSeconds", minArgs: 2, maxArgs: 2 },
subtractMinutes: { clickhouseName: "subtractMinutes", minArgs: 2, maxArgs: 2 },
subtractHours: { clickhouseName: "subtractHours", minArgs: 2, maxArgs: 2 },
subtractDays: { clickhouseName: "subtractDays", minArgs: 2, maxArgs: 2 },
subtractWeeks: { clickhouseName: "subtractWeeks", minArgs: 2, maxArgs: 2 },
subtractMonths: { clickhouseName: "subtractMonths", minArgs: 2, maxArgs: 2 },
subtractQuarters: { clickhouseName: "subtractQuarters", minArgs: 2, maxArgs: 2 },
subtractYears: { clickhouseName: "subtractYears", minArgs: 2, maxArgs: 2 },
toTimeZone: { clickhouseName: "toTimeZone", minArgs: 2, maxArgs: 2 },
formatDateTime: { clickhouseName: "formatDateTime", minArgs: 2, maxArgs: 3 },
parseDateTime: { clickhouseName: "parseDateTime", minArgs: 2, maxArgs: 3 },
parseDateTimeBestEffort: {
clickhouseName: "parseDateTimeBestEffort",
minArgs: 1,
maxArgs: 2,
tzAware: true,
},
parseDateTimeBestEffortOrNull: {
clickhouseName: "parseDateTimeBestEffortOrNull",
minArgs: 1,
maxArgs: 2,
tzAware: true,
},
parseDateTimeBestEffortOrZero: {
clickhouseName: "parseDateTimeBestEffortOrZero",
minArgs: 1,
maxArgs: 2,
tzAware: true,
},
parseDateTime64BestEffort: {
clickhouseName: "parseDateTime64BestEffort",
minArgs: 1,
maxArgs: 3,
tzAware: true,
},
parseDateTime64BestEffortOrNull: {
clickhouseName: "parseDateTime64BestEffortOrNull",
minArgs: 1,
maxArgs: 3,
tzAware: true,
},
parseDateTime64BestEffortOrZero: {
clickhouseName: "parseDateTime64BestEffortOrZero",
minArgs: 1,
maxArgs: 3,
tzAware: true,
},
// Interval functions
toIntervalSecond: { clickhouseName: "toIntervalSecond", minArgs: 1, maxArgs: 1 },
toIntervalMinute: { clickhouseName: "toIntervalMinute", minArgs: 1, maxArgs: 1 },
toIntervalHour: { clickhouseName: "toIntervalHour", minArgs: 1, maxArgs: 1 },
toIntervalDay: { clickhouseName: "toIntervalDay", minArgs: 1, maxArgs: 1 },
toIntervalWeek: { clickhouseName: "toIntervalWeek", minArgs: 1, maxArgs: 1 },
toIntervalMonth: { clickhouseName: "toIntervalMonth", minArgs: 1, maxArgs: 1 },
toIntervalQuarter: { clickhouseName: "toIntervalQuarter", minArgs: 1, maxArgs: 1 },
toIntervalYear: { clickhouseName: "toIntervalYear", minArgs: 1, maxArgs: 1 },
// Array functions
array: { clickhouseName: "array", minArgs: 0 },
range: { clickhouseName: "range", minArgs: 1, maxArgs: 3 },
arrayElement: { clickhouseName: "arrayElement", minArgs: 2, maxArgs: 2 },
has: { clickhouseName: "has", minArgs: 2, maxArgs: 2 },
hasAll: { clickhouseName: "hasAll", minArgs: 2, maxArgs: 2 },
hasAny: { clickhouseName: "hasAny", minArgs: 2, maxArgs: 2 },
hasSubstr: { clickhouseName: "hasSubstr", minArgs: 2, maxArgs: 2 },
indexOf: { clickhouseName: "indexOf", minArgs: 2, maxArgs: 2 },
arrayCount: { clickhouseName: "arrayCount", minArgs: 1, maxArgs: 2 },
countEqual: { clickhouseName: "countEqual", minArgs: 2, maxArgs: 2 },
arrayEnumerate: { clickhouseName: "arrayEnumerate", minArgs: 1, maxArgs: 1 },
arrayEnumerateDense: { clickhouseName: "arrayEnumerateDense", minArgs: 1 },
arrayEnumerateUniq: { clickhouseName: "arrayEnumerateUniq", minArgs: 1 },
arrayEnumerateUniqRanked: { clickhouseName: "arrayEnumerateUniqRanked", minArgs: 1 },
arrayPopBack: { clickhouseName: "arrayPopBack", minArgs: 1, maxArgs: 1 },
arrayPopFront: { clickhouseName: "arrayPopFront", minArgs: 1, maxArgs: 1 },
arrayPushBack: { clickhouseName: "arrayPushBack", minArgs: 2, maxArgs: 2 },
arrayPushFront: { clickhouseName: "arrayPushFront", minArgs: 2, maxArgs: 2 },
arrayResize: { clickhouseName: "arrayResize", minArgs: 2, maxArgs: 3 },
arraySlice: { clickhouseName: "arraySlice", minArgs: 2, maxArgs: 3 },
arraySort: { clickhouseName: "arraySort", minArgs: 1, maxArgs: 2 },
arrayPartialSort: { clickhouseName: "arrayPartialSort", minArgs: 2, maxArgs: 3 },
arrayReverseSort: { clickhouseName: "arrayReverseSort", minArgs: 1, maxArgs: 2 },
arrayPartialReverseSort: { clickhouseName: "arrayPartialReverseSort", minArgs: 2, maxArgs: 3 },
arrayShuffle: { clickhouseName: "arrayShuffle", minArgs: 1, maxArgs: 2 },
arrayUniq: { clickhouseName: "arrayUniq", minArgs: 1 },
arrayJoin: { clickhouseName: "arrayJoin", minArgs: 1, maxArgs: 1 },
arrayDifference: { clickhouseName: "arrayDifference", minArgs: 1, maxArgs: 1 },
arrayDistinct: { clickhouseName: "arrayDistinct", minArgs: 1, maxArgs: 1 },
arrayIntersect: { clickhouseName: "arrayIntersect", minArgs: 1 },
arrayReduce: { clickhouseName: "arrayReduce", minArgs: 2 },
arrayReverse: { clickhouseName: "arrayReverse", minArgs: 1, maxArgs: 1 },
arrayFlatten: { clickhouseName: "arrayFlatten", minArgs: 1, maxArgs: 1 },
arrayCompact: { clickhouseName: "arrayCompact", minArgs: 1, maxArgs: 1 },
arrayZip: { clickhouseName: "arrayZip", minArgs: 1 },
arrayMin: { clickhouseName: "arrayMin", minArgs: 1, maxArgs: 2 },
arrayMax: { clickhouseName: "arrayMax", minArgs: 1, maxArgs: 2 },
arraySum: { clickhouseName: "arraySum", minArgs: 1, maxArgs: 2 },
arrayAvg: { clickhouseName: "arrayAvg", minArgs: 1, maxArgs: 2 },
arrayCumSum: { clickhouseName: "arrayCumSum", minArgs: 1, maxArgs: 2 },
arrayCumSumNonNegative: { clickhouseName: "arrayCumSumNonNegative", minArgs: 1, maxArgs: 2 },
arrayProduct: { clickhouseName: "arrayProduct", minArgs: 1, maxArgs: 1 },
// JSON functions
JSONHas: { clickhouseName: "JSONHas", minArgs: 1 },
JSONLength: { clickhouseName: "JSONLength", minArgs: 1 },
JSONType: { clickhouseName: "JSONType", minArgs: 1 },
JSONExtractUInt: { clickhouseName: "JSONExtractUInt", minArgs: 1 },
JSONExtractInt: { clickhouseName: "JSONExtractInt", minArgs: 1 },
JSONExtractFloat: { clickhouseName: "JSONExtractFloat", minArgs: 1 },
JSONExtractBool: { clickhouseName: "JSONExtractBool", minArgs: 1 },
JSONExtractString: { clickhouseName: "JSONExtractString", minArgs: 1 },
JSONExtract: { clickhouseName: "JSONExtract", minArgs: 2 },
JSONExtractRaw: { clickhouseName: "JSONExtractRaw", minArgs: 1 },
JSONExtractArrayRaw: { clickhouseName: "JSONExtractArrayRaw", minArgs: 1 },
JSONExtractKeysAndValues: { clickhouseName: "JSONExtractKeysAndValues", minArgs: 2, maxArgs: 2 },
JSONExtractKeys: { clickhouseName: "JSONExtractKeys", minArgs: 1 },
toJSONString: { clickhouseName: "toJSONString", minArgs: 1, maxArgs: 1 },
// Tuple functions
tuple: { clickhouseName: "tuple", minArgs: 0 },
tupleElement: { clickhouseName: "tupleElement", minArgs: 2, maxArgs: 3 },
untuple: { clickhouseName: "untuple", minArgs: 1, maxArgs: 1 },
// Map functions
map: { clickhouseName: "map", minArgs: 0 },
mapFromArrays: { clickhouseName: "mapFromArrays", minArgs: 2, maxArgs: 2 },
mapContains: { clickhouseName: "mapContains", minArgs: 2, maxArgs: 2 },
mapKeys: { clickhouseName: "mapKeys", minArgs: 1, maxArgs: 1 },
mapValues: { clickhouseName: "mapValues", minArgs: 1, maxArgs: 1 },
// Hash functions
MD5: { clickhouseName: "MD5", minArgs: 1, maxArgs: 1 },
SHA1: { clickhouseName: "SHA1", minArgs: 1, maxArgs: 1 },
SHA224: { clickhouseName: "SHA224", minArgs: 1, maxArgs: 1 },
SHA256: { clickhouseName: "SHA256", minArgs: 1, maxArgs: 1 },
SHA384: { clickhouseName: "SHA384", minArgs: 1, maxArgs: 1 },
SHA512: { clickhouseName: "SHA512", minArgs: 1, maxArgs: 1 },
sipHash64: { clickhouseName: "sipHash64", minArgs: 1 },
sipHash128: { clickhouseName: "sipHash128", minArgs: 1 },
cityHash64: { clickhouseName: "cityHash64", minArgs: 1 },
intHash32: { clickhouseName: "intHash32", minArgs: 1, maxArgs: 1 },
intHash64: { clickhouseName: "intHash64", minArgs: 1, maxArgs: 1 },
farmHash64: { clickhouseName: "farmHash64", minArgs: 1 },
farmFingerprint64: { clickhouseName: "farmFingerprint64", minArgs: 1 },
xxHash32: { clickhouseName: "xxHash32", minArgs: 1 },
xxHash64: { clickhouseName: "xxHash64", minArgs: 1 },
murmurHash2_32: { clickhouseName: "murmurHash2_32", minArgs: 1 },
murmurHash2_64: { clickhouseName: "murmurHash2_64", minArgs: 1 },
murmurHash3_32: { clickhouseName: "murmurHash3_32", minArgs: 1 },
murmurHash3_64: { clickhouseName: "murmurHash3_64", minArgs: 1 },
murmurHash3_128: { clickhouseName: "murmurHash3_128", minArgs: 1 },
hex: { clickhouseName: "hex", minArgs: 1, maxArgs: 1 },
unhex: { clickhouseName: "unhex", minArgs: 1, maxArgs: 1 },
// URL functions
protocol: { clickhouseName: "protocol", minArgs: 1, maxArgs: 1 },
domain: { clickhouseName: "domain", minArgs: 1, maxArgs: 1 },
domainWithoutWWW: { clickhouseName: "domainWithoutWWW", minArgs: 1, maxArgs: 1 },
topLevelDomain: { clickhouseName: "topLevelDomain", minArgs: 1, maxArgs: 1 },
firstSignificantSubdomain: {
clickhouseName: "firstSignificantSubdomain",
minArgs: 1,
maxArgs: 1,
},
cutToFirstSignificantSubdomain: {
clickhouseName: "cutToFirstSignificantSubdomain",
minArgs: 1,
maxArgs: 1,
},
cutToFirstSignificantSubdomainWithWWW: {
clickhouseName: "cutToFirstSignificantSubdomainWithWWW",
minArgs: 1,
maxArgs: 1,
},
port: { clickhouseName: "port", minArgs: 1, maxArgs: 2 },
path: { clickhouseName: "path", minArgs: 1, maxArgs: 1 },
pathFull: { clickhouseName: "pathFull", minArgs: 1, maxArgs: 1 },
queryString: { clickhouseName: "queryString", minArgs: 1, maxArgs: 1 },
fragment: { clickhouseName: "fragment", minArgs: 1, maxArgs: 1 },
extractURLParameter: { clickhouseName: "extractURLParameter", minArgs: 2, maxArgs: 2 },
extractURLParameters: { clickhouseName: "extractURLParameters", minArgs: 1, maxArgs: 1 },
encodeURLComponent: { clickhouseName: "encodeURLComponent", minArgs: 1, maxArgs: 1 },
decodeURLComponent: { clickhouseName: "decodeURLComponent", minArgs: 1, maxArgs: 1 },
// UUID functions
generateUUIDv4: { clickhouseName: "generateUUIDv4", minArgs: 0, maxArgs: 0 },
UUIDStringToNum: { clickhouseName: "UUIDStringToNum", minArgs: 1, maxArgs: 1 },
UUIDNumToString: { clickhouseName: "UUIDNumToString", minArgs: 1, maxArgs: 1 },
// Other functions
isFinite: { clickhouseName: "isFinite", minArgs: 1, maxArgs: 1 },
isInfinite: { clickhouseName: "isInfinite", minArgs: 1, maxArgs: 1 },
ifNotFinite: { clickhouseName: "ifNotFinite", minArgs: 2, maxArgs: 2 },
isNaN: { clickhouseName: "isNaN", minArgs: 1, maxArgs: 1 },
bar: { clickhouseName: "bar", minArgs: 4, maxArgs: 4 },
transform: { clickhouseName: "transform", minArgs: 3, maxArgs: 4 },
formatReadableDecimalSize: {
clickhouseName: "formatReadableDecimalSize",
minArgs: 1,
maxArgs: 1,
},
formatReadableSize: { clickhouseName: "formatReadableSize", minArgs: 1, maxArgs: 1 },
formatReadableQuantity: { clickhouseName: "formatReadableQuantity", minArgs: 1, maxArgs: 1 },
formatReadableTimeDelta: { clickhouseName: "formatReadableTimeDelta", minArgs: 1, maxArgs: 2 },
least: { clickhouseName: "least", minArgs: 2, maxArgs: 2, caseSensitive: false },
greatest: { clickhouseName: "greatest", minArgs: 2, maxArgs: 2, caseSensitive: false },
min2: { clickhouseName: "min2", minArgs: 2, maxArgs: 2 },
max2: { clickhouseName: "max2", minArgs: 2, maxArgs: 2 },
runningDifference: { clickhouseName: "runningDifference", minArgs: 1, maxArgs: 1 },
runningDifferenceStartingWithFirstValue: {
clickhouseName: "runningDifferenceStartingWithFirstValue",
minArgs: 1,
maxArgs: 1,
},
neighbor: { clickhouseName: "neighbor", minArgs: 2, maxArgs: 3 },
// Window functions
rank: { clickhouseName: "rank", minArgs: 0, maxArgs: 0 },
dense_rank: { clickhouseName: "dense_rank", minArgs: 0, maxArgs: 0 },
row_number: { clickhouseName: "row_number", minArgs: 0, maxArgs: 0 },
first_value: { clickhouseName: "first_value", minArgs: 1, maxArgs: 1 },
last_value: { clickhouseName: "last_value", minArgs: 1, maxArgs: 1 },
nth_value: { clickhouseName: "nth_value", minArgs: 2, maxArgs: 2 },
lagInFrame: { clickhouseName: "lagInFrame", minArgs: 1, maxArgs: 3 },
leadInFrame: { clickhouseName: "leadInFrame", minArgs: 1, maxArgs: 3 },
lag: { clickhouseName: "lagInFrame", minArgs: 1, maxArgs: 3 },
lead: { clickhouseName: "leadInFrame", minArgs: 1, maxArgs: 3 },
};
/**
* Aggregate functions available in TSQL
* Port of HOGQL_AGGREGATIONS from aggregations.py
*/
export const TSQL_AGGREGATIONS: Record<string, TSQLFunctionMeta> = {
// Standard aggregate functions
count: { clickhouseName: "count", minArgs: 0, maxArgs: 1, aggregate: true, caseSensitive: false },
countIf: { clickhouseName: "countIf", minArgs: 1, maxArgs: 2, aggregate: true },
countDistinct: { clickhouseName: "countDistinct", minArgs: 1, maxArgs: 1, aggregate: true },
countDistinctIf: { clickhouseName: "countDistinctIf", minArgs: 1, maxArgs: 2, aggregate: true },
min: { clickhouseName: "min", minArgs: 1, maxArgs: 1, aggregate: true, caseSensitive: false },
minIf: { clickhouseName: "minIf", minArgs: 2, maxArgs: 2, aggregate: true },
max: { clickhouseName: "max", minArgs: 1, maxArgs: 1, aggregate: true, caseSensitive: false },
maxIf: { clickhouseName: "maxIf", minArgs: 2, maxArgs: 2, aggregate: true },
sum: { clickhouseName: "sum", minArgs: 1, maxArgs: 1, aggregate: true, caseSensitive: false },
sumIf: { clickhouseName: "sumIf", minArgs: 2, maxArgs: 2, aggregate: true },
avg: { clickhouseName: "avg", minArgs: 1, maxArgs: 1, aggregate: true, caseSensitive: false },
avgIf: { clickhouseName: "avgIf", minArgs: 2, maxArgs: 2, aggregate: true },
any: { clickhouseName: "any", minArgs: 1, maxArgs: 1, aggregate: true },
anyIf: { clickhouseName: "anyIf", minArgs: 2, maxArgs: 2, aggregate: true },
anyLast: { clickhouseName: "anyLast", minArgs: 1, maxArgs: 1, aggregate: true },
anyLastIf: { clickhouseName: "anyLastIf", minArgs: 2, maxArgs: 2, aggregate: true },
anyHeavy: { clickhouseName: "anyHeavy", minArgs: 1, maxArgs: 1, aggregate: true },
anyHeavyIf: { clickhouseName: "anyHeavyIf", minArgs: 2, maxArgs: 2, aggregate: true },
argMin: { clickhouseName: "argMin", minArgs: 2, maxArgs: 2, aggregate: true },
argMinIf: { clickhouseName: "argMinIf", minArgs: 3, maxArgs: 3, aggregate: true },
argMax: { clickhouseName: "argMax", minArgs: 2, maxArgs: 2, aggregate: true },
argMaxIf: { clickhouseName: "argMaxIf", minArgs: 3, maxArgs: 3, aggregate: true },
stddevPop: { clickhouseName: "stddevPop", minArgs: 1, maxArgs: 1, aggregate: true },
stddevSamp: { clickhouseName: "stddevSamp", minArgs: 1, maxArgs: 1, aggregate: true },
varPop: { clickhouseName: "varPop", minArgs: 1, maxArgs: 1, aggregate: true },
varSamp: { clickhouseName: "varSamp", minArgs: 1, maxArgs: 1, aggregate: true },
covarPop: { clickhouseName: "covarPop", minArgs: 2, maxArgs: 2, aggregate: true },
covarSamp: { clickhouseName: "covarSamp", minArgs: 2, maxArgs: 2, aggregate: true },
corr: { clickhouseName: "corr", minArgs: 2, maxArgs: 2, aggregate: true },
// Array aggregations
groupArray: { clickhouseName: "groupArray", minArgs: 1, maxArgs: 1, aggregate: true },
groupArrayIf: { clickhouseName: "groupArrayIf", minArgs: 2, maxArgs: 2, aggregate: true },
groupUniqArray: { clickhouseName: "groupUniqArray", minArgs: 1, maxArgs: 1, aggregate: true },
groupUniqArrayIf: { clickhouseName: "groupUniqArrayIf", minArgs: 2, maxArgs: 2, aggregate: true },
groupArrayInsertAt: {
clickhouseName: "groupArrayInsertAt",
minArgs: 2,
maxArgs: 2,
aggregate: true,
},
groupArrayMovingAvg: {
clickhouseName: "groupArrayMovingAvg",
minArgs: 1,
maxArgs: 1,
aggregate: true,
},
groupArrayMovingSum: {
clickhouseName: "groupArrayMovingSum",
minArgs: 1,
maxArgs: 1,
aggregate: true,
},
groupArraySample: {
clickhouseName: "groupArraySample",
minArgs: 1,
maxArgs: 1,
minParams: 1,
maxParams: 2,
aggregate: true,
},
array_agg: { clickhouseName: "groupArray", minArgs: 1, maxArgs: 1, aggregate: true },
// Bitmap aggregations
groupBitmap: { clickhouseName: "groupBitmap", minArgs: 1, maxArgs: 1, aggregate: true },
groupBitmapAnd: { clickhouseName: "groupBitmapAnd", minArgs: 1, maxArgs: 1, aggregate: true },
groupBitmapOr: { clickhouseName: "groupBitmapOr", minArgs: 1, maxArgs: 1, aggregate: true },
groupBitmapXor: { clickhouseName: "groupBitmapXor", minArgs: 1, maxArgs: 1, aggregate: true },
// Uniq functions
uniq: { clickhouseName: "uniq", minArgs: 1, aggregate: true },
uniqIf: { clickhouseName: "uniqIf", minArgs: 2, aggregate: true },
uniqExact: { clickhouseName: "uniqExact", minArgs: 1, aggregate: true },
uniqExactIf: { clickhouseName: "uniqExactIf", minArgs: 2, aggregate: true },
uniqHLL12: { clickhouseName: "uniqHLL12", minArgs: 1, aggregate: true },
uniqTheta: { clickhouseName: "uniqTheta", minArgs: 1, aggregate: true },
// Quantile functions
median: { clickhouseName: "median", minArgs: 1, maxArgs: 1, aggregate: true },
medianIf: { clickhouseName: "medianIf", minArgs: 2, maxArgs: 2, aggregate: true },
medianExact: { clickhouseName: "medianExact", minArgs: 1, maxArgs: 1, aggregate: true },
quantile: {
clickhouseName: "quantile",
minArgs: 1,
maxArgs: 1,
minParams: 1,
maxParams: 1,
aggregate: true,
},
quantileIf: {
clickhouseName: "quantileIf",
minArgs: 2,
maxArgs: 2,
minParams: 1,
maxParams: 1,
aggregate: true,
},
quantiles: { clickhouseName: "quantiles", minArgs: 1, aggregate: true },
// -Merge combinators for AggregatingMergeTree tables
quantilesMerge: {
clickhouseName: "quantilesMerge",
minArgs: 1,
maxArgs: 1,
minParams: 1,
aggregate: true,
},
quantileMerge: {
clickhouseName: "quantileMerge",
minArgs: 1,
maxArgs: 1,
minParams: 1,
maxParams: 1,
aggregate: true,
},
sumMerge: { clickhouseName: "sumMerge", minArgs: 1, maxArgs: 1, aggregate: true },
avgMerge: { clickhouseName: "avgMerge", minArgs: 1, maxArgs: 1, aggregate: true },
countMerge: { clickhouseName: "countMerge", minArgs: 1, maxArgs: 1, aggregate: true },
minMerge: { clickhouseName: "minMerge", minArgs: 1, maxArgs: 1, aggregate: true },
maxMerge: { clickhouseName: "maxMerge", minArgs: 1, maxArgs: 1, aggregate: true },
// Statistical functions
simpleLinearRegression: {
clickhouseName: "simpleLinearRegression",
minArgs: 2,
maxArgs: 2,
aggregate: true,
},
contingency: { clickhouseName: "contingency", minArgs: 2, maxArgs: 2, aggregate: true },
cramersV: { clickhouseName: "cramersV", minArgs: 2, maxArgs: 2, aggregate: true },
theilsU: { clickhouseName: "theilsU", minArgs: 2, maxArgs: 2, aggregate: true },
// Sum/Map variants
sumMap: { clickhouseName: "sumMap", minArgs: 1, maxArgs: 2, aggregate: true },
minMap: { clickhouseName: "minMap", minArgs: 1, maxArgs: 2, aggregate: true },
maxMap: { clickhouseName: "maxMap", minArgs: 1, maxArgs: 2, aggregate: true },
// TopK
topK: {
clickhouseName: "topK",
minArgs: 1,
maxArgs: 1,
minParams: 1,
maxParams: 1,
aggregate: true,
},
// Funnel
windowFunnel: { clickhouseName: "windowFunnel", minArgs: 1, maxArgs: 99, aggregate: true },
};
/**
* Build a lowercase lookup map from a functions record.
* Uses a null-prototype object to avoid Object.prototype pollution (e.g. "toString").
*/
function buildLowercaseMap(
functions: Record<string, TSQLFunctionMeta>
): Record<string, TSQLFunctionMeta> {
const map: Record<string, TSQLFunctionMeta> = Object.create(null);
for (const [key, value] of Object.entries(functions)) {
map[key.toLowerCase()] = value;
}
return map;
}
const FUNCTIONS_LOWERCASE = buildLowercaseMap(TSQL_CLICKHOUSE_FUNCTIONS);
const AGGREGATIONS_LOWERCASE = buildLowercaseMap(TSQL_AGGREGATIONS);
/**
* Find a function in the TSQL functions map.
* Supports case-insensitive lookup for non-case-sensitive functions.
*
* @param functions - The canonical functions record (exact-match lookup)
* @param lowercaseMap - Pre-computed lowercase lookup (null-prototype, safe from prototype pollution)
*/
function findFunction(
name: string,
functions: Record<string, TSQLFunctionMeta>,
lowercaseMap: Record<string, TSQLFunctionMeta>
): TSQLFunctionMeta | undefined {
if (Object.prototype.hasOwnProperty.call(functions, name)) {
return functions[name];
}
// Case-insensitive fallback using the pre-computed lowercase map
const lowerFunc = lowercaseMap[name.toLowerCase()];
if (lowerFunc === undefined) {
return undefined;
}
// If the function is case-sensitive, only the exact-match above should find it
if (lowerFunc.caseSensitive) {
return undefined;
}
return lowerFunc;
}
/**
* Find a TSQL aggregation function by name
*/
export function findTSQLAggregation(name: string): TSQLFunctionMeta | undefined {
return findFunction(name, TSQL_AGGREGATIONS, AGGREGATIONS_LOWERCASE);
}
/**
* Find a TSQL function by name
*/
export function findTSQLFunction(name: string): TSQLFunctionMeta | undefined {
return findFunction(name, TSQL_CLICKHOUSE_FUNCTIONS, FUNCTIONS_LOWERCASE);
}
/**
* Get all exposed function names (for autocomplete, suggestions, etc.)
*/
export function getAllExposedFunctionNames(): string[] {
const functionNames = Object.keys(TSQL_CLICKHOUSE_FUNCTIONS).filter(
(name) => !name.startsWith("_")
);
const aggregationNames = Object.keys(TSQL_AGGREGATIONS).filter((name) => !name.startsWith("_"));
return [...functionNames, ...aggregationNames];
}
/**
* Validate function arguments
*/
export function validateFunctionArgs(
args: unknown[],
minArgs: number,
maxArgs: number | undefined,
functionName: string,
options: {
functionTerm?: string;
argumentTerm?: string;
} = {}
): void {
const { functionTerm = "function", argumentTerm = "argument" } = options;
const tooFew = args.length < minArgs;
const tooMany = maxArgs !== undefined && args.length > maxArgs;
if (minArgs === maxArgs && (tooFew || tooMany)) {
throw new Error(
`${functionTerm.charAt(0).toUpperCase() + functionTerm.slice(1)} '${functionName}' expects ${minArgs} ${argumentTerm}${minArgs !== 1 ? "s" : ""}, found ${args.length}`
);
}
if (tooFew) {
throw new Error(
`${functionTerm.charAt(0).toUpperCase() + functionTerm.slice(1)} '${functionName}' expects at least ${minArgs} ${argumentTerm}${minArgs !== 1 ? "s" : ""}, found ${args.length}`
);
}
if (tooMany) {
throw new Error(
`${functionTerm.charAt(0).toUpperCase() + functionTerm.slice(1)} '${functionName}' expects at most ${maxArgs} ${argumentTerm}${maxArgs !== 1 ? "s" : ""}, found ${args.length}`
);
}
}
internal-packages/tsql/src/query/models.ts
+253
View File
@@ -0,0 +1,253 @@
// TypeScript translation of posthog/hogql/database/models.py
import type { Expr, ConstantType } from "./ast";
import type { TSQLContext } from "./context";
export interface FieldOrTable {
hidden?: boolean;
}
export interface DatabaseField extends FieldOrTable {
name: string;
array?: boolean;
nullable?: boolean;
is_nullable?(): boolean;
get_constant_type?(): ConstantType;
default_value?(): any;
}
export interface IntegerDatabaseField extends DatabaseField {}
export interface FloatDatabaseField extends DatabaseField {}
export interface DecimalDatabaseField extends DatabaseField {}
export interface StringDatabaseField extends DatabaseField {}
export interface UnknownDatabaseField extends DatabaseField {}
export interface StringJSONDatabaseField extends DatabaseField {}
export interface StringArrayDatabaseField extends DatabaseField {}
export interface FloatArrayDatabaseField extends DatabaseField {}
export interface DateDatabaseField extends DatabaseField {}
export interface DateTimeDatabaseField extends DatabaseField {}
export interface BooleanDatabaseField extends DatabaseField {}
export interface UUIDDatabaseField extends DatabaseField {}
export interface ExpressionField extends DatabaseField {
expr: Expr;
isolate_scope?: boolean;
}
export interface FieldTraverser extends FieldOrTable {
chain: Array<string | number>;
}
export interface Table extends FieldOrTable {
fields: Record<string, FieldOrTable>;
has_field?(name: string | number): boolean;
get_field?(name: string | number): FieldOrTable;
to_printed_clickhouse?(context: TSQLContext): string;
to_printed_tsql?(): string;
avoid_asterisk_fields?(): string[];
get_asterisk?(): Record<string, FieldOrTable>;
}
export interface LazyJoin extends FieldOrTable {
join_function?(from_table: Table, to_table: Table, requesting_table: Table): Expr;
resolve_table?(context: TSQLContext): Table;
}
export interface LazyTable extends Table {}
export interface VirtualTable extends Table {}
export interface SavedQuery extends Table {
query: Expr;
}
export interface FunctionCallTable extends Table {
call_function?(context: TSQLContext): Expr;
}
export interface TableNode {
name: "root" | string;
table?: FieldOrTable | null;
children: Record<string, TableNode>;
get?(): FieldOrTable;
has_child?(path: string[]): boolean;
get_child?(path: string[]): TableNode;
add_child?(
child: TableNode,
options?: {
table_conflict_mode?: "override" | "ignore";
children_conflict_mode?: "override" | "merge" | "ignore";
}
): void;
merge_with?(
other: TableNode,
options?: {
table_conflict_mode?: "override" | "ignore";
children_conflict_mode?: "override" | "merge" | "ignore";
}
): void;
resolve_all_table_names?(): string[];
}
// Basic TableNode implementation class
export class TableNodeImpl implements TableNode {
name: "root" | string;
table?: FieldOrTable | null;
children: Record<string, TableNode>;
constructor(name: "root" | string = "root", table?: FieldOrTable | null) {
this.name = name;
this.table = table || null;
this.children = {};
}
get(): FieldOrTable {
if (this.table === null || this.table === undefined) {
throw new Error(`Table is not set at \`${this.name}\``);
}
return this.table;
}
has_child(path: string[]): boolean {
if (path.length === 0) {
return this.table !== null && this.table !== undefined;
}
const [first, ...restOfPath] = path;
if (!(first in this.children)) {
return false;
}
return this.children[first].has_child ? this.children[first].has_child!(restOfPath) : false;
}
get_child(path: string[]): TableNode {
if (path.length === 0) {
return this;
}
const [first, ...restOfPath] = path;
if (!(first in this.children)) {
throw new Error(`Unknown child \`${first}\` at \`${this.name}\`.`);
}
return this.children[first].get_child
? this.children[first].get_child!(restOfPath)
: this.children[first];
}
add_child(
child: TableNode,
options?: {
table_conflict_mode?: "override" | "ignore";
children_conflict_mode?: "override" | "merge" | "ignore";
}
): void {
const tableConflictMode = options?.table_conflict_mode || "ignore";
const childrenConflictMode = options?.children_conflict_mode || "merge";
if (child.name in this.children) {
if (childrenConflictMode === "override") {
this.children[child.name] = child;
} else if (childrenConflictMode === "merge") {
const existing = this.children[child.name];
if (existing.merge_with) {
existing.merge_with(child, {
table_conflict_mode: tableConflictMode,
children_conflict_mode: childrenConflictMode,
});
}
}
// ignore mode: do nothing
return;
}
this.children[child.name] = child;
}
merge_with(
other: TableNode,
options?: {
table_conflict_mode?: "override" | "ignore";
children_conflict_mode?: "override" | "merge" | "ignore";
}
): void {
const tableConflictMode = options?.table_conflict_mode || "ignore";
const childrenConflictMode = options?.children_conflict_mode || "merge";
if (other.table !== null && other.table !== undefined) {
if (this.table === null || this.table === undefined) {
this.table = other.table;
} else {
// Conflict - check conflict mode
if (tableConflictMode === "override") {
this.table = other.table;
}
// ignore mode: do nothing
}
}
for (const child of Object.values(other.children)) {
this.add_child(child, {
table_conflict_mode: tableConflictMode,
children_conflict_mode: childrenConflictMode,
});
}
}
resolve_all_table_names(): string[] {
const names: string[] = [];
if (this.table !== null && this.table !== undefined) {
names.push(this.name);
}
for (const child of Object.values(this.children)) {
const childNames = child.resolve_all_table_names ? child.resolve_all_table_names() : [];
// The root node should NOT include itself in the names
if (this.name === "root") {
names.push(...childNames);
} else {
names.push(...childNames.map((x) => `${this.name}.${x}`));
}
}
return names;
}
static createNestedForChain(chain: string[], table: Table): TableNode {
if (chain.length === 0) {
throw new Error("Chain must have at least one element");
}
const start = new TableNodeImpl(chain[0]);
let current: TableNode = start;
for (let i = 1; i < chain.length; i++) {
const child = new TableNodeImpl(chain[i]);
if (current.add_child) {
current.add_child(child);
} else {
current.children[child.name] = child;
}
current = child;
}
current.table = table;
return start;
}
}
export interface LazyTableToAdd {
lazy_table: LazyTable;
fields_accessed: Record<string, Array<string | number>>;
}
export interface LazyJoinToAdd {
from_table: string;
to_table: string;
lazy_join: LazyJoin;
lazy_join_type: any; // LazyJoinType from ast.ts
fields_accessed: Record<string, Array<string | number>>;
}
internal-packages/tsql/src/query/parse_string.ts
+69
View File
@@ -0,0 +1,69 @@
// TypeScript translation of posthog/hogql/parse_string.py
// Keep this file in sync with the Python version
import { SyntaxError } from "./errors";
function replaceCommonEscapeCharacters(text: string): string {
// copied from clickhouse_driver/util/escape.py
// Note: \a (bell) and \v (vertical tab) are not directly supported in JavaScript strings
// but we handle them as escape sequences that get replaced
text = text.replace(/\\b/g, "\b");
text = text.replace(/\\f/g, "\f");
text = text.replace(/\\r/g, "\r");
text = text.replace(/\\n/g, "\n");
text = text.replace(/\\t/g, "\t");
text = text.replace(/\\0/g, ""); // NUL characters are ignored
text = text.replace(/\\a/g, "\x07"); // Bell character (ASCII 7)
text = text.replace(/\\v/g, "\x0B"); // Vertical tab (ASCII 11)
text = text.replace(/\\\\/g, "\\");
return text;
}
export function parseStringLiteralText(text: string): string {
/** Converts a string received from antlr via ctx.getText() into a JavaScript string */
let result: string;
if (text.startsWith("'") && text.endsWith("'")) {
result = text.slice(1, -1);
result = result.replace(/''/g, "'");
result = result.replace(/\\'/g, "'");
} else if (text.startsWith('"') && text.endsWith('"')) {
result = text.slice(1, -1);
result = result.replace(/""/g, '"');
result = result.replace(/\\"/g, '"');
} else if (text.startsWith("`") && text.endsWith("`")) {
result = text.slice(1, -1);
result = result.replace(/``/g, "`");
result = result.replace(/\\`/g, "`");
} else if (text.startsWith("{") && text.endsWith("}")) {
result = text.slice(1, -1);
result = result.replace(/{{/g, "{");
result = result.replace(/\\{/g, "{");
} else {
throw new SyntaxError(
`Invalid string literal, must start and end with the same quote type: ${text}`
);
}
return replaceCommonEscapeCharacters(result);
}
export function parseStringLiteralCtx(ctx: { getText(): string }): string {
/** Converts a STRING_LITERAL received from antlr via ctx.getText() into a JavaScript string */
const text = ctx.getText();
return parseStringLiteralText(text);
}
export function parseStringTextCtx(
ctx: { getText(): string },
escapeQuotes: boolean = true
): string {
/** Converts a STRING_TEXT received from antlr via ctx.getText() into a JavaScript string */
let text = ctx.getText();
if (escapeQuotes) {
text = text.replace(/''/g, "'");
text = text.replace(/\\'/g, "'");
}
text = text.replace(/\\{/g, "{");
return replaceCommonEscapeCharacters(text);
}
internal-packages/tsql/src/query/parser.test.ts
+629
View File
@@ -0,0 +1,629 @@
import { describe, it, expect } from "vitest";
import { CharStreams, CommonTokenStream } from "antlr4ts";
import { TSQLLexer } from "../grammar/TSQLLexer.js";
import { TSQLParser } from "../grammar/TSQLParser.js";
import { TSQLParseTreeConverter } from "./parser.js";
import { ArithmeticOperationOp, CompareOperationOp } from "./ast.js";
import { SyntaxError } from "./errors.js";
/**
* Helper function to parse TSQL input and convert to AST
*/
function parseAndConvert(input: string) {
const inputStream = CharStreams.fromString(input);
const lexer = new TSQLLexer(inputStream);
const tokenStream = new CommonTokenStream(lexer);
const parser = new TSQLParser(tokenStream);
const parseTree = parser.select();
const converter = new TSQLParseTreeConverter();
return converter.visit(parseTree);
}
describe("TSQLParseTreeConverter", () => {
describe("SELECT statements", () => {
it("should convert a simple SELECT statement", () => {
const ast = parseAndConvert("SELECT * FROM users");
expect(ast).toMatchObject({
expression_type: "select_query",
select: [{ expression_type: "field", chain: ["*"] }],
select_from: {
expression_type: "join_expr",
table: { expression_type: "field", chain: ["users"] },
},
});
});
it("should convert SELECT with multiple columns", () => {
const ast = parseAndConvert("SELECT id, name, email FROM users");
expect(ast).toMatchObject({
expression_type: "select_query",
select: [
{ expression_type: "field", chain: ["id"] },
{ expression_type: "field", chain: ["name"] },
{ expression_type: "field", chain: ["email"] },
],
select_from: {
expression_type: "join_expr",
table: { expression_type: "field", chain: ["users"] },
},
});
});
it("should convert SELECT with DISTINCT", () => {
const ast = parseAndConvert("SELECT DISTINCT id FROM users");
expect(ast).toMatchObject({
expression_type: "select_query",
distinct: true,
select: [{ expression_type: "field", chain: ["id"] }],
});
});
it("should convert SELECT with WHERE clause", () => {
const ast = parseAndConvert("SELECT * FROM users WHERE id = 1");
expect(ast).toMatchObject({
expression_type: "select_query",
select: [{ expression_type: "field", chain: ["*"] }],
select_from: {
expression_type: "join_expr",
table: { expression_type: "field", chain: ["users"] },
},
where: {
expression_type: "compare_operation",
op: CompareOperationOp.Eq,
left: { expression_type: "field", chain: ["id"] },
right: { expression_type: "constant", value: 1 },
},
});
});
it("should convert SELECT with ORDER BY", () => {
const ast = parseAndConvert("SELECT * FROM users ORDER BY id DESC");
expect(ast).toMatchObject({
expression_type: "select_query",
select: [{ expression_type: "field", chain: ["*"] }],
order_by: [
{
expression_type: "order_expr",
order: "DESC",
expr: { expression_type: "field", chain: ["id"] },
},
],
});
});
it("should convert SELECT with LIMIT", () => {
const ast = parseAndConvert("SELECT * FROM users LIMIT 10");
expect(ast).toMatchObject({
expression_type: "select_query",
select: [{ expression_type: "field", chain: ["*"] }],
limit: { expression_type: "constant", value: 10 },
});
});
it("should convert SELECT with LIMIT and OFFSET", () => {
const ast = parseAndConvert("SELECT * FROM users LIMIT 10 OFFSET 5");
expect(ast).toMatchObject({
expression_type: "select_query",
select: [{ expression_type: "field", chain: ["*"] }],
limit: { expression_type: "constant", value: 10 },
offset: { expression_type: "constant", value: 5 },
});
});
it("should convert SELECT with GROUP BY", () => {
const ast = parseAndConvert("SELECT category, COUNT(*) FROM products GROUP BY category");
expect(ast).toMatchObject({
expression_type: "select_query",
select: [
{ expression_type: "field", chain: ["category"] },
{ expression_type: "call", name: "COUNT" },
],
group_by: [{ expression_type: "field", chain: ["category"] }],
});
});
it("should convert SELECT with HAVING", () => {
const ast = parseAndConvert(
"SELECT category FROM products GROUP BY category HAVING COUNT(*) > 10"
);
expect(ast).toMatchObject({
expression_type: "select_query",
select: [{ expression_type: "field", chain: ["category"] }],
group_by: [{ expression_type: "field", chain: ["category"] }],
having: {
expression_type: "compare_operation",
op: CompareOperationOp.Gt,
left: { expression_type: "call", name: "COUNT" },
right: { expression_type: "constant", value: 10 },
},
});
});
});
describe("expressions", () => {
it("should convert numeric constants", () => {
const ast = parseAndConvert("SELECT 42 FROM users");
expect(ast).toMatchObject({
expression_type: "select_query",
select: [{ expression_type: "constant", value: 42 }],
});
});
it("should convert string constants", () => {
const ast = parseAndConvert("SELECT 'hello' FROM users");
expect(ast).toMatchObject({
expression_type: "select_query",
select: [{ expression_type: "constant", value: "hello" }],
});
});
it("should convert boolean constants", () => {
const ast = parseAndConvert("SELECT true FROM users");
expect(ast).toMatchObject({
expression_type: "select_query",
select: [{ expression_type: "constant", value: true }],
});
});
it("should convert arithmetic addition", () => {
const ast = parseAndConvert("SELECT 1 + 2 FROM users");
expect(ast).toMatchObject({
expression_type: "select_query",
select: [
{
expression_type: "arithmetic_operation",
op: ArithmeticOperationOp.Add,
left: { expression_type: "constant", value: 1 },
right: { expression_type: "constant", value: 2 },
},
],
});
});
it("should convert arithmetic subtraction", () => {
const ast = parseAndConvert("SELECT 5 - 3 FROM users");
expect(ast).toMatchObject({
expression_type: "select_query",
select: [
{
expression_type: "arithmetic_operation",
op: ArithmeticOperationOp.Sub,
left: { expression_type: "constant", value: 5 },
right: { expression_type: "constant", value: 3 },
},
],
});
});
it("should convert arithmetic multiplication", () => {
const ast = parseAndConvert("SELECT 2 * 3 FROM users");
expect(ast).toMatchObject({
expression_type: "select_query",
select: [
{
expression_type: "arithmetic_operation",
op: ArithmeticOperationOp.Mult,
left: { expression_type: "constant", value: 2 },
right: { expression_type: "constant", value: 3 },
},
],
});
});
it("should convert arithmetic division", () => {
const ast = parseAndConvert("SELECT 10 / 2 FROM users");
expect(ast).toMatchObject({
expression_type: "select_query",
select: [
{
expression_type: "arithmetic_operation",
op: ArithmeticOperationOp.Div,
left: { expression_type: "constant", value: 10 },
right: { expression_type: "constant", value: 2 },
},
],
});
});
it("should convert comparison equals", () => {
const ast = parseAndConvert("SELECT * FROM users WHERE id = 1");
expect(ast).toMatchObject({
expression_type: "select_query",
where: {
expression_type: "compare_operation",
op: CompareOperationOp.Eq,
left: { expression_type: "field", chain: ["id"] },
right: { expression_type: "constant", value: 1 },
},
});
});
it("should convert comparison not equals", () => {
const ast = parseAndConvert("SELECT * FROM users WHERE id != 1");
expect(ast).toMatchObject({
expression_type: "select_query",
where: {
expression_type: "compare_operation",
op: CompareOperationOp.NotEq,
left: { expression_type: "field", chain: ["id"] },
right: { expression_type: "constant", value: 1 },
},
});
});
it("should convert comparison less than", () => {
const ast = parseAndConvert("SELECT * FROM users WHERE id < 10");
expect(ast).toMatchObject({
expression_type: "select_query",
where: {
expression_type: "compare_operation",
op: CompareOperationOp.Lt,
left: { expression_type: "field", chain: ["id"] },
right: { expression_type: "constant", value: 10 },
},
});
});
it("should convert comparison greater than", () => {
const ast = parseAndConvert("SELECT * FROM users WHERE id > 5");
expect(ast).toMatchObject({
expression_type: "select_query",
where: {
expression_type: "compare_operation",
op: CompareOperationOp.Gt,
left: { expression_type: "field", chain: ["id"] },
right: { expression_type: "constant", value: 5 },
},
});
});
it("should convert LIKE comparison", () => {
const ast = parseAndConvert("SELECT * FROM users WHERE name LIKE '%john%'");
expect(ast).toMatchObject({
expression_type: "select_query",
where: {
expression_type: "compare_operation",
op: CompareOperationOp.Like,
left: { expression_type: "field", chain: ["name"] },
right: { expression_type: "constant", value: "%john%" },
},
});
});
it("should convert IN comparison", () => {
const ast = parseAndConvert("SELECT * FROM users WHERE id IN (1, 2, 3)");
expect(ast).toMatchObject({
expression_type: "select_query",
where: {
expression_type: "compare_operation",
op: CompareOperationOp.In,
left: { expression_type: "field", chain: ["id"] },
right: {
expression_type: "tuple",
exprs: [
{ expression_type: "constant", value: 1 },
{ expression_type: "constant", value: 2 },
{ expression_type: "constant", value: 3 },
],
},
},
});
});
it("should convert function calls", () => {
const ast = parseAndConvert("SELECT COUNT(*) FROM users");
expect(ast).toMatchObject({
expression_type: "select_query",
select: [
{
expression_type: "call",
name: "COUNT",
args: [{ expression_type: "field", chain: ["*"] }],
},
],
});
});
it("should convert nested field access", () => {
const ast = parseAndConvert("SELECT user.profile.name FROM users");
expect(ast).toMatchObject({
expression_type: "select_query",
select: [{ expression_type: "field", chain: ["user", "profile", "name"] }],
});
});
it("should convert aliased expressions", () => {
const ast = parseAndConvert("SELECT id AS user_id FROM users");
expect(ast).toMatchObject({
expression_type: "select_query",
select: [
{
expression_type: "alias",
alias: "user_id",
expr: { expression_type: "field", chain: ["id"] },
},
],
});
});
});
describe("JOINs", () => {
it("should convert INNER JOIN", () => {
const ast = parseAndConvert(
"SELECT * FROM users INNER JOIN orders ON users.id = orders.user_id"
);
expect(ast).toMatchObject({
expression_type: "select_query",
select: [{ expression_type: "field", chain: ["*"] }],
select_from: {
expression_type: "join_expr",
table: { expression_type: "field", chain: ["users"] },
next_join: {
expression_type: "join_expr",
join_type: "INNER JOIN",
table: { expression_type: "field", chain: ["orders"] },
constraint: {
expression_type: "join_constraint",
constraint_type: "ON",
expr: {
expression_type: "compare_operation",
op: CompareOperationOp.Eq,
},
},
},
},
});
});
it("should convert LEFT JOIN", () => {
const ast = parseAndConvert(
"SELECT * FROM users LEFT JOIN orders ON users.id = orders.user_id"
);
expect(ast).toMatchObject({
expression_type: "select_query",
select_from: {
expression_type: "join_expr",
table: { expression_type: "field", chain: ["users"] },
next_join: {
expression_type: "join_expr",
join_type: "LEFT JOIN",
table: { expression_type: "field", chain: ["orders"] },
constraint: { constraint_type: "ON" },
},
},
});
});
it("should convert CROSS JOIN", () => {
const ast = parseAndConvert("SELECT * FROM users CROSS JOIN orders");
expect(ast).toMatchObject({
expression_type: "select_query",
select_from: {
expression_type: "join_expr",
table: { expression_type: "field", chain: ["users"] },
next_join: {
expression_type: "join_expr",
join_type: "CROSS JOIN",
table: { expression_type: "field", chain: ["orders"] },
},
},
});
});
});
describe("UNION queries", () => {
it("should convert UNION DISTINCT query", () => {
// Grammar supports UNION ALL, UNION DISTINCT, INTERSECT, INTERSECT DISTINCT, EXCEPT
// Bare UNION (without ALL/DISTINCT) is not supported
const ast = parseAndConvert("SELECT id FROM users UNION DISTINCT SELECT id FROM customers");
expect(ast).toMatchObject({
expression_type: "select_set_query",
initial_select_query: {
expression_type: "select_query",
select: [{ expression_type: "field", chain: ["id"] }],
select_from: {
table: { expression_type: "field", chain: ["users"] },
},
},
subsequent_select_queries: [
{
set_operator: "UNION DISTINCT",
select_query: {
expression_type: "select_query",
select: [{ expression_type: "field", chain: ["id"] }],
select_from: {
table: { expression_type: "field", chain: ["customers"] },
},
},
},
],
});
});
it("should convert UNION ALL query", () => {
const ast = parseAndConvert("SELECT id FROM users UNION ALL SELECT id FROM customers");
expect(ast).toMatchObject({
expression_type: "select_set_query",
initial_select_query: {
expression_type: "select_query",
select: [{ expression_type: "field", chain: ["id"] }],
},
subsequent_select_queries: [
{
set_operator: "UNION ALL",
select_query: {
expression_type: "select_query",
select: [{ expression_type: "field", chain: ["id"] }],
},
},
],
});
});
});
describe("WITH clauses (CTEs)", () => {
it("should convert SELECT with WITH clause", () => {
const ast = parseAndConvert(
"WITH recent_users AS (SELECT * FROM users WHERE created_at > '2024-01-01') SELECT * FROM recent_users"
);
expect(ast).toMatchObject({
expression_type: "select_query",
ctes: {
recent_users: {
expression_type: "cte",
name: "recent_users",
cte_type: "subquery",
expr: {
expression_type: "select_query",
select: [{ expression_type: "field", chain: ["*"] }],
select_from: {
table: { expression_type: "field", chain: ["users"] },
},
where: {
expression_type: "compare_operation",
op: CompareOperationOp.Gt,
},
},
},
},
select: [{ expression_type: "field", chain: ["*"] }],
select_from: {
table: { expression_type: "field", chain: ["recent_users"] },
},
});
});
});
describe("error handling", () => {
it("should preserve position information in errors", () => {
const input = "SELECT * FROM users WHERE invalid syntax";
const inputStream = CharStreams.fromString(input);
const lexer = new TSQLLexer(inputStream);
const tokenStream = new CommonTokenStream(lexer);
const parser = new TSQLParser(tokenStream);
// This might not parse correctly, but if it does and we visit an error node,
// it should throw with position info
try {
const parseTree = parser.select();
const converter = new TSQLParseTreeConverter();
converter.visit(parseTree);
} catch (error) {
if (error instanceof SyntaxError) {
// Error should have position information if available
expect(error).toBeInstanceOf(SyntaxError);
}
}
});
});
describe("complex queries", () => {
it("should convert a complex query with multiple clauses", () => {
const ast = parseAndConvert(
"SELECT category, COUNT(*) as count " +
"FROM products " +
"WHERE price > 100 " +
"GROUP BY category " +
"HAVING COUNT(*) > 5 " +
"ORDER BY count DESC " +
"LIMIT 10"
);
expect(ast).toMatchObject({
expression_type: "select_query",
select: [
{ expression_type: "field", chain: ["category"] },
{
expression_type: "alias",
alias: "count",
expr: { expression_type: "call", name: "COUNT" },
},
],
select_from: {
table: { expression_type: "field", chain: ["products"] },
},
where: {
expression_type: "compare_operation",
op: CompareOperationOp.Gt,
left: { expression_type: "field", chain: ["price"] },
right: { expression_type: "constant", value: 100 },
},
group_by: [{ expression_type: "field", chain: ["category"] }],
having: {
expression_type: "compare_operation",
op: CompareOperationOp.Gt,
left: { expression_type: "call", name: "COUNT" },
right: { expression_type: "constant", value: 5 },
},
order_by: [
{
expression_type: "order_expr",
order: "DESC",
expr: { expression_type: "field", chain: ["count"] },
},
],
limit: { expression_type: "constant", value: 10 },
});
});
it("should convert query with multiple JOINs", () => {
const ast = parseAndConvert(
"SELECT * FROM users " +
"INNER JOIN orders ON users.id = orders.user_id " +
"LEFT JOIN products ON orders.product_id = products.id"
);
expect(ast).toMatchObject({
expression_type: "select_query",
select: [{ expression_type: "field", chain: ["*"] }],
select_from: {
expression_type: "join_expr",
table: { expression_type: "field", chain: ["users"] },
next_join: {
expression_type: "join_expr",
join_type: "INNER JOIN",
table: { expression_type: "field", chain: ["orders"] },
constraint: { constraint_type: "ON" },
next_join: {
expression_type: "join_expr",
join_type: "LEFT JOIN",
table: { expression_type: "field", chain: ["products"] },
constraint: { constraint_type: "ON" },
},
},
},
});
});
});
});
internal-packages/tsql/src/query/parser.ts
+1839
View File
File diff suppressed because it is too large Load Diff
internal-packages/tsql/src/query/printer.test.ts
+3916
View File
File diff suppressed because it is too large Load Diff
internal-packages/tsql/src/query/printer.ts
+3285
View File
File diff suppressed because it is too large Load Diff
internal-packages/tsql/src/query/printer_context.ts
+293
View File
@@ -0,0 +1,293 @@
// TypeScript port of posthog/hogql/context.py
// Adapted for ClickHouse client's {param: Type} syntax
import { getClickHouseType } from "./escape";
import type { SchemaRegistry, FieldMappings } from "./schema";
/**
* Settings that control query execution behavior
*/
export interface QuerySettings {
/** Maximum number of rows to return */
maxRows?: number;
/** Timezone for date/time operations */
timezone?: string;
/** Whether to allow full table scans */
allowFullTableScans?: boolean;
/** Query timeout in seconds */
timeoutSeconds?: number;
}
/**
* A simple comparison condition (e.g., column > value)
*/
export interface SimpleComparisonCondition {
/** The comparison operator */
op: "eq" | "neq" | "gt" | "gte" | "lt" | "lte";
/** The value to compare against */
value: Date | string | number;
}
/**
* A between condition (e.g., column BETWEEN low AND high)
*/
export interface BetweenCondition {
/** The between operator */
op: "between";
/** The low bound of the range */
low: Date | string | number;
/** The high bound of the range */
high: Date | string | number;
}
/**
* An IN condition (e.g., column IN ('a', 'b', 'c'))
*/
export interface InCondition {
/** The in operator */
op: "in";
/** The values to check against */
values: (string | number)[];
}
/**
* A WHERE clause condition that can be either a simple comparison, a BETWEEN, or an IN.
* Used for both enforcedWhereClause (always applied) and whereClauseFallback (default when user doesn't filter).
*/
export type WhereClauseCondition = SimpleComparisonCondition | BetweenCondition | InCondition;
/**
* A time range used by `timeBucket()` to determine the appropriate bucket interval.
*/
export interface TimeRange {
/** Start of the time range (inclusive) */
from: Date;
/** End of the time range (inclusive) */
to: Date;
}
/**
* Default query settings
*/
export const DEFAULT_QUERY_SETTINGS: Required<QuerySettings> = {
maxRows: 10000,
timezone: "UTC",
allowFullTableScans: false,
timeoutSeconds: 60,
};
/**
* A warning or notice collected during query printing
*/
export interface QueryNotice {
code: string;
message: string;
start?: number;
end?: number;
}
/**
* Context for the TSQL to ClickHouse printer
*
* Holds:
* - Enforced WHERE conditions for tenant isolation and plan limits
* - Schema registry for table/column validation
* - Parameter accumulator for SQL injection safety
* - Query settings and execution options
* - Field mappings for runtime value translation
*/
export class PrinterContext {
/** Accumulated parameter values for parameterized query */
private values: Record<string, unknown> = {};
/** Counter for generating unique parameter names */
private paramCounter = 0;
/** Warnings collected during printing */
readonly warnings: QueryNotice[] = [];
/** Errors collected during printing */
readonly errors: QueryNotice[] = [];
/** Runtime field mappings for dynamic value translation */
readonly fieldMappings: FieldMappings;
/**
* Enforced WHERE conditions that are ALWAYS applied at the table level.
* Used for tenant isolation (org_id, project_id, env_id) and plan-based limits.
* Applied to every table reference including subqueries, CTEs, and JOINs.
*/
readonly enforcedWhereClause: Record<string, WhereClauseCondition>;
/**
* Time range for `timeBucket()` interval calculation.
* When provided, `timeBucket()` uses this to determine the appropriate bucket size.
*/
readonly timeRange?: TimeRange;
constructor(
/** Schema registry containing allowed tables and columns */
public readonly schema: SchemaRegistry,
/** Query execution settings */
public readonly settings: QuerySettings = {},
/** Runtime field mappings for dynamic value translation */
fieldMappings: FieldMappings = {},
/**
* Enforced WHERE conditions that are ALWAYS applied at the table level.
* Must include tenant columns (e.g., organization_id) for multi-tenant tables.
*/
enforcedWhereClause: Record<string, WhereClauseCondition> = {},
/** Time range for timeBucket() interval calculation */
timeRange?: TimeRange
) {
// Initialize with default settings
this.settings = { ...DEFAULT_QUERY_SETTINGS, ...settings };
this.fieldMappings = fieldMappings;
this.enforcedWhereClause = enforcedWhereClause;
this.timeRange = timeRange;
}
/**
* Get the timezone setting
*/
get timezone(): string {
return this.settings.timezone ?? DEFAULT_QUERY_SETTINGS.timezone;
}
/**
* Get the max rows setting
*/
get maxRows(): number {
return this.settings.maxRows ?? DEFAULT_QUERY_SETTINGS.maxRows;
}
/**
* Add a value to the parameter map and return a ClickHouse placeholder
*
* @param value The value to parameterize
* @returns A placeholder string like "{tsql_val_0: String}"
*/
addValue(value: unknown): string {
const key = `tsql_val_${this.paramCounter++}`;
this.values[key] = value;
const chType = getClickHouseType(value);
return `{${key}: ${chType}}`;
}
/**
* Add a value with a specific key (for named parameters)
*
* @param key The parameter name
* @param value The value
* @returns A placeholder string like "{key: Type}"
*/
addNamedValue(key: string, value: unknown): string {
this.values[key] = value;
const chType = getClickHouseType(value);
return `{${key}: ${chType}}`;
}
/**
* Get all accumulated parameter values
*/
getParams(): Record<string, unknown> {
return { ...this.values };
}
/**
* Add a warning notice
*/
addWarning(code: string, message: string, start?: number, end?: number): void {
this.warnings.push({ code, message, start, end });
}
/**
* Add an error notice
*/
addError(code: string, message: string, start?: number, end?: number): void {
this.errors.push({ code, message, start, end });
}
/**
* Check if any errors were collected
*/
hasErrors(): boolean {
return this.errors.length > 0;
}
/**
* Create a child context that shares the same parameter accumulator
* Useful for handling subqueries while keeping parameters unified
*/
createChildContext(): PrinterContext {
const child = new PrinterContext(
this.schema,
this.settings,
this.fieldMappings,
this.enforcedWhereClause,
this.timeRange
);
// Share the same values map so parameters are unified
child.values = this.values;
// Share the same counter reference via closure
// eslint-disable-next-line no-this-alias
const parentThis = this;
Object.defineProperty(child, "paramCounter", {
get() {
return parentThis.paramCounter;
},
set(v: number) {
parentThis.paramCounter = v;
},
});
return child;
}
}
/**
* Options for creating a printer context
*/
export interface PrinterContextOptions {
/** Schema registry containing allowed tables and columns */
schema: SchemaRegistry;
/** Query execution settings */
settings?: QuerySettings;
/**
* Runtime field mappings for dynamic value translation.
* Maps internal ClickHouse values to external user-facing values.
*/
fieldMappings?: FieldMappings;
/**
* REQUIRED: Conditions always applied at the table level.
* Must include tenant columns (e.g., organization_id) for multi-tenant tables.
* Applied to every table reference including subqueries, CTEs, and JOINs.
*
* @example
* ```typescript
* {
* organization_id: { op: "eq", value: "org_123" },
* project_id: { op: "eq", value: "proj_456" },
* triggered_at: { op: "gte", value: new Date(Date.now() - 7 * 24 * 60 * 60 * 1000) }
* }
* ```
*/
enforcedWhereClause: Record<string, WhereClauseCondition>;
/**
* Time range for `timeBucket()` interval calculation.
* When provided, `timeBucket()` uses this to determine the appropriate bucket size.
*/
timeRange?: TimeRange;
}
/**
* Create a new PrinterContext
*/
export function createPrinterContext(options: PrinterContextOptions): PrinterContext {
return new PrinterContext(
options.schema,
options.settings,
options.fieldMappings,
options.enforcedWhereClause,
options.timeRange
);
}
internal-packages/tsql/src/query/property_types.ts
+720
View File
@@ -0,0 +1,720 @@
// TypeScript translation of posthog/hogql/transforms/property_types.py
import type {
AST,
Expr,
Field,
PropertyType,
FieldType,
BaseTableType,
VirtualTableType,
LazyJoinType,
LazyTableType,
Call,
Constant,
CallType,
DateTimeType,
} from "./ast";
import type { TSQLContext } from "./context";
import type { BooleanDatabaseField, DateTimeDatabaseField } from "./models";
// Helper function to escape TSQL identifiers
function escapeTSQLIdentifier(identifier: string | number): string {
if (typeof identifier === "number") {
return String(identifier);
}
if (identifier.includes("%")) {
throw new Error(
`The TSQL identifier "${identifier}" is not permitted as it contains the "%" character`
);
}
// TSQL allows dollars in the identifier
if (/^[A-Za-z_$][A-Za-z0-9_$]*$/.test(identifier)) {
return identifier;
}
// Escape backticks and other special characters
const backquoteEscapeChars: Record<string, string> = {
"\b": "\\b",
"\f": "\\f",
"\r": "\\r",
"\n": "\\n",
"\t": "\\t",
"\0": "\\0",
a: "\\a",
"\v": "\\v",
"\\": "\\\\",
"`": "\\`",
};
return `\`${identifier
.split("")
.map((c) => backquoteEscapeChars[c] || c)
.join("")}\``;
}
// Visitor dispatcher - converts node type to visitor method name
// Matches Python's camel_case_pattern.sub("_", class_name).lower() logic
function getVisitorMethodName(node: AST): string {
// Get the constructor name or use a type guard to determine the type
const nodeType = (node as any).constructor?.name || detectNodeType(node);
if (!nodeType) {
return "visit_unknown";
}
// Convert CamelCase to snake_case (e.g., "PropertyType" -> "property_type")
const snakeCase = nodeType
.replace(/([A-Z])/g, "_$1")
.toLowerCase()
.replace(/^_/, "");
// Handle special cases (matching Python replacements)
const replacements: Record<string, string> = {
tsqlxtag: "tsqlx_tag",
tsqlxattribute: "tsqlx_attribute",
uuidtype: "uuid_type",
string_jsontype: "string_json_type",
};
return replacements[snakeCase] || snakeCase;
}
// Type detection helper (since we can't use instanceof with interfaces)
function detectNodeType(node: AST): string {
// Use property presence to detect node types
if ("chain" in node && "type" in node && !("name" in node)) {
return "Field";
}
if ("chain" in node && "field_type" in node) {
return "PropertyType";
}
if ("name" in node && "args" in node && !("expr" in node)) {
return "Call";
}
if ("value" in node && !("name" in node) && !("args" in node)) {
return "Constant";
}
// Add more type detection as needed
return "";
}
// Base visitor class - matches Python Visitor pattern
abstract class Visitor<T> {
visit(node: AST | null | undefined): T {
if (node === null || node === undefined) {
return node as T;
}
// Try using accept method if available (double dispatch)
if (node.accept) {
return node.accept(this) as T;
}
// Fallback: use dispatcher
const methodName = getVisitorMethodName(node);
const method = (this as any)[methodName];
if (method && typeof method === "function") {
return method.call(this, node) as T;
}
// Try visit_unknown as fallback
if ((this as any).visit_unknown) {
return (this as any).visit_unknown(node) as T;
}
throw new Error(`${this.constructor.name} has no method ${methodName} or visit_unknown`);
}
}
// TraversingVisitor - matches Python TraversingVisitor
class TraversingVisitor extends Visitor<void> {
visitPropertyType(node: PropertyType): void {
this.visit(node.field_type);
}
visitField(node: Field): void {
if (node.type) {
this.visit(node.type as any);
}
}
visitCall(node: Call): void {
for (const arg of node.args) {
this.visit(arg);
}
if (node.params) {
for (const param of node.params) {
this.visit(param);
}
}
}
visitConstant(node: Constant): void {
if (node.type) {
this.visit(node.type as any);
}
}
// Default handler for unknown types - traverse common properties
visit_unknown(node: AST): void {
// Traverse children based on common AST node properties
if ("expr" in node) {
this.visit((node as any).expr);
}
if ("exprs" in node) {
for (const expr of (node as any).exprs) {
this.visit(expr);
}
}
if ("left" in node && "right" in node) {
this.visit((node as any).left);
this.visit((node as any).right);
}
if ("args" in node) {
for (const arg of (node as any).args) {
this.visit(arg);
}
}
if ("type" in node) {
this.visit((node as any).type);
}
}
}
// CloningVisitor - matches Python CloningVisitor
class CloningVisitor extends Visitor<any> {
protected clearTypes: boolean;
protected clearLocations: boolean;
constructor(clearTypes: boolean = true, clearLocations: boolean = false) {
super();
this.clearTypes = clearTypes;
this.clearLocations = clearLocations;
}
visitField(node: Field): Field {
return {
...node,
type: this.clearTypes ? undefined : node.type ? this.visit(node.type as any) : node.type,
start: this.clearLocations ? undefined : node.start,
end: this.clearLocations ? undefined : node.end,
};
}
visitPropertyType(node: PropertyType): PropertyType {
return {
...node,
field_type: this.visit(node.field_type) as FieldType,
start: this.clearLocations ? undefined : node.start,
end: this.clearLocations ? undefined : node.end,
};
}
visitCall(node: Call): Call {
return {
...node,
args: node.args.map((arg) => this.visit(arg)),
params: node.params ? node.params.map((param) => this.visit(param)) : undefined,
start: this.clearLocations ? undefined : node.start,
end: this.clearLocations ? undefined : node.end,
type: this.clearTypes ? undefined : node.type,
};
}
visitConstant(node: Constant): Constant {
return {
...node,
start: this.clearLocations ? undefined : node.start,
end: this.clearLocations ? undefined : node.end,
type: this.clearTypes ? undefined : node.type,
};
}
// Default handler for unknown types - shallow clone
visit_unknown(node: AST): any {
const cloned: any = { ...node };
// Clone common properties
if ("expr" in node) {
cloned.expr = this.visit((node as any).expr);
}
if ("exprs" in node) {
cloned.exprs = (node as any).exprs.map((e: any) => this.visit(e));
}
if ("left" in node && "right" in node) {
cloned.left = this.visit((node as any).left);
cloned.right = this.visit((node as any).right);
}
if ("args" in node) {
cloned.args = (node as any).args.map((a: any) => this.visit(a));
}
if ("type" in node) {
cloned.type = this.clearTypes ? undefined : this.visit((node as any).type);
}
if (this.clearLocations) {
cloned.start = undefined;
cloned.end = undefined;
}
return cloned;
}
}
// PropertyFinder: Traverses AST to find all property references
class PropertyFinder extends TraversingVisitor {
context: TSQLContext;
personProperties: Set<string> = new Set();
eventProperties: Set<string> = new Set();
groupProperties: Map<number, Set<string>> = new Map();
foundTimestamps: boolean = false;
constructor(context: TSQLContext) {
super();
this.context = context;
}
visitPropertyType(node: PropertyType): void {
if (node.field_type.name === "properties" && node.chain.length === 1) {
const tableType = node.field_type.table_type;
if (this.isBaseTableType(tableType)) {
const table = tableType.resolve_database_table?.(this.context);
if (table) {
const tableName = table.to_printed_tsql?.() || "";
const propertyName = String(node.chain[0]);
if (tableName === "persons" || tableName === "raw_persons") {
this.personProperties.add(propertyName);
} else if (tableName === "groups") {
if (this.isLazyJoinType(tableType)) {
if (tableType.field.startsWith("group_")) {
const groupId = parseInt(tableType.field.split("_")[1], 10);
if (!this.groupProperties.has(groupId)) {
this.groupProperties.set(groupId, new Set());
}
this.groupProperties.get(groupId)!.add(propertyName);
}
} else if (this.isLazyTableType(tableType)) {
const globalGroupId = this.context.globals?.group_id;
if (typeof globalGroupId === "number") {
if (!this.groupProperties.has(globalGroupId)) {
this.groupProperties.set(globalGroupId, new Set());
}
this.groupProperties.get(globalGroupId)!.add(propertyName);
}
}
} else if (tableName === "events") {
if (this.isVirtualTableType(tableType) && tableType.field === "poe") {
this.personProperties.add(propertyName);
} else {
this.eventProperties.add(propertyName);
}
}
}
}
}
super.visitPropertyType(node);
}
visitField(node: Field): void {
super.visitField(node);
if (this.isFieldType(node.type)) {
const dbField = (node.type as any).resolve_database_field?.(this.context);
if (this.isDateTimeDatabaseField(dbField)) {
this.foundTimestamps = true;
}
}
}
private isBaseTableType(type: any): type is BaseTableType {
return type && typeof type.resolve_database_table === "function";
}
private isLazyJoinType(type: any): type is LazyJoinType {
return type && "lazy_join" in type && "field" in type;
}
private isLazyTableType(type: any): type is LazyTableType {
return type && "table" in type && !("lazy_join" in type);
}
private isVirtualTableType(type: any): type is VirtualTableType {
return type && "virtual_table" in type && "field" in type;
}
private isFieldType(type: any): type is FieldType {
return type && typeof type.resolve_database_field === "function";
}
private isDateTimeDatabaseField(field: any): field is DateTimeDatabaseField {
return field && "name" in field; // Simplified check
}
}
// PropertySwapper: Transforms property accesses with type conversions
export class PropertySwapper extends CloningVisitor {
timezone: string;
eventProperties: Map<string, string>;
personProperties: Map<string, string>;
groupProperties: Map<string, string>;
context: TSQLContext;
setTimeZones: boolean;
constructor(
timezone: string,
eventProperties: Map<string, string> | Record<string, string>,
personProperties: Map<string, string> | Record<string, string>,
groupProperties: Map<string, string> | Record<string, string>,
context: TSQLContext,
setTimeZones: boolean
) {
super(false); // Don't clear types
this.timezone = timezone;
this.eventProperties =
eventProperties instanceof Map ? eventProperties : new Map(Object.entries(eventProperties));
this.personProperties =
personProperties instanceof Map
? personProperties
: new Map(Object.entries(personProperties));
this.groupProperties =
groupProperties instanceof Map ? groupProperties : new Map(Object.entries(groupProperties));
this.context = context;
this.setTimeZones = setTimeZones;
}
visitField(node: Field): any {
if (this.isFieldType(node.type)) {
if (this.setTimeZones) {
const dbField = (node.type as any).resolve_database_field?.(this.context);
if (this.isDateTimeDatabaseField(dbField)) {
return this.createToTimeZoneCall(node);
}
}
if (this.isLazyJoinType(node.type.table_type)) {
const lazyJoinType = node.type.table_type;
const resolvedTable = lazyJoinType.lazy_join.resolve_table?.(this.context);
// Check if it's an S3Table-like table (has fields property)
if (resolvedTable && "fields" in resolvedTable) {
const field = node.chain[node.chain.length - 1];
const fieldType = resolvedTable.fields[String(field)];
let propType = "String";
if (this.isDateTimeDatabaseField(fieldType)) {
propType = "DateTime";
} else if (this.isBooleanDatabaseField(fieldType)) {
propType = "Boolean";
}
return this.fieldTypeToPropertyCall(node, propType);
}
}
}
const type = node.type;
if (
this.isPropertyType(type) &&
type.field_type.name === "properties" &&
type.chain.length === 1
) {
const propertyName = String(type.chain[0]);
const tableType = type.field_type.table_type;
if (this.isVirtualTableType(tableType) && tableType.field === "poe") {
if (this.personProperties.has(propertyName)) {
return this.convertStringPropertyToType(node, "person", propertyName);
}
} else if (this.isBaseTableType(tableType)) {
const table = tableType.resolve_database_table?.(this.context);
if (table) {
const tableName = table.to_printed_tsql?.() || "";
if (tableName === "persons" || tableName === "raw_persons") {
if (this.personProperties.has(propertyName)) {
return this.convertStringPropertyToType(node, "person", propertyName);
}
} else if (tableName === "groups") {
if (this.isLazyJoinType(tableType)) {
if (tableType.field.startsWith("group_")) {
const groupId = parseInt(tableType.field.split("_")[1], 10);
const groupKey = `${groupId}_${propertyName}`;
if (this.groupProperties.has(groupKey)) {
return this.convertStringPropertyToType(node, "group", groupKey);
}
}
} else if (this.isLazyTableType(tableType)) {
const globalGroupId = this.context.globals?.group_id;
if (typeof globalGroupId === "number") {
const groupKey = `${globalGroupId}_${propertyName}`;
if (this.groupProperties.has(groupKey)) {
return this.convertStringPropertyToType(node, "group", groupKey);
}
}
}
} else if (tableName === "events") {
if (this.eventProperties.has(propertyName)) {
return this.convertStringPropertyToType(node, "event", propertyName);
}
}
}
}
}
if (
this.isPropertyType(type) &&
type.field_type.name === "person_properties" &&
type.chain.length === 1
) {
const propertyName = String(type.chain[0]);
const tableType = type.field_type.table_type;
if (this.isBaseTableType(tableType)) {
const table = tableType.resolve_database_table?.(this.context);
if (table) {
const tableName = table.to_printed_tsql?.() || "";
if (tableName === "events") {
if (this.personProperties.has(propertyName)) {
return this.convertStringPropertyToType(node, "person", propertyName);
}
}
}
}
}
return super.visitField(node);
}
private convertStringPropertyToType(
node: Field,
propertyType: "event" | "person" | "group",
propertyName: string
): Expr {
let fieldTypeValue: string | undefined;
if (propertyType === "person") {
fieldTypeValue = this.personProperties.get(propertyName);
} else if (propertyType === "group") {
fieldTypeValue = this.groupProperties.get(propertyName);
} else {
fieldTypeValue = this.eventProperties.get(propertyName);
}
const fieldType = fieldTypeValue === "Numeric" ? "Float" : fieldTypeValue || "String";
this.addPropertyNotice(node, propertyType, fieldType);
return this.fieldTypeToPropertyCall(node, fieldType);
}
private fieldTypeToPropertyCall(node: Field, fieldType: string): Expr {
if (fieldType === "DateTime") {
return this.createToDateTimeCall(node);
}
if (fieldType === "Float") {
return this.createToFloatCall(node);
}
if (fieldType === "Boolean") {
return this.createToBoolCall(node);
}
return node;
}
private createToTimeZoneCall(node: Field): Call {
return {
expression_type: "call",
name: "toTimeZone",
args: [node, this.createConstant(this.timezone)],
type: {
name: "toTimeZone",
arg_types: [{ data_type: "datetime" } as DateTimeType],
return_type: { data_type: "datetime" } as DateTimeType,
} as CallType,
start: node.start,
end: node.end,
} as Call;
}
private createToDateTimeCall(node: Field): Call {
return {
expression_type: "call",
name: "toDateTime",
args: [node],
start: node.start,
end: node.end,
};
}
private createToFloatCall(node: Field): Call {
return {
expression_type: "call",
name: "toFloat",
args: [node],
start: node.start,
end: node.end,
};
}
private createToBoolCall(node: Field): Call {
return {
expression_type: "call",
name: "toBool",
args: [
{
name: "transform",
args: [
{
name: "toString",
args: [node],
start: node.start,
end: node.end,
} as Call,
this.createConstant(["true", "false"]),
this.createConstant([1, 0]),
this.createConstant(null),
],
start: node.start,
end: node.end,
} as Call,
],
start: node.start,
end: node.end,
} as Call;
}
private createConstant(value: any): Constant {
return {
expression_type: "constant",
value,
};
}
private addPropertyNotice(
node: Field,
propertyType: "event" | "person" | "group",
fieldType: string
): void {
let propertyName = String(node.chain[node.chain.length - 1]);
let materializedColumn: any = null; // MaterializedColumn type not yet defined
if (propertyType === "person") {
// if (this.context.modifiers.personsOnEventsMode !== "disabled") {
// materializedColumn = getMaterializedColumnForProperty('events', propertyName, 'person_properties');
// } else {
// materializedColumn = getMaterializedColumnForProperty('person', propertyName, 'properties');
// }
} else if (propertyType === "group") {
const nameParts = propertyName.split("_");
nameParts.shift();
propertyName = nameParts.join("_");
// materializedColumn = getMaterializedColumnForProperty('groups', propertyName, 'properties');
} else {
// materializedColumn = getMaterializedColumnForProperty('events', propertyName, 'properties');
}
let message = `${
propertyType.charAt(0).toUpperCase() + propertyType.slice(1)
} property '${propertyName}' is of type '${fieldType}'.`;
if (this.context.debug) {
if (materializedColumn !== null) {
message += " This property is materialized ⚡️.";
} else {
message += " This property is not materialized 🐢.";
}
}
this.addNotice(node, message);
}
private addNotice(node: Field, message: string): void {
if (node.start === undefined || node.end === undefined) {
return; // Don't add notices for nodes without location
}
// Only highlight the last part of the chain
const lastPart = node.chain[node.chain.length - 1];
const identifierLength = escapeTSQLIdentifier(lastPart).length;
this.context.notices.push({
start: Math.max(node.start, node.end - identifierLength),
end: node.end,
message,
});
}
private isFieldType(type: any): type is FieldType {
return type && typeof type.resolve_database_field === "function";
}
private isPropertyType(type: any): type is PropertyType {
return type && "field_type" in type && "chain" in type;
}
private isBaseTableType(type: any): type is BaseTableType {
return type && typeof type.resolve_database_table === "function";
}
private isLazyJoinType(type: any): type is LazyJoinType {
return type && "lazy_join" in type && "field" in type;
}
private isLazyTableType(type: any): type is LazyTableType {
return type && "table" in type && !("lazy_join" in type);
}
private isVirtualTableType(type: any): type is VirtualTableType {
return type && "virtual_table" in type && "field" in type;
}
private isDateTimeDatabaseField(field: any): field is DateTimeDatabaseField {
return field && "name" in field; // Simplified check
}
private isBooleanDatabaseField(field: any): field is BooleanDatabaseField {
return field && "name" in field; // Simplified check
}
}
// Main function to build property swapper
export function buildPropertySwapper(node: AST, context: TSQLContext): void {
if (!context || !context.team_id) {
return;
}
// NOTE: In TypeScript, you'll need to fetch the team from your database/ORM
// This is a placeholder - replace with your actual team fetching logic
// if (!context.team) {
// context.team = await Team.findById(context.team_id);
// }
if (!context.team) {
return;
}
// Find all properties
const propertyFinder = new PropertyFinder(context);
propertyFinder.visit(node);
// NOTE: In TypeScript, you'll need to query PropertyDefinition from your database
// This is a placeholder - replace with your actual property definition fetching logic
// const eventPropertyValues = await PropertyDefinition.find({
// project_id: context.team.project_id,
// name: { $in: Array.from(propertyFinder.eventProperties) },
// type: { $in: [null, 'event'] },
// }).select('name property_type');
// const eventProperties = new Map(
// eventPropertyValues.filter((p: any) => p.property_type).map((p: any) => [p.name, p.property_type])
// );
const eventProperties = new Map<string, string>();
const personProperties = new Map<string, string>();
const groupProperties = new Map<string, string>();
// TODO: Implement actual property definition fetching from database
// For now, these are empty maps
const timezone = (context.database as any)?._timezone || "UTC";
context.property_swapper = new PropertySwapper(
timezone,
eventProperties,
personProperties,
groupProperties,
context,
true
);
}
internal-packages/tsql/src/query/results.test.ts
+231
View File
@@ -0,0 +1,231 @@
import { describe, it, expect } from "vitest";
import { transformResults, createResultTransformer } from "./results.js";
import { column, type TableSchema } from "./schema.js";
/**
* Test schema with valueMap
*/
const taskRunsSchema: TableSchema = {
name: "task_runs",
clickhouseName: "trigger_dev.task_runs_v2",
columns: {
id: { name: "id", ...column("String") },
status: {
name: "status",
...column("String"),
valueMap: {
COMPLETED_SUCCESSFULLY: "Completed",
COMPLETED_WITH_ERRORS: "Completed with errors",
SYSTEM_FAILURE: "System failure",
PENDING: "Pending",
EXECUTING: "Running",
FAILED: "Failed",
CANCELLED: "Cancelled",
},
},
environment_type: {
name: "environment_type",
...column("String"),
valueMap: {
DEVELOPMENT: "Development",
STAGING: "Staging",
PRODUCTION: "Production",
},
},
task_identifier: { name: "task_identifier", ...column("String") },
organization_id: { name: "organization_id", ...column("String") },
project_id: { name: "project_id", ...column("String") },
environment_id: { name: "environment_id", ...column("String") },
},
tenantColumns: {
organizationId: "organization_id",
projectId: "project_id",
environmentId: "environment_id",
},
};
/**
* Schema without valueMap
*/
const simpleSchema: TableSchema = {
name: "simple",
clickhouseName: "trigger_dev.simple",
columns: {
id: { name: "id", ...column("String") },
name: { name: "name", ...column("String") },
organization_id: { name: "organization_id", ...column("String") },
project_id: { name: "project_id", ...column("String") },
environment_id: { name: "environment_id", ...column("String") },
},
tenantColumns: {
organizationId: "organization_id",
projectId: "project_id",
environmentId: "environment_id",
},
};
describe("transformResults", () => {
it("should transform internal values to user-friendly values", () => {
const rows = [
{ id: "run_1", status: "COMPLETED_SUCCESSFULLY", task_identifier: "my-task" },
{ id: "run_2", status: "PENDING", task_identifier: "other-task" },
{ id: "run_3", status: "FAILED", task_identifier: "my-task" },
];
const transformed = transformResults(rows, [taskRunsSchema]);
expect(transformed[0].status).toBe("Completed");
expect(transformed[1].status).toBe("Pending");
expect(transformed[2].status).toBe("Failed");
});
it("should transform multiple columns with valueMaps", () => {
const rows = [
{ id: "run_1", status: "COMPLETED_SUCCESSFULLY", environment_type: "PRODUCTION" },
{ id: "run_2", status: "PENDING", environment_type: "DEVELOPMENT" },
];
const transformed = transformResults(rows, [taskRunsSchema]);
expect(transformed[0].status).toBe("Completed");
expect(transformed[0].environment_type).toBe("Production");
expect(transformed[1].status).toBe("Pending");
expect(transformed[1].environment_type).toBe("Development");
});
it("should not modify columns without valueMap", () => {
const rows = [{ id: "run_1", status: "COMPLETED_SUCCESSFULLY", task_identifier: "my-task" }];
const transformed = transformResults(rows, [taskRunsSchema]);
// id and task_identifier should be unchanged
expect(transformed[0].id).toBe("run_1");
expect(transformed[0].task_identifier).toBe("my-task");
});
it("should pass through values not in valueMap unchanged", () => {
const rows = [{ id: "run_1", status: "UNKNOWN_STATUS", task_identifier: "my-task" }];
const transformed = transformResults(rows, [taskRunsSchema]);
// UNKNOWN_STATUS is not in the valueMap, should be passed through
expect(transformed[0].status).toBe("UNKNOWN_STATUS");
});
it("should return original rows if no columns have valueMap", () => {
const rows = [
{ id: "run_1", name: "test" },
{ id: "run_2", name: "other" },
];
const transformed = transformResults(rows, [simpleSchema]);
// Should return the same array (reference equality)
expect(transformed).toBe(rows);
});
it("should handle empty rows array", () => {
const rows: Array<{ id: string; status: string }> = [];
const transformed = transformResults(rows, [taskRunsSchema]);
expect(transformed).toEqual([]);
});
it("should handle case-insensitive internal value matching", () => {
const rows = [
{ id: "run_1", status: "completed_successfully" },
{ id: "run_2", status: "COMPLETED_SUCCESSFULLY" },
{ id: "run_3", status: "Completed_Successfully" },
];
const transformed = transformResults(rows, [taskRunsSchema]);
// All should map to "Completed"
expect(transformed[0].status).toBe("Completed");
expect(transformed[1].status).toBe("Completed");
expect(transformed[2].status).toBe("Completed");
});
it("should preserve non-string column values", () => {
const rows = [{ id: "run_1", status: "COMPLETED_SUCCESSFULLY", count: 42, active: true }];
const transformed = transformResults(rows, [taskRunsSchema]);
expect(transformed[0].count).toBe(42);
expect(transformed[0].active).toBe(true);
expect(transformed[0].status).toBe("Completed");
});
it("should preserve row reference if no changes made", () => {
const rows = [{ id: "run_1", status: "UNKNOWN_STATUS" }];
const transformed = transformResults(rows, [taskRunsSchema]);
// The row has status that doesn't match any valueMap entry
// But the column does have a valueMap, so we still check it
// Since the value doesn't change, the row reference should be preserved
expect(transformed[0]).toBe(rows[0]);
});
it("should handle multiple table schemas", () => {
const anotherSchema: TableSchema = {
name: "events",
clickhouseName: "trigger_dev.events",
columns: {
id: { name: "id", ...column("String") },
event_type: {
name: "event_type",
...column("String"),
valueMap: {
TASK_START: "Started",
TASK_END: "Ended",
},
},
organization_id: { name: "organization_id", ...column("String") },
project_id: { name: "project_id", ...column("String") },
environment_id: { name: "environment_id", ...column("String") },
},
tenantColumns: {
organizationId: "organization_id",
projectId: "project_id",
environmentId: "environment_id",
},
};
const rows = [
{ status: "COMPLETED_SUCCESSFULLY", event_type: "TASK_START" },
{ status: "PENDING", event_type: "TASK_END" },
];
const transformed = transformResults(rows, [taskRunsSchema, anotherSchema]);
expect(transformed[0].status).toBe("Completed");
expect(transformed[0].event_type).toBe("Started");
expect(transformed[1].status).toBe("Pending");
expect(transformed[1].event_type).toBe("Ended");
});
});
describe("createResultTransformer", () => {
it("should create a reusable transformer function", () => {
const transform = createResultTransformer([taskRunsSchema]);
const rows1 = [{ id: "1", status: "COMPLETED_SUCCESSFULLY" }];
const rows2 = [{ id: "2", status: "FAILED" }];
const transformed1 = transform(rows1);
const transformed2 = transform(rows2);
expect(transformed1[0].status).toBe("Completed");
expect(transformed2[0].status).toBe("Failed");
});
it("should return original rows if no valueMap columns exist", () => {
const transform = createResultTransformer([simpleSchema]);
const rows = [{ id: "1", name: "test" }];
const transformed = transform(rows);
expect(transformed).toBe(rows);
});
});
internal-packages/tsql/src/query/results.ts
+200
View File
@@ -0,0 +1,200 @@
/**
* Result transformation utilities for TSQL queries
*
* Transforms query result values from internal ClickHouse values
* to user-friendly display names using the column valueMap or fieldMapping.
*/
import type { TableSchema, ColumnSchema, FieldMappings } from "./schema.js";
import { getUserFriendlyValue, hasFieldMapping, getExternalValue } from "./schema.js";
/**
* Options for transforming query results
*/
export interface TransformResultsOptions {
/**
* If true, transform values even if the column was aliased (e.g., SELECT status AS s)
* Default: false (aliased columns are not transformed since the user explicitly chose a different name)
*/
transformAliased?: boolean;
/**
* Runtime field mappings for dynamic value translation.
* Maps internal ClickHouse values to external user-facing values.
* Values not found in the mapping will be returned as null.
*/
fieldMappings?: FieldMappings;
}
/**
* Transform query result rows, mapping internal values to user-friendly display names
*
* This function iterates over result rows and transforms any column values that have
* a `valueMap` or `fieldMapping` defined in their schema, converting internal ClickHouse values
* (e.g., 'COMPLETED_SUCCESSFULLY') back to user-friendly display names (e.g., 'Completed').
*
* For columns with `fieldMapping`, values not found in the mapping will be returned as null.
*
* @param rows - Array of result rows to transform
* @param schema - Array of table schemas containing column definitions with valueMaps
* @param options - Optional transformation options
* @returns New array of rows with transformed values
*
* @example
* ```typescript
* const schema: TableSchema[] = [{
* name: "task_runs",
* clickhouseName: "trigger_dev.task_runs_v2",
* columns: {
* status: {
* name: "status",
* type: "String",
* valueMap: {
* "COMPLETED_SUCCESSFULLY": "Completed",
* "PENDING": "Pending",
* },
* },
* project_ref: {
* name: "project_ref",
* clickhouseName: "project_id",
* type: "String",
* fieldMapping: "project",
* },
* },
* tenantColumns: { organizationId: "organization_id", projectId: "project_id", environmentId: "environment_id" },
* }];
*
* const results = [{ status: "COMPLETED_SUCCESSFULLY", project_ref: "cm12345" }];
* const transformed = transformResults(results, schema, {
* fieldMappings: { project: { "cm12345": "my-project-ref" } },
* });
* // transformed = [{ status: "Completed", project_ref: "my-project-ref" }]
* ```
*/
export function transformResults<T extends Record<string, unknown>>(
rows: T[],
schema: TableSchema[],
options: TransformResultsOptions = {}
): T[] {
// Build a map of column names to their schemas (for columns that have transformations)
const columnTransformMaps = buildColumnTransformMaps(schema);
// If no columns have transformations, return the original rows unchanged
if (columnTransformMaps.size === 0) {
return rows;
}
// Transform each row
return rows.map((row) => transformRow(row, columnTransformMaps, options.fieldMappings));
}
/**
* Build a map of column names to their schemas for columns that have transformations
* (either valueMap or fieldMapping)
*/
function buildColumnTransformMaps(schema: TableSchema[]): Map<string, ColumnSchema> {
const columnMaps = new Map<string, ColumnSchema>();
for (const table of schema) {
for (const [columnName, columnSchema] of Object.entries(table.columns)) {
const hasValueMap = columnSchema.valueMap && Object.keys(columnSchema.valueMap).length > 0;
const hasFieldMap = hasFieldMapping(columnSchema);
if (hasValueMap || hasFieldMap) {
// Use the TSQL-exposed column name (not the ClickHouse name)
columnMaps.set(columnName, columnSchema);
}
}
}
return columnMaps;
}
/**
* Transform a single value using the column's valueMap or fieldMapping
* Returns the transformed value, or the original value if no transformation applies
* For fieldMapping, returns null if the value is not found in the mapping
*/
function transformSingleValue(
columnSchema: ColumnSchema,
value: string,
fieldMappings?: FieldMappings
): string | null {
// First try static valueMap (always returns the original if no match)
if (columnSchema.valueMap && Object.keys(columnSchema.valueMap).length > 0) {
return getUserFriendlyValue(columnSchema, value);
}
// Then try runtime fieldMapping (returns null if not found)
if (hasFieldMapping(columnSchema) && columnSchema.fieldMapping && fieldMappings) {
const externalValue = getExternalValue(fieldMappings, columnSchema.fieldMapping, value);
// For fieldMapping, return null if not found (per user requirement)
return externalValue;
}
return value;
}
/**
* Transform a single row's values using the column valueMaps and fieldMappings
*/
function transformRow<T extends Record<string, unknown>>(
row: T,
columnTransformMaps: Map<string, ColumnSchema>,
fieldMappings?: FieldMappings
): T {
const transformedRow: Record<string, unknown> = {};
let hasChanges = false;
for (const [key, value] of Object.entries(row)) {
const columnSchema = columnTransformMaps.get(key);
if (columnSchema && typeof value === "string") {
const transformedValue = transformSingleValue(columnSchema, value, fieldMappings);
transformedRow[key] = transformedValue;
if (transformedValue !== value) {
hasChanges = true;
}
} else {
transformedRow[key] = value;
}
}
// Return original row if no changes were made (preserves reference equality)
return hasChanges ? (transformedRow as T) : row;
}
/**
* Create a result transformer bound to a specific schema
*
* Useful when you need to transform multiple result sets with the same schema.
*
* @param schema - Array of table schemas
* @param options - Optional transformation options (including fieldMappings)
* @returns A function that transforms result rows
*
* @example
* ```typescript
* const transform = createResultTransformer(schema, {
* fieldMappings: { project: { "cm12345": "my-project-ref" } },
* });
*
* const results1 = await query1();
* const transformed1 = transform(results1);
*
* const results2 = await query2();
* const transformed2 = transform(results2);
* ```
*/
export function createResultTransformer(
schema: TableSchema[],
options: TransformResultsOptions = {}
): <T extends Record<string, unknown>>(rows: T[]) => T[] {
const columnTransformMaps = buildColumnTransformMaps(schema);
return <T extends Record<string, unknown>>(rows: T[]): T[] => {
if (columnTransformMaps.size === 0) {
return rows;
}
return rows.map((row) => transformRow(row, columnTransformMaps, options.fieldMappings));
};
}
internal-packages/tsql/src/query/schema.test.ts
+792
View File
@@ -0,0 +1,792 @@
import { describe, it, expect } from "vitest";
import {
column,
getUserFriendlyValue,
getInternalValue,
getAllowedUserValues,
isValidUserValue,
isVirtualColumn,
getVirtualColumnExpression,
hasFieldMapping,
getExternalValue,
getInternalValueFromMapping,
getInternalValueFromMappingCaseInsensitive,
sanitizeErrorMessage,
type ColumnSchema,
type FieldMappings,
type TableSchema,
} from "./schema.js";
describe("Value mapping helper functions", () => {
const columnWithValueMap: ColumnSchema = {
name: "status",
...column("String"),
valueMap: {
COMPLETED_SUCCESSFULLY: "Completed",
COMPLETED_WITH_ERRORS: "Completed with errors",
SYSTEM_FAILURE: "System failure",
PENDING: "Pending",
EXECUTING: "Running",
FAILED: "Failed",
},
};
const columnWithAllowedValues: ColumnSchema = {
name: "status",
...column("String"),
allowedValues: ["completed", "pending", "failed"],
};
const columnWithNoRestrictions: ColumnSchema = {
name: "task_identifier",
...column("String"),
};
describe("getUserFriendlyValue", () => {
it("should return user-friendly value for internal value", () => {
expect(getUserFriendlyValue(columnWithValueMap, "COMPLETED_SUCCESSFULLY")).toBe("Completed");
expect(getUserFriendlyValue(columnWithValueMap, "PENDING")).toBe("Pending");
expect(getUserFriendlyValue(columnWithValueMap, "EXECUTING")).toBe("Running");
});
it("should be case-insensitive for internal value lookup", () => {
expect(getUserFriendlyValue(columnWithValueMap, "completed_successfully")).toBe("Completed");
expect(getUserFriendlyValue(columnWithValueMap, "Completed_Successfully")).toBe("Completed");
expect(getUserFriendlyValue(columnWithValueMap, "COMPLETED_SUCCESSFULLY")).toBe("Completed");
});
it("should return original value if no mapping exists", () => {
expect(getUserFriendlyValue(columnWithValueMap, "UNKNOWN_STATUS")).toBe("UNKNOWN_STATUS");
});
it("should return original value if column has no valueMap", () => {
expect(getUserFriendlyValue(columnWithNoRestrictions, "any_value")).toBe("any_value");
});
});
describe("getInternalValue", () => {
it("should return internal value for user-friendly value", () => {
expect(getInternalValue(columnWithValueMap, "Completed")).toBe("COMPLETED_SUCCESSFULLY");
expect(getInternalValue(columnWithValueMap, "Pending")).toBe("PENDING");
expect(getInternalValue(columnWithValueMap, "Running")).toBe("EXECUTING");
});
it("should be case-insensitive for user-friendly value lookup", () => {
expect(getInternalValue(columnWithValueMap, "completed")).toBe("COMPLETED_SUCCESSFULLY");
expect(getInternalValue(columnWithValueMap, "COMPLETED")).toBe("COMPLETED_SUCCESSFULLY");
expect(getInternalValue(columnWithValueMap, "Completed")).toBe("COMPLETED_SUCCESSFULLY");
});
it("should return original value if no mapping exists", () => {
expect(getInternalValue(columnWithValueMap, "Unknown")).toBe("Unknown");
});
it("should return original value if column has no valueMap", () => {
expect(getInternalValue(columnWithNoRestrictions, "any_value")).toBe("any_value");
});
it("should handle multi-word user-friendly values", () => {
expect(getInternalValue(columnWithValueMap, "Completed with errors")).toBe(
"COMPLETED_WITH_ERRORS"
);
expect(getInternalValue(columnWithValueMap, "completed with errors")).toBe(
"COMPLETED_WITH_ERRORS"
);
expect(getInternalValue(columnWithValueMap, "System failure")).toBe("SYSTEM_FAILURE");
});
});
describe("getAllowedUserValues", () => {
it("should return user-friendly values from valueMap", () => {
const values = getAllowedUserValues(columnWithValueMap);
expect(values).toContain("Completed");
expect(values).toContain("Pending");
expect(values).toContain("Running");
expect(values).toContain("Failed");
expect(values).toContain("Completed with errors");
expect(values).toContain("System failure");
expect(values).toHaveLength(6);
});
it("should return allowedValues if no valueMap exists", () => {
const values = getAllowedUserValues(columnWithAllowedValues);
expect(values).toEqual(["completed", "pending", "failed"]);
});
it("should prefer valueMap over allowedValues", () => {
const columnWithBoth: ColumnSchema = {
name: "status",
...column("String"),
allowedValues: ["internal1", "internal2"],
valueMap: {
internal1: "User 1",
internal2: "User 2",
},
};
const values = getAllowedUserValues(columnWithBoth);
expect(values).toEqual(["User 1", "User 2"]);
});
it("should return empty array for column with no restrictions", () => {
const values = getAllowedUserValues(columnWithNoRestrictions);
expect(values).toEqual([]);
});
});
describe("isValidUserValue", () => {
it("should return true for valid user-friendly values", () => {
expect(isValidUserValue(columnWithValueMap, "Completed")).toBe(true);
expect(isValidUserValue(columnWithValueMap, "Pending")).toBe(true);
expect(isValidUserValue(columnWithValueMap, "Running")).toBe(true);
});
it("should be case-insensitive", () => {
expect(isValidUserValue(columnWithValueMap, "completed")).toBe(true);
expect(isValidUserValue(columnWithValueMap, "COMPLETED")).toBe(true);
expect(isValidUserValue(columnWithValueMap, "running")).toBe(true);
});
it("should return false for invalid values", () => {
expect(isValidUserValue(columnWithValueMap, "Unknown")).toBe(false);
expect(isValidUserValue(columnWithValueMap, "COMPLETED_SUCCESSFULLY")).toBe(false); // internal value, not user-friendly
});
it("should return true for any value if column has no restrictions", () => {
expect(isValidUserValue(columnWithNoRestrictions, "any_value")).toBe(true);
expect(isValidUserValue(columnWithNoRestrictions, "another")).toBe(true);
});
it("should validate against allowedValues if no valueMap", () => {
expect(isValidUserValue(columnWithAllowedValues, "completed")).toBe(true);
expect(isValidUserValue(columnWithAllowedValues, "COMPLETED")).toBe(true);
expect(isValidUserValue(columnWithAllowedValues, "unknown")).toBe(false);
});
it("should handle multi-word values", () => {
expect(isValidUserValue(columnWithValueMap, "Completed with errors")).toBe(true);
expect(isValidUserValue(columnWithValueMap, "completed with errors")).toBe(true);
expect(isValidUserValue(columnWithValueMap, "System failure")).toBe(true);
});
});
});
describe("Virtual column helper functions", () => {
const virtualColumn: ColumnSchema = {
name: "execution_duration",
...column("Nullable(Int64)"),
expression: "dateDiff('millisecond', started_at, completed_at)",
description: "Time between started_at and completed_at in milliseconds",
};
const regularColumn: ColumnSchema = {
name: "status",
...column("String"),
};
const columnWithEmptyExpression: ColumnSchema = {
name: "bad_column",
...column("String"),
expression: "",
};
describe("isVirtualColumn", () => {
it("should return true for columns with expression defined", () => {
expect(isVirtualColumn(virtualColumn)).toBe(true);
});
it("should return false for regular columns without expression", () => {
expect(isVirtualColumn(regularColumn)).toBe(false);
});
it("should return false for columns with empty expression", () => {
expect(isVirtualColumn(columnWithEmptyExpression)).toBe(false);
});
it("should return false for columns with undefined expression", () => {
const col: ColumnSchema = {
name: "test",
...column("String"),
expression: undefined,
};
expect(isVirtualColumn(col)).toBe(false);
});
});
describe("getVirtualColumnExpression", () => {
it("should return the expression for virtual columns", () => {
expect(getVirtualColumnExpression(virtualColumn)).toBe(
"dateDiff('millisecond', started_at, completed_at)"
);
});
it("should return undefined for regular columns", () => {
expect(getVirtualColumnExpression(regularColumn)).toBeUndefined();
});
it("should return undefined for columns with empty expression", () => {
expect(getVirtualColumnExpression(columnWithEmptyExpression)).toBeUndefined();
});
});
describe("virtual column schema definition", () => {
it("should allow defining virtual columns with all standard column options", () => {
const virtualWithOptions: ColumnSchema = {
name: "computed_value",
type: "Float64",
expression: "usage_duration_ms / 1000.0",
selectable: true,
filterable: true,
sortable: true,
groupable: false, // Might not want to group by computed values
description: "Usage duration in seconds",
};
expect(isVirtualColumn(virtualWithOptions)).toBe(true);
expect(virtualWithOptions.groupable).toBe(false);
expect(virtualWithOptions.selectable).toBe(true);
});
it("should support complex expressions with ClickHouse functions", () => {
const complexVirtual: ColumnSchema = {
name: "is_long_running",
...column("UInt8"),
expression:
"if(completed_at IS NOT NULL AND started_at IS NOT NULL, dateDiff('second', started_at, completed_at) > 60, 0)",
};
expect(isVirtualColumn(complexVirtual)).toBe(true);
expect(getVirtualColumnExpression(complexVirtual)).toContain("dateDiff");
expect(getVirtualColumnExpression(complexVirtual)).toContain("if(");
});
});
});
describe("Field mapping helper functions (runtime dynamic mappings)", () => {
const fieldMappings: FieldMappings = {
project: {
cm12345: "my-project-ref",
cm67890: "other-project",
cmABCDE: "Mixed-Case-Project",
},
environment: {
env123: "production",
env456: "staging",
},
};
const columnWithFieldMapping: ColumnSchema = {
name: "project_ref",
clickhouseName: "project_id",
...column("String"),
fieldMapping: "project",
};
const columnWithoutFieldMapping: ColumnSchema = {
name: "status",
...column("String"),
};
const columnWithEmptyFieldMapping: ColumnSchema = {
name: "test",
...column("String"),
fieldMapping: "",
};
describe("hasFieldMapping", () => {
it("should return true for columns with fieldMapping defined", () => {
expect(hasFieldMapping(columnWithFieldMapping)).toBe(true);
});
it("should return false for columns without fieldMapping", () => {
expect(hasFieldMapping(columnWithoutFieldMapping)).toBe(false);
});
it("should return false for columns with empty fieldMapping", () => {
expect(hasFieldMapping(columnWithEmptyFieldMapping)).toBe(false);
});
it("should return false for columns with undefined fieldMapping", () => {
const col: ColumnSchema = {
name: "test",
...column("String"),
fieldMapping: undefined,
};
expect(hasFieldMapping(col)).toBe(false);
});
});
describe("getExternalValue", () => {
it("should return external value for internal value", () => {
expect(getExternalValue(fieldMappings, "project", "cm12345")).toBe("my-project-ref");
expect(getExternalValue(fieldMappings, "project", "cm67890")).toBe("other-project");
expect(getExternalValue(fieldMappings, "environment", "env123")).toBe("production");
});
it("should return null if internal value is not found in mapping", () => {
expect(getExternalValue(fieldMappings, "project", "unknown_id")).toBeNull();
expect(getExternalValue(fieldMappings, "environment", "unknown_env")).toBeNull();
});
it("should return null if mapping name does not exist", () => {
expect(getExternalValue(fieldMappings, "nonexistent", "cm12345")).toBeNull();
});
it("should return null for empty mappings", () => {
expect(getExternalValue({}, "project", "cm12345")).toBeNull();
});
it("should be case-sensitive for internal values", () => {
// Internal IDs should be matched exactly
expect(getExternalValue(fieldMappings, "project", "CM12345")).toBeNull();
expect(getExternalValue(fieldMappings, "project", "cm12345")).toBe("my-project-ref");
});
});
describe("getInternalValueFromMapping", () => {
it("should return internal value for external value", () => {
expect(getInternalValueFromMapping(fieldMappings, "project", "my-project-ref")).toBe(
"cm12345"
);
expect(getInternalValueFromMapping(fieldMappings, "project", "other-project")).toBe(
"cm67890"
);
expect(getInternalValueFromMapping(fieldMappings, "environment", "production")).toBe(
"env123"
);
});
it("should return null if external value is not found", () => {
expect(getInternalValueFromMapping(fieldMappings, "project", "unknown-ref")).toBeNull();
});
it("should return null if mapping name does not exist", () => {
expect(
getInternalValueFromMapping(fieldMappings, "nonexistent", "my-project-ref")
).toBeNull();
});
it("should return null for empty mappings", () => {
expect(getInternalValueFromMapping({}, "project", "my-project-ref")).toBeNull();
});
it("should be case-sensitive for external values", () => {
// This is the case-sensitive version
expect(getInternalValueFromMapping(fieldMappings, "project", "MY-PROJECT-REF")).toBeNull();
expect(getInternalValueFromMapping(fieldMappings, "project", "my-project-ref")).toBe(
"cm12345"
);
});
});
describe("getInternalValueFromMappingCaseInsensitive", () => {
it("should return internal value for external value (case-insensitive)", () => {
expect(
getInternalValueFromMappingCaseInsensitive(fieldMappings, "project", "my-project-ref")
).toBe("cm12345");
expect(
getInternalValueFromMappingCaseInsensitive(fieldMappings, "project", "MY-PROJECT-REF")
).toBe("cm12345");
expect(
getInternalValueFromMappingCaseInsensitive(fieldMappings, "project", "My-Project-Ref")
).toBe("cm12345");
});
it("should return null if external value is not found (even case-insensitive)", () => {
expect(
getInternalValueFromMappingCaseInsensitive(fieldMappings, "project", "unknown-ref")
).toBeNull();
});
it("should return null if mapping name does not exist", () => {
expect(
getInternalValueFromMappingCaseInsensitive(fieldMappings, "nonexistent", "my-project-ref")
).toBeNull();
});
it("should return null for empty mappings", () => {
expect(
getInternalValueFromMappingCaseInsensitive({}, "project", "my-project-ref")
).toBeNull();
});
it("should handle mixed case external values correctly", () => {
expect(
getInternalValueFromMappingCaseInsensitive(fieldMappings, "project", "mixed-case-project")
).toBe("cmABCDE");
expect(
getInternalValueFromMappingCaseInsensitive(fieldMappings, "project", "MIXED-CASE-PROJECT")
).toBe("cmABCDE");
});
});
describe("field mapping column schema definition", () => {
it("should allow defining columns with fieldMapping", () => {
const col: ColumnSchema = {
name: "project_ref",
clickhouseName: "project_id",
type: "String",
fieldMapping: "project",
description: "Project reference (external identifier)",
};
expect(hasFieldMapping(col)).toBe(true);
expect(col.clickhouseName).toBe("project_id");
expect(col.fieldMapping).toBe("project");
});
it("should allow fieldMapping with all standard column options", () => {
const col: ColumnSchema = {
name: "project_ref",
clickhouseName: "project_id",
...column("String"),
fieldMapping: "project",
selectable: true,
filterable: true,
sortable: true,
groupable: true,
};
expect(hasFieldMapping(col)).toBe(true);
expect(col.selectable).toBe(true);
expect(col.filterable).toBe(true);
});
});
});
describe("Error message sanitization", () => {
// Test schema mimicking the real runs schema
const runsSchema: TableSchema = {
name: "runs",
clickhouseName: "trigger_dev.task_runs_v2",
description: "Task runs table",
tenantColumns: {
organizationId: "organization_id",
projectId: "project_id",
environmentId: "environment_id",
},
columns: {
run_id: {
name: "run_id",
clickhouseName: "friendly_id",
...column("String"),
},
triggered_at: {
name: "triggered_at",
clickhouseName: "created_at",
...column("DateTime64"),
},
machine: {
name: "machine",
clickhouseName: "machine_preset",
...column("String"),
},
status: {
name: "status",
// No clickhouseName - same as name
...column("String"),
},
task_identifier: {
name: "task_identifier",
// No clickhouseName - same as name
...column("String"),
},
},
};
describe("sanitizeErrorMessage", () => {
it("should replace fully qualified table.column references", () => {
const error = "Missing column trigger_dev.task_runs_v2.friendly_id in query";
const sanitized = sanitizeErrorMessage(error, [runsSchema]);
expect(sanitized).toBe("Missing column runs.run_id in query");
});
it("should replace standalone table names", () => {
const error = "Table trigger_dev.task_runs_v2 does not exist";
const sanitized = sanitizeErrorMessage(error, [runsSchema]);
expect(sanitized).toBe("Table runs does not exist");
});
it("should replace standalone column names with different clickhouseName", () => {
const error = "Unknown identifier: friendly_id";
const sanitized = sanitizeErrorMessage(error, [runsSchema]);
expect(sanitized).toBe("Unknown identifier: run_id");
});
it("should replace multiple occurrences in the same message", () => {
const error =
"Cannot compare friendly_id with created_at: incompatible types in trigger_dev.task_runs_v2";
const sanitized = sanitizeErrorMessage(error, [runsSchema]);
expect(sanitized).toBe("Cannot compare run_id with triggered_at: incompatible types in runs");
});
it("should not replace column names that have no clickhouseName mapping", () => {
const error = "Invalid value for column status";
const sanitized = sanitizeErrorMessage(error, [runsSchema]);
expect(sanitized).toBe("Invalid value for column status");
});
it("should handle error messages with quoted identifiers", () => {
const error = "Column 'machine_preset' is not of type String";
const sanitized = sanitizeErrorMessage(error, [runsSchema]);
expect(sanitized).toBe("Column 'machine' is not of type String");
});
it("should handle error messages with backtick identifiers", () => {
const error = "Unknown column `friendly_id` in table";
const sanitized = sanitizeErrorMessage(error, [runsSchema]);
expect(sanitized).toBe("Unknown column `run_id` in table");
});
it("should not replace partial matches within larger identifiers", () => {
const error = "Column my_friendly_id_column not found";
const sanitized = sanitizeErrorMessage(error, [runsSchema]);
// Should not replace "friendly_id" within "my_friendly_id_column"
expect(sanitized).toBe("Column my_friendly_id_column not found");
});
it("should return original message if no schemas provided", () => {
const error = "Some error with trigger_dev.task_runs_v2";
const sanitized = sanitizeErrorMessage(error, []);
expect(sanitized).toBe("Some error with trigger_dev.task_runs_v2");
});
it("should return original message if no matches found", () => {
const error = "Generic database error occurred";
const sanitized = sanitizeErrorMessage(error, [runsSchema]);
expect(sanitized).toBe("Generic database error occurred");
});
it("should handle multiple tables", () => {
const eventsSchema: TableSchema = {
name: "events",
clickhouseName: "trigger_dev.task_events",
description: "Task events table",
tenantColumns: {
organizationId: "organization_id",
projectId: "project_id",
environmentId: "environment_id",
},
columns: {
event_id: {
name: "event_id",
clickhouseName: "internal_event_id",
...column("String"),
},
},
};
const error =
"Cannot join trigger_dev.task_runs_v2 with trigger_dev.task_events on internal_event_id";
const sanitized = sanitizeErrorMessage(error, [runsSchema, eventsSchema]);
expect(sanitized).toBe("Cannot join runs with events on event_id");
});
it("should handle real ClickHouse error format", () => {
const error =
"Unable to query clickhouse: Code: 47. DB::Exception: Missing columns: 'friendly_id' while processing query";
const sanitized = sanitizeErrorMessage(error, [runsSchema]);
// Should remove the "Unable to query clickhouse:" prefix
expect(sanitized).toBe(
"Code: 47. DB::Exception: Missing columns: 'run_id' while processing query"
);
});
it("should remove 'Unable to query clickhouse:' prefix", () => {
const error = "Unable to query clickhouse: Something went wrong";
const sanitized = sanitizeErrorMessage(error, [runsSchema]);
expect(sanitized).toBe("Something went wrong");
expect(sanitized).not.toContain("Unable to query clickhouse");
});
it("should handle error with column in parentheses", () => {
const error = "Function count(friendly_id) expects different arguments";
const sanitized = sanitizeErrorMessage(error, [runsSchema]);
expect(sanitized).toBe("Function count(run_id) expects different arguments");
});
it("should handle error with column after comma", () => {
const error = "SELECT friendly_id, created_at FROM table";
const sanitized = sanitizeErrorMessage(error, [runsSchema]);
expect(sanitized).toBe("SELECT run_id, triggered_at FROM table");
});
it("should NOT replace column names in 'Did you mean' error messages", () => {
// When a user types an internal ClickHouse column name, we show "Did you mean X?"
// The sanitizer should NOT replace the column name in this case, as it would
// turn "Unknown column 'created_at'. Did you mean 'triggered_at'?" into
// "Unknown column 'triggered_at'. Did you mean 'triggered_at'?" which is confusing
const error = 'Unknown column "created_at". Did you mean "triggered_at"?';
const sanitized = sanitizeErrorMessage(error, [runsSchema]);
// Should preserve both the original column name AND the suggestion
expect(sanitized).toBe('Unknown column "created_at". Did you mean "triggered_at"?');
});
it("should prioritize longer matches (table.column before standalone column)", () => {
// This tests that we replace "trigger_dev.task_runs_v2.friendly_id" as a unit,
// not "trigger_dev.task_runs_v2" and then "friendly_id" separately
const error = "Error in trigger_dev.task_runs_v2.friendly_id";
const sanitized = sanitizeErrorMessage(error, [runsSchema]);
expect(sanitized).toBe("Error in runs.run_id");
});
it("should remove tenant isolation filters from error messages", () => {
const error =
"Unknown identifier in scope SELECT run_id FROM runs WHERE ((organization_id = 'org123') AND (project_id = 'proj456') AND (environment_id = 'env789')) AND (status = 'Failed')";
const sanitized = sanitizeErrorMessage(error, [runsSchema]);
expect(sanitized).not.toContain("organization_id");
expect(sanitized).not.toContain("project_id");
expect(sanitized).not.toContain("environment_id");
expect(sanitized).toContain("status = 'Failed'");
});
it("should remove redundant column aliases like 'run_id AS run_id'", () => {
const error = "Error in SELECT run_id AS run_id, machine AS machine FROM runs";
const sanitized = sanitizeErrorMessage(error, [runsSchema]);
expect(sanitized).toBe("Error in SELECT run_id, machine FROM runs");
});
it("should remove redundant table aliases like 'runs AS runs'", () => {
const error = "Error in SELECT * FROM runs AS runs WHERE status = 'Failed'";
const sanitized = sanitizeErrorMessage(error, [runsSchema]);
expect(sanitized).toBe("Error in SELECT * FROM runs WHERE status = 'Failed'");
});
it("should handle real error with tenant filters and aliases", () => {
const error =
"Unable to query clickhouse: Unknown expression identifier `triggered_ata` in scope SELECT run_id AS run_id, machine AS machine FROM runs AS runs WHERE ((organization_id = 'cm5qtzpb800007cp7h6ebwt2i') AND (project_id = 'cme2p1yep00007calt8ugarkr') AND (environment_id = 'cme2p1ygj00027caln51kyiwl')) AND (status = 'Complted') ORDER BY triggered_ata DESC LIMIT 100.";
const sanitized = sanitizeErrorMessage(error, [runsSchema]);
// Should not contain internal prefix
expect(sanitized).not.toContain("Unable to query clickhouse");
// Should not contain internal IDs
expect(sanitized).not.toContain("cm5qtzpb800007cp7h6ebwt2i");
expect(sanitized).not.toContain("cme2p1yep00007calt8ugarkr");
expect(sanitized).not.toContain("cme2p1ygj00027caln51kyiwl");
expect(sanitized).not.toContain("organization_id");
expect(sanitized).not.toContain("project_id");
expect(sanitized).not.toContain("environment_id");
// Should not have redundant aliases
expect(sanitized).not.toContain("run_id AS run_id");
expect(sanitized).not.toContain("machine AS machine");
expect(sanitized).not.toContain("runs AS runs");
// Should still have the user's query parts
expect(sanitized).toContain("status = 'Complted'");
expect(sanitized).toContain("triggered_ata");
expect(sanitized).toContain("LIMIT 100");
});
it("should remove required filters like engine = 'V2'", () => {
// Schema with required filters
const schemaWithRequiredFilters: TableSchema = {
name: "runs",
clickhouseName: "trigger_dev.task_runs_v2",
description: "Task runs table",
tenantColumns: {
organizationId: "organization_id",
projectId: "project_id",
environmentId: "environment_id",
},
requiredFilters: [{ column: "engine", value: "V2" }],
columns: {
run_id: {
name: "run_id",
...column("String"),
},
},
};
const error =
"Error in SELECT run_id FROM runs WHERE ((organization_id = 'org1') AND (engine = 'V2')) AND (status = 'Failed')";
const sanitized = sanitizeErrorMessage(error, [schemaWithRequiredFilters]);
expect(sanitized).not.toContain("engine = 'V2'");
expect(sanitized).not.toContain("organization_id");
expect(sanitized).toContain("status = 'Failed'");
});
it("should handle project and environment field mappings in tenant columns", () => {
// The schema uses 'project' and 'environment' as column names with field mappings
const schemaWithFieldMappedTenants: TableSchema = {
name: "runs",
clickhouseName: "trigger_dev.task_runs_v2",
description: "Task runs table",
tenantColumns: {
organizationId: "organization_id",
projectId: "project",
environmentId: "environment",
},
columns: {
run_id: {
name: "run_id",
...column("String"),
},
},
};
const error =
"Error WHERE ((organization_id = 'org1') AND (project = 'proj1') AND (environment = 'env1')) AND (status = 'ok')";
const sanitized = sanitizeErrorMessage(error, [schemaWithFieldMappedTenants]);
expect(sanitized).not.toContain("organization_id = 'org1'");
expect(sanitized).not.toContain("project = 'proj1'");
expect(sanitized).not.toContain("environment = 'env1'");
expect(sanitized).toContain("status = 'ok'");
});
it("should handle queries with only automatic WHERE filters (no user WHERE clause)", () => {
// When user writes: SELECT * FROM runs LIMIT 10
// The compiled query becomes: SELECT * FROM runs WHERE (org_id = '...') AND (proj_id = '...') AND (env_id = '...')
const error =
"Unable to query clickhouse: Some error in SELECT run_id FROM runs WHERE ((organization_id = 'org1') AND (project_id = 'proj1') AND (environment_id = 'env1')) LIMIT 10";
const sanitized = sanitizeErrorMessage(error, [runsSchema]);
expect(sanitized).not.toContain("Unable to query clickhouse");
expect(sanitized).not.toContain("organization_id");
expect(sanitized).not.toContain("project_id");
expect(sanitized).not.toContain("environment_id");
expect(sanitized).not.toContain("WHERE");
expect(sanitized).toContain("SELECT run_id FROM runs");
expect(sanitized).toContain("LIMIT 10");
});
it("should handle queries with only automatic filters including engine filter", () => {
const schemaWithEngine: TableSchema = {
name: "runs",
clickhouseName: "trigger_dev.task_runs_v2",
description: "Task runs table",
tenantColumns: {
organizationId: "organization_id",
projectId: "project_id",
environmentId: "environment_id",
},
requiredFilters: [{ column: "engine", value: "V2" }],
columns: {
run_id: {
name: "run_id",
...column("String"),
},
},
};
const error =
"Error in SELECT * FROM runs WHERE ((organization_id = 'org1') AND (project_id = 'proj1') AND (environment_id = 'env1') AND (engine = 'V2')) ORDER BY run_id";
const sanitized = sanitizeErrorMessage(error, [schemaWithEngine]);
expect(sanitized).not.toContain("organization_id");
expect(sanitized).not.toContain("project_id");
expect(sanitized).not.toContain("environment_id");
expect(sanitized).not.toContain("engine");
expect(sanitized).not.toContain("WHERE");
expect(sanitized).toContain("SELECT * FROM runs");
expect(sanitized).toContain("ORDER BY run_id");
});
});
});
internal-packages/tsql/src/query/schema.ts
+1017
View File
File diff suppressed because it is too large Load Diff
internal-packages/tsql/src/query/security.test.ts
+718
View File
@@ -0,0 +1,718 @@
/**
* Security Tests for TSQL
*
* These tests verify that the TSQL parser and printer correctly prevent:
* 1. Cross-tenant data access
* 2. SQL injection attacks
*/
import { describe, expect, it } from "vitest";
import { compileTSQL, type CompileTSQLOptions } from "../index.js";
import { column, type TableSchema } from "./schema.js";
/**
* Test schemas
*/
const taskRunsSchema: TableSchema = {
name: "task_runs",
clickhouseName: "trigger_dev.task_runs_v2",
columns: {
id: { name: "id", ...column("String") },
status: { name: "status", ...column("String") },
task_identifier: { name: "task_identifier", ...column("String") },
created_at: { name: "created_at", ...column("DateTime64") },
duration_ms: { name: "duration_ms", ...column("Nullable(UInt64)") },
organization_id: { name: "organization_id", ...column("String") },
project_id: { name: "project_id", ...column("String") },
environment_id: { name: "environment_id", ...column("String") },
payload: { name: "payload", ...column("String") },
},
tenantColumns: {
organizationId: "organization_id",
projectId: "project_id",
environmentId: "environment_id",
},
};
const taskEventsSchema: TableSchema = {
name: "task_events",
clickhouseName: "trigger_dev.task_events_v2",
columns: {
id: { name: "id", ...column("String") },
run_id: { name: "run_id", ...column("String") },
event_type: { name: "event_type", ...column("String") },
timestamp: { name: "timestamp", ...column("DateTime64") },
organization_id: { name: "organization_id", ...column("String") },
project_id: { name: "project_id", ...column("String") },
environment_id: { name: "environment_id", ...column("String") },
},
tenantColumns: {
organizationId: "organization_id",
projectId: "project_id",
environmentId: "environment_id",
},
};
const defaultOptions: CompileTSQLOptions = {
tableSchema: [taskRunsSchema, taskEventsSchema],
enforcedWhereClause: {
organization_id: { op: "eq", value: "org_tenant1" },
project_id: { op: "eq", value: "proj_tenant1" },
environment_id: { op: "eq", value: "env_tenant1" },
},
};
function compile(query: string, options: Partial<CompileTSQLOptions> = {}) {
return compileTSQL(query, { ...defaultOptions, ...options });
}
describe("Cross-Tenant Security", () => {
describe("Tenant guards are always injected", () => {
it("should inject tenant guards on simple SELECT", () => {
const { sql, params } = compile("SELECT * FROM task_runs");
// Must contain all three tenant columns
expect(sql).toContain("organization_id");
expect(sql).toContain("project_id");
expect(sql).toContain("environment_id");
// Tenant values must be parameterized
expect(Object.values(params)).toContain("org_tenant1");
expect(Object.values(params)).toContain("proj_tenant1");
expect(Object.values(params)).toContain("env_tenant1");
});
it("should inject tenant guards even with user WHERE clause", () => {
const { sql, params } = compile("SELECT * FROM task_runs WHERE status = 'completed'");
// Must still have tenant guards
expect(sql).toContain("organization_id");
expect(sql).toContain("project_id");
expect(sql).toContain("environment_id");
expect(Object.values(params)).toContain("org_tenant1");
});
it("should inject tenant guards on all tables in JOIN", () => {
const { sql } = compile(`
SELECT r.id, e.event_type
FROM task_runs r
JOIN task_events e ON r.id = e.run_id
`);
// Both tables should have tenant guards
// Count occurrences of organization_id - should appear twice (once per table)
const orgIdMatches = sql.match(/organization_id/g) || [];
expect(orgIdMatches.length).toBeGreaterThanOrEqual(2);
});
it("should inject tenant guards in subqueries", () => {
const { sql } = compile(`
SELECT * FROM task_runs
WHERE id IN (SELECT run_id FROM task_events WHERE event_type = 'completed')
`);
// Should have tenant guards in both main query and subquery
const orgIdMatches = sql.match(/organization_id/g) || [];
expect(orgIdMatches.length).toBeGreaterThanOrEqual(2);
});
it("should inject tenant guards on UNION queries", () => {
const { sql } = compile(`
SELECT id, status FROM task_runs WHERE status = 'completed'
UNION ALL
SELECT id, status FROM task_runs WHERE status = 'failed'
`);
// Both sides of UNION should have tenant guards
const orgIdMatches = sql.match(/organization_id/g) || [];
expect(orgIdMatches.length).toBeGreaterThanOrEqual(2);
});
});
describe("Cannot bypass tenant guards", () => {
it("should not allow OR clause to bypass tenant guard", () => {
const { sql, params } = compile("SELECT * FROM task_runs WHERE status = 'completed' OR 1=1");
// The tenant guards should be ANDed with the entire WHERE clause
// So even with OR 1=1, the tenant guard still applies
expect(sql).toContain("organization_id");
expect(Object.values(params)).toContain("org_tenant1");
// The structure should be: and(tenant_guards, or(user_conditions))
// The user's OR should be nested inside the outer AND with tenant guards
// This ensures tenant_guard AND (status='completed' OR 1=1)
// NOT: tenant_guard AND status='completed' OR 1=1 (which would bypass)
// Verify the OR is contained within an outer AND structure
// The tenant guards use and() and the user's OR uses or()
expect(sql).toContain("or(");
expect(sql).toContain("and(");
// The and() should wrap everything - find where tenant columns appear
// They should be at the same level as the user's condition, both inside and()
const whereClause = sql.substring(sql.indexOf("WHERE"));
expect(whereClause).toMatch(/and\([^)]*organization_id/);
});
it("should not allow accessing other tenant's data via explicit condition", () => {
const { sql: _sql, params } = compile(
"SELECT * FROM task_runs WHERE organization_id = 'org_other_tenant'"
);
// Even if user specifies a different org_id, our tenant guard should override
// The compiled SQL should still use our tenant's ID
expect(Object.values(params)).toContain("org_tenant1");
});
it("should not allow UNION with unguarded query", () => {
// This should be rejected or the second part should still be guarded
const { sql } = compile(`
SELECT id FROM task_runs
UNION ALL
SELECT id FROM task_runs
`);
// Both parts must have tenant guards
const orgIdMatches = sql.match(/organization_id/g) || [];
expect(orgIdMatches.length).toBeGreaterThanOrEqual(2);
});
it("should not allow subquery to access other tenants", () => {
const { sql, params } = compile(`
SELECT * FROM task_runs
WHERE id IN (
SELECT run_id FROM task_events
)
`);
// Subquery must also have tenant guards
expect(Object.values(params)).toContain("org_tenant1");
const orgIdMatches = sql.match(/organization_id/g) || [];
expect(orgIdMatches.length).toBeGreaterThanOrEqual(2);
});
});
describe("Table allowlisting", () => {
it("should reject queries to unknown tables", () => {
expect(() => {
compile("SELECT * FROM users");
}).toThrow();
});
it("should reject queries to system tables", () => {
expect(() => {
compile("SELECT * FROM system.tables");
}).toThrow();
});
it("should reject queries trying to use database prefix", () => {
expect(() => {
compile("SELECT * FROM other_database.task_runs");
}).toThrow();
});
});
});
describe("SQL Injection Prevention", () => {
describe("String value injection", () => {
it("should reject queries with stacked statements in strings", () => {
// The parser correctly rejects this at parse time
expect(() => {
compile("SELECT * FROM task_runs WHERE status = 'completed'; DROP TABLE task_runs; --'");
}).toThrow();
});
it("should parameterize malicious-looking string values", () => {
const { sql, params } = compile("SELECT * FROM task_runs WHERE status = 'DROP TABLE users'");
// The malicious payload should be in params, not in SQL
expect(sql).not.toContain("DROP TABLE");
expect(Object.values(params)).toContain("DROP TABLE users");
});
it("should handle quote escape attempts", () => {
const { sql: _sql, params } = compile(
"SELECT * FROM task_runs WHERE status = 'test''injection'"
);
// Should be safely parameterized
expect(Object.values(params).some((v) => typeof v === "string")).toBe(true);
});
it("should handle backslash escape attempts", () => {
const { sql, params: _params } = compile(
"SELECT * FROM task_runs WHERE status = 'test\\'injection'"
);
// Should be safely parameterized
expect(sql).not.toContain("injection'");
});
it("should handle unicode characters in strings", () => {
const { sql: _sql, params } = compile(
"SELECT * FROM task_runs WHERE status = 'test™injection'"
);
// Should be safely parameterized
expect(Object.values(params).some((v) => typeof v === "string")).toBe(true);
});
it("should handle null byte injection", () => {
const { sql: _sql, params } = compile(
"SELECT * FROM task_runs WHERE status = 'test\\0injection'"
);
expect(Object.values(params).some((v) => typeof v === "string")).toBe(true);
});
});
describe("Comment injection", () => {
it("should not allow -- comments to truncate query", () => {
// The parser should either reject this or handle it safely
const result = compile("SELECT * FROM task_runs WHERE status = 'completed'");
// Tenant guards must still be present
expect(result.sql).toContain("organization_id");
});
it("should not allow /* */ comments for injection", () => {
const result = compile("SELECT * FROM task_runs WHERE status = 'completed'");
// Tenant guards must still be present
expect(result.sql).toContain("organization_id");
});
});
describe("Identifier injection", () => {
it("should reject identifiers with backtick injection", () => {
expect(() => {
compile("SELECT * FROM task_runs WHERE `status`; DROP TABLE users; --` = 'test'");
}).toThrow();
});
it("should not expose column names that could be used for injection", () => {
// Column names in the output are validated identifiers from the schema
// Malicious column names would need to be in the schema first
const { sql } = compile("SELECT id, status FROM task_runs");
// Column names should be simple identifiers without injection
expect(sql).toContain("id");
expect(sql).toContain("status");
expect(sql).not.toContain(";");
});
it("should reject table names with special characters", () => {
expect(() => {
compile("SELECT * FROM `task_runs; DROP TABLE users`");
}).toThrow();
});
});
describe("Numeric injection", () => {
it("should handle numeric values safely", () => {
const { sql } = compile("SELECT * FROM task_runs WHERE duration_ms > 1000");
// Numbers should be safely inlined or parameterized
expect(sql).toContain("1000");
expect(sql).not.toContain(";");
});
it("should handle negative numbers safely", () => {
const { sql } = compile("SELECT * FROM task_runs WHERE duration_ms > -1");
expect(sql).not.toContain(";");
});
it("should handle floating point safely", () => {
const { sql } = compile("SELECT * FROM task_runs WHERE duration_ms > 1.5");
expect(sql).toContain("1.5");
});
});
describe("Function injection", () => {
it("should only allow known safe functions", () => {
// Unknown functions should be rejected
expect(() => {
compile("SELECT file('/etc/passwd') FROM task_runs");
}).toThrow();
});
it("should reject system functions", () => {
expect(() => {
compile("SELECT system.tables() FROM task_runs");
}).toThrow();
});
it("should allow known aggregate functions", () => {
const { sql } = compile("SELECT count(*), sum(duration_ms) FROM task_runs");
expect(sql).toContain("count(*)");
expect(sql).toContain("sum(duration_ms)");
});
});
describe("Stacked query prevention", () => {
it("should not allow semicolon to start new statement", () => {
expect(() => {
compile("SELECT * FROM task_runs; DELETE FROM task_runs");
}).toThrow();
});
it("should not allow multiple statements", () => {
expect(() => {
compile("SELECT * FROM task_runs; SELECT * FROM task_events");
}).toThrow();
});
});
describe("UNION-based injection", () => {
it("should apply tenant guards to all UNION parts", () => {
const { sql, params } = compile(`
SELECT id, status FROM task_runs WHERE status = 'a'
UNION ALL
SELECT id, status FROM task_runs WHERE status = 'b'
`);
// Both parts should have tenant guards
expect(Object.values(params)).toContain("org_tenant1");
const orgIdMatches = sql.match(/organization_id/g) || [];
expect(orgIdMatches.length).toBeGreaterThanOrEqual(2);
});
});
});
describe("Parameter Safety", () => {
it("should use typed parameters", () => {
const { sql } = compile("SELECT * FROM task_runs WHERE status = 'test'");
// Parameters should have type annotations like {param: String}
expect(sql).toMatch(/\{tsql_\w+: \w+\}/);
});
it("should generate unique parameter names", () => {
const { params } = compile(`
SELECT * FROM task_runs
WHERE status = 'a' AND task_identifier = 'b' AND payload = 'c'
`);
// All parameter keys should be unique
const keys = Object.keys(params);
expect(new Set(keys).size).toBe(keys.length);
});
it("should not include raw values in SQL for strings", () => {
const { sql } = compile("SELECT * FROM task_runs WHERE status = 'user_provided_value'");
// The literal string should not appear in SQL
expect(sql).not.toContain("user_provided_value");
});
});
describe("Optional Tenant Filters", () => {
/**
* Helper to extract the WHERE clause from SQL for more precise testing.
* This is needed because SELECT * expansion includes all columns,
* but we only want to check what's in the WHERE clause for tenant filtering.
*/
function getWhereClause(sql: string): string {
const whereMatch = sql.match(/WHERE\s+(.*?)(?:\s+(?:ORDER|GROUP|LIMIT|$))/is);
return whereMatch ? whereMatch[1] : "";
}
describe("Organization ID is always required", () => {
it("should always inject organization guard even with optional project/env", () => {
const { sql, params } = compile("SELECT * FROM task_runs", {
enforcedWhereClause: {
organization_id: { op: "eq", value: "org_tenant1" },
// project_id and environment_id omitted
},
});
const whereClause = getWhereClause(sql);
// Must contain organization_id in WHERE clause
expect(whereClause).toContain("organization_id");
expect(Object.values(params)).toContain("org_tenant1");
// Should NOT contain project_id or environment_id guards in WHERE clause
expect(whereClause).not.toContain("project_id");
expect(whereClause).not.toContain("environment_id");
});
});
describe("Project ID is optional", () => {
it("should inject org and project guards when project is provided", () => {
const { sql, params } = compile("SELECT * FROM task_runs", {
enforcedWhereClause: {
organization_id: { op: "eq", value: "org_tenant1" },
project_id: { op: "eq", value: "proj_tenant1" },
// environment_id omitted
},
});
const whereClause = getWhereClause(sql);
// Must contain organization_id and project_id in WHERE clause
expect(whereClause).toContain("organization_id");
expect(whereClause).toContain("project_id");
expect(Object.values(params)).toContain("org_tenant1");
expect(Object.values(params)).toContain("proj_tenant1");
// Should NOT contain environment_id guard in WHERE clause
expect(whereClause).not.toContain("environment_id");
});
it("should allow querying across all projects when projectId is omitted", () => {
const { sql, params } = compile("SELECT * FROM task_runs", {
enforcedWhereClause: {
organization_id: { op: "eq", value: "org_tenant1" },
// project_id and environment_id omitted
},
});
const whereClause = getWhereClause(sql);
// Only org guard should be present in WHERE clause
expect(whereClause).toContain("organization_id");
expect(Object.values(params)).toContain("org_tenant1");
expect(whereClause).not.toContain("project_id");
});
});
describe("Environment ID is optional", () => {
it("should inject org, project, and env guards when all provided", () => {
const { sql, params } = compile("SELECT * FROM task_runs");
const whereClause = getWhereClause(sql);
// All three should be present in WHERE clause (default options include all)
expect(whereClause).toContain("organization_id");
expect(whereClause).toContain("project_id");
expect(whereClause).toContain("environment_id");
expect(Object.values(params)).toContain("org_tenant1");
expect(Object.values(params)).toContain("proj_tenant1");
expect(Object.values(params)).toContain("env_tenant1");
});
it("should allow querying across all environments when environmentId is omitted", () => {
const { sql, params } = compile("SELECT * FROM task_runs", {
enforcedWhereClause: {
organization_id: { op: "eq", value: "org_tenant1" },
project_id: { op: "eq", value: "proj_tenant1" },
// environment_id omitted
},
});
const whereClause = getWhereClause(sql);
// Org and project guards should be present in WHERE clause
expect(whereClause).toContain("organization_id");
expect(whereClause).toContain("project_id");
expect(Object.values(params)).toContain("org_tenant1");
expect(Object.values(params)).toContain("proj_tenant1");
// Environment guard should NOT be present in WHERE clause
expect(whereClause).not.toContain("environment_id");
});
});
describe("Cross-tenant security with optional filters", () => {
it("should still prevent cross-org access with org-only filter", () => {
const { sql: _sql, params } = compile(
"SELECT * FROM task_runs WHERE organization_id = 'org_other'",
{
enforcedWhereClause: {
organization_id: { op: "eq", value: "org_tenant1" },
// project_id and environment_id omitted
},
}
);
// Our org guard should still be enforced
expect(Object.values(params)).toContain("org_tenant1");
});
it("should apply org guard to all tables in JOIN when using org-only filter", () => {
const { sql } = compile(
`
SELECT r.id, e.event_type
FROM task_runs r
JOIN task_events e ON r.id = e.run_id
`,
{
enforcedWhereClause: {
organization_id: { op: "eq", value: "org_tenant1" },
// project_id and environment_id omitted
},
}
);
// Both tables should have org guards
const orgIdMatches = sql.match(/organization_id/g) || [];
expect(orgIdMatches.length).toBeGreaterThanOrEqual(2);
// Project and environment should NOT appear
expect(sql).not.toContain("project_id");
expect(sql).not.toContain("environment_id");
});
it("should apply org guard to UNION queries when using org-only filter", () => {
const { sql } = compile(
`
SELECT id, status FROM task_runs WHERE status = 'completed'
UNION ALL
SELECT id, status FROM task_runs WHERE status = 'failed'
`,
{
enforcedWhereClause: {
organization_id: { op: "eq", value: "org_tenant1" },
// project_id and environment_id omitted
},
}
);
// Both parts should have org guards
const orgIdMatches = sql.match(/organization_id/g) || [];
expect(orgIdMatches.length).toBeGreaterThanOrEqual(2);
});
it("should apply org guard to subqueries when using org-only filter", () => {
const { sql, params } = compile(
`
SELECT * FROM task_runs
WHERE id IN (SELECT run_id FROM task_events)
`,
{
enforcedWhereClause: {
organization_id: { op: "eq", value: "org_tenant1" },
// project_id and environment_id omitted
},
}
);
// Both main query and subquery should have org guards
expect(Object.values(params)).toContain("org_tenant1");
const orgIdMatches = sql.match(/organization_id/g) || [];
expect(orgIdMatches.length).toBeGreaterThanOrEqual(2);
});
});
});
describe("Multi-join Tenant Guard Qualification", () => {
/**
* Security Test: Verifies that tenant guards are properly table-qualified in multi-join queries.
*
* The bug: createEnforcedGuard was building unqualified guard expressions like:
* organization_id = 'org_tenant1'
*
* In a multi-table join where both tables have the same column (organization_id),
* an unqualified reference could potentially bind to the wrong table during resolution,
* or be ambiguous. The guards should be qualified like:
* r.organization_id = 'org_tenant1' AND e.organization_id = 'org_tenant1'
*
* This ensures each table's guard binds to the correct table, not just any matching column.
*/
it("should qualify tenant guards with table alias in JOIN queries", () => {
const { sql } = compile(`
SELECT r.id, e.event_type
FROM task_runs r
JOIN task_events e ON r.id = e.run_id
`);
// The guards should be table-qualified to prevent binding to the wrong table
// Look for pattern like: r.organization_id and e.organization_id (with table alias prefix)
// The exact format in ClickHouse SQL is just "alias.column" after resolution
// Count qualified organization_id references (should have table prefixes)
// In the WHERE clause, we should see both r.organization_id and e.organization_id
const whereClause = sql.substring(sql.indexOf("WHERE"));
// Both tables should have their own qualified tenant guards
// The pattern should be: table_alias.organization_id for each table
expect(whereClause).toMatch(/\br\b[^,]*organization_id/);
expect(whereClause).toMatch(/\be\b[^,]*organization_id/);
});
it("should qualify tenant guards with table alias in LEFT JOIN queries", () => {
const { sql } = compile(`
SELECT r.id, e.event_type
FROM task_runs r
LEFT JOIN task_events e ON r.id = e.run_id
`);
const whereClause = sql.substring(sql.indexOf("WHERE"));
// Both tables should have qualified guards
expect(whereClause).toMatch(/\br\b[^,]*organization_id/);
expect(whereClause).toMatch(/\be\b[^,]*organization_id/);
});
it("should qualify tenant guards in multi-way JOIN queries", () => {
const { sql } = compile(`
SELECT r.id, e1.event_type, e2.event_type
FROM task_runs r
JOIN task_events e1 ON r.id = e1.run_id
JOIN task_events e2 ON r.id = e2.run_id
`);
const whereClause = sql.substring(sql.indexOf("WHERE"));
// All three table aliases should have qualified guards
expect(whereClause).toMatch(/\br\b[^,]*organization_id/);
expect(whereClause).toMatch(/\be1\b[^,]*organization_id/);
expect(whereClause).toMatch(/\be2\b[^,]*organization_id/);
});
it("should ensure guards cannot bind to wrong table by verifying separate qualifications", () => {
const { sql, params } = compile(`
SELECT r.id, e.event_type
FROM task_runs r
JOIN task_events e ON r.id = e.run_id
WHERE r.status = 'completed'
`);
// Count organization_id occurrences with different table prefixes
// This ensures each table gets its own guard, not shared/ambiguous references
const orgIdPattern = /(\w+)\.organization_id/g;
const matches = [...sql.matchAll(orgIdPattern)];
const tableAliases = matches.map((m) => m[1]);
// Should have at least 2 different table aliases for organization_id
// (one for task_runs alias 'r' and one for task_events alias 'e')
expect(tableAliases).toContain("r");
expect(tableAliases).toContain("e");
// Both should use the same tenant value (parameterized)
expect(Object.values(params)).toContain("org_tenant1");
});
});
describe("Edge Cases", () => {
it("should handle empty string values", () => {
const { params } = compile("SELECT * FROM task_runs WHERE status = ''");
expect(Object.values(params)).toContain("");
});
it("should handle very long strings", () => {
const longString = "a".repeat(10000);
const { params } = compile(`SELECT * FROM task_runs WHERE status = '${longString}'`);
expect(Object.values(params)).toContain(longString);
});
it("should handle strings with newlines", () => {
const { params } = compile("SELECT * FROM task_runs WHERE status = 'line1\nline2'");
// Should handle newlines safely
expect(Object.values(params).some((v) => typeof v === "string" && v.includes("\n"))).toBe(true);
});
it("should handle special SQL keywords in strings", () => {
const { sql, params } = compile("SELECT * FROM task_runs WHERE status = 'SELECT * FROM users'");
// The SQL keywords should be in params, not interpreted
expect(sql).not.toMatch(/SELECT \* FROM users/);
expect(Object.values(params)).toContain("SELECT * FROM users");
});
});
internal-packages/tsql/src/query/time_buckets.test.ts
+181
View File
@@ -0,0 +1,181 @@
import { describe, it, expect } from "vitest";
import { calculateTimeBucketInterval, type TimeBucketInterval } from "./time_buckets.js";
/**
* Helper to create a Date range from a start date and a duration
*/
function makeRange(from: Date, durationMs: number): { from: Date; to: Date } {
return { from, to: new Date(from.getTime() + durationMs) };
}
const SECOND = 1000;
const MINUTE = 60 * SECOND;
const HOUR = 60 * MINUTE;
const DAY = 24 * HOUR;
describe("calculateTimeBucketInterval", () => {
describe("small ranges (seconds-level buckets)", () => {
it("should return 5 SECOND for a 1-minute range", () => {
const { from, to } = makeRange(new Date("2024-01-01T00:00:00Z"), 1 * MINUTE);
expect(calculateTimeBucketInterval(from, to)).toEqual<TimeBucketInterval>({
value: 5,
unit: "SECOND",
});
});
it("should return 5 SECOND for a 4-minute range", () => {
const { from, to } = makeRange(new Date("2024-01-01T00:00:00Z"), 4 * MINUTE);
expect(calculateTimeBucketInterval(from, to)).toEqual<TimeBucketInterval>({
value: 5,
unit: "SECOND",
});
});
it("should return 30 SECOND for a 10-minute range", () => {
const { from, to } = makeRange(new Date("2024-01-01T00:00:00Z"), 10 * MINUTE);
expect(calculateTimeBucketInterval(from, to)).toEqual<TimeBucketInterval>({
value: 30,
unit: "SECOND",
});
});
it("should return 30 SECOND for a 29-minute range", () => {
const { from, to } = makeRange(new Date("2024-01-01T00:00:00Z"), 29 * MINUTE);
expect(calculateTimeBucketInterval(from, to)).toEqual<TimeBucketInterval>({
value: 30,
unit: "SECOND",
});
});
});
describe("medium ranges (minute-level buckets)", () => {
it("should return 1 MINUTE for a 45-minute range", () => {
const { from, to } = makeRange(new Date("2024-01-01T00:00:00Z"), 45 * MINUTE);
expect(calculateTimeBucketInterval(from, to)).toEqual<TimeBucketInterval>({
value: 1,
unit: "MINUTE",
});
});
it("should return 1 MINUTE for a 1-hour range", () => {
const { from, to } = makeRange(new Date("2024-01-01T00:00:00Z"), 1 * HOUR);
expect(calculateTimeBucketInterval(from, to)).toEqual<TimeBucketInterval>({
value: 1,
unit: "MINUTE",
});
});
it("should return 5 MINUTE for a 3-hour range", () => {
const { from, to } = makeRange(new Date("2024-01-01T00:00:00Z"), 3 * HOUR);
expect(calculateTimeBucketInterval(from, to)).toEqual<TimeBucketInterval>({
value: 5,
unit: "MINUTE",
});
});
it("should return 15 MINUTE for a 12-hour range", () => {
const { from, to } = makeRange(new Date("2024-01-01T00:00:00Z"), 12 * HOUR);
expect(calculateTimeBucketInterval(from, to)).toEqual<TimeBucketInterval>({
value: 15,
unit: "MINUTE",
});
});
});
describe("large ranges (hour/day-level buckets)", () => {
it("should return 1 HOUR for a 2-day range", () => {
const { from, to } = makeRange(new Date("2024-01-01T00:00:00Z"), 2 * DAY);
expect(calculateTimeBucketInterval(from, to)).toEqual<TimeBucketInterval>({
value: 1,
unit: "HOUR",
});
});
it("should return 6 HOUR for a 7-day range", () => {
const { from, to } = makeRange(new Date("2024-01-01T00:00:00Z"), 7 * DAY);
expect(calculateTimeBucketInterval(from, to)).toEqual<TimeBucketInterval>({
value: 6,
unit: "HOUR",
});
});
it("should return 1 DAY for a 30-day range", () => {
const { from, to } = makeRange(new Date("2024-01-01T00:00:00Z"), 30 * DAY);
expect(calculateTimeBucketInterval(from, to)).toEqual<TimeBucketInterval>({
value: 1,
unit: "DAY",
});
});
it("should return 1 WEEK for a 90-day range", () => {
const { from, to } = makeRange(new Date("2024-01-01T00:00:00Z"), 90 * DAY);
expect(calculateTimeBucketInterval(from, to)).toEqual<TimeBucketInterval>({
value: 1,
unit: "WEEK",
});
});
});
describe("very large ranges (month-level buckets)", () => {
it("should return 1 MONTH for a 365-day range", () => {
const { from, to } = makeRange(new Date("2024-01-01T00:00:00Z"), 365 * DAY);
expect(calculateTimeBucketInterval(from, to)).toEqual<TimeBucketInterval>({
value: 1,
unit: "MONTH",
});
});
it("should return 1 MONTH for a 2-year range", () => {
const { from, to } = makeRange(new Date("2024-01-01T00:00:00Z"), 730 * DAY);
expect(calculateTimeBucketInterval(from, to)).toEqual<TimeBucketInterval>({
value: 1,
unit: "MONTH",
});
});
});
describe("edge cases", () => {
it("should handle zero-length range (from === to)", () => {
const date = new Date("2024-01-01T00:00:00Z");
const result = calculateTimeBucketInterval(date, date);
// Zero range is under 5 minutes, so 5 SECOND
expect(result).toEqual<TimeBucketInterval>({ value: 5, unit: "SECOND" });
});
it("should handle reversed dates (to < from) using absolute difference", () => {
const from = new Date("2024-01-08T00:00:00Z");
const to = new Date("2024-01-01T00:00:00Z");
// 7 days reversed → same as 7 days forward
expect(calculateTimeBucketInterval(from, to)).toEqual<TimeBucketInterval>({
value: 6,
unit: "HOUR",
});
});
it("should handle boundary exactly at 5 minutes", () => {
const { from, to } = makeRange(new Date("2024-01-01T00:00:00Z"), 5 * MINUTE);
// Exactly 5 minutes is NOT under 5 minutes, so should be 30 SECOND
expect(calculateTimeBucketInterval(from, to)).toEqual<TimeBucketInterval>({
value: 30,
unit: "SECOND",
});
});
it("should handle boundary exactly at 24 hours", () => {
const { from, to } = makeRange(new Date("2024-01-01T00:00:00Z"), 24 * HOUR);
// Exactly 24 hours is NOT under 24 hours, so should be 1 HOUR
expect(calculateTimeBucketInterval(from, to)).toEqual<TimeBucketInterval>({
value: 1,
unit: "HOUR",
});
});
it("should handle very small range (1 second)", () => {
const { from, to } = makeRange(new Date("2024-01-01T00:00:00Z"), 1 * SECOND);
expect(calculateTimeBucketInterval(from, to)).toEqual<TimeBucketInterval>({
value: 5,
unit: "SECOND",
});
});
});
});
internal-packages/tsql/src/query/time_buckets.ts
+100
View File
@@ -0,0 +1,100 @@
/**
* Time bucket interval calculation for the `timeBucket()` TSQL function.
*
* Given a time range, determines the most appropriate bucket interval
* to produce a reasonable number of data points (~50-100 buckets).
*/
/**
* A time bucket interval with a numeric value and time unit.
* Used to generate ClickHouse `INTERVAL N UNIT` syntax.
*/
export interface TimeBucketInterval {
/** The numeric value of the interval (e.g., 5 for "5 MINUTE") */
value: number;
/** The time unit */
unit: "SECOND" | "MINUTE" | "HOUR" | "DAY" | "WEEK" | "MONTH";
}
/**
* A threshold mapping a maximum time range duration to a bucket interval.
*/
export interface BucketThreshold {
/** Maximum range duration in seconds for this threshold to apply */
maxRangeSeconds: number;
/** The bucket interval to use when the range is under maxRangeSeconds */
interval: TimeBucketInterval;
}
/**
* Default time bucket thresholds: each entry defines a maximum time range duration (in seconds)
* and the corresponding bucket interval to use.
*
* The intervals are chosen to produce roughly 50-100 data points for the given range.
* Entries are ordered from smallest to largest range.
*/
export const BUCKET_THRESHOLDS: BucketThreshold[] = [
// Under 5 minutes → 5 second buckets (max 60 buckets)
{ maxRangeSeconds: 5 * 60, interval: { value: 5, unit: "SECOND" } },
// Under 30 minutes → 30 second buckets (max 60 buckets)
{ maxRangeSeconds: 30 * 60, interval: { value: 30, unit: "SECOND" } },
// Under 2 hours → 1 minute buckets (max 120 buckets)
{ maxRangeSeconds: 2 * 60 * 60, interval: { value: 1, unit: "MINUTE" } },
// Under 6 hours → 5 minute buckets (max 72 buckets)
{ maxRangeSeconds: 6 * 60 * 60, interval: { value: 5, unit: "MINUTE" } },
// Under 24 hours → 15 minute buckets (max 96 buckets)
{ maxRangeSeconds: 24 * 60 * 60, interval: { value: 15, unit: "MINUTE" } },
// Under 3 days → 1 hour buckets (max 72 buckets)
{ maxRangeSeconds: 3 * 24 * 60 * 60, interval: { value: 1, unit: "HOUR" } },
// Under 14 days → 6 hour buckets (max 56 buckets)
{ maxRangeSeconds: 14 * 24 * 60 * 60, interval: { value: 6, unit: "HOUR" } },
// Under 60 days → 1 day buckets (max 60 buckets)
{ maxRangeSeconds: 60 * 24 * 60 * 60, interval: { value: 1, unit: "DAY" } },
// Under 365 days → 1 week buckets (max ~52 buckets)
{ maxRangeSeconds: 365 * 24 * 60 * 60, interval: { value: 1, unit: "WEEK" } },
];
/** Default interval for very large ranges (365+ days) */
const DEFAULT_LARGE_INTERVAL: TimeBucketInterval = { value: 1, unit: "MONTH" };
/**
* Calculate the most appropriate time bucket interval for a given time range.
*
* The interval is chosen to produce a reasonable number of data points (~50-100 buckets).
* For very small ranges (< 5 minutes), uses 5-second buckets.
* For very large ranges (> 365 days), uses 1-month buckets.
*
* @param from - Start of the time range
* @param to - End of the time range
* @returns The recommended bucket interval
*
* @example
* ```typescript
* // 1 hour range → 1 minute buckets
* calculateTimeBucketInterval(
* new Date("2024-01-01T00:00:00Z"),
* new Date("2024-01-01T01:00:00Z"),
* ); // { value: 1, unit: "MINUTE" }
*
* // 7 day range → 6 hour buckets
* calculateTimeBucketInterval(
* new Date("2024-01-01"),
* new Date("2024-01-08"),
* ); // { value: 6, unit: "HOUR" }
* ```
*/
export function calculateTimeBucketInterval(
from: Date,
to: Date,
thresholds?: BucketThreshold[]
): TimeBucketInterval {
const rangeSeconds = Math.abs(to.getTime() - from.getTime()) / 1000;
for (const threshold of thresholds ?? BUCKET_THRESHOLDS) {
if (rangeSeconds < threshold.maxRangeSeconds) {
return threshold.interval;
}
}
return DEFAULT_LARGE_INTERVAL;
}
internal-packages/tsql/src/query/timings.ts
+109
View File
@@ -0,0 +1,109 @@
// TypeScript translation of posthog/hogql/timings.py
/**
* Get performance counter in milliseconds (Node.js equivalent of perf_counter)
* Uses performance.now() which is available in:
* - Node.js 18+ (global)
* - Browser (global)
* - Node.js <18 via perf_hooks module
*/
function getPerformanceNow(): number {
// Check for global performance (Node.js 18+ or browser)
if (typeof globalThis !== "undefined" && "performance" in globalThis) {
const perf = (globalThis as any).performance;
if (perf && typeof perf.now === "function") {
return perf.now();
}
}
// Fallback to Date.now() if performance API is not available
// Note: This is less precise but works everywhere
return Date.now();
}
export interface QueryTiming {
key: string; // Key identifying the timing measurement
time: number; // Time in seconds
}
const TIMING_DECIMAL_PLACES = 3; // round to milliseconds
// Not thread safe.
// See trends_query_runner for an example of how to use for multithreaded queries
export class TSQLTimings {
// Completed time in seconds for different parts of the TSQL query
timings: Record<string, number> = {};
// Used for housekeeping
private _timingPointer: string;
private _timingStarts: Record<string, number> = {};
constructor(_timingPointer: string = ".") {
this._timingPointer = _timingPointer;
this._timingStarts[this._timingPointer] = this.perfCounter();
}
cloneForSubquery(seriesIndex: number): TSQLTimings {
return new TSQLTimings(`${this._timingPointer}/series_${seriesIndex}`);
}
clearTimings(): void {
this.timings = {};
}
/**
* Measure execution time of a function.
* Usage: timings.measure('operation', () => { ... });
*/
measure<T>(key: string, fn: () => T): T {
const lastKey = this._timingPointer;
const fullKey = `${this._timingPointer}/${key}`;
this._timingPointer = fullKey;
this._timingStarts[fullKey] = this.perfCounter();
try {
return fn();
} finally {
const duration = (this.perfCounter() - this._timingStarts[fullKey]) / 1000; // Convert to seconds
this.timings[fullKey] = (this.timings[fullKey] || 0.0) + duration;
delete this._timingStarts[fullKey];
this._timingPointer = lastKey;
}
}
/**
* Get performance counter in milliseconds (Node.js equivalent of perf_counter)
*/
private perfCounter(): number {
return getPerformanceNow();
}
toDict(): Record<string, number> {
const timings = { ...this.timings };
// Process in reverse order to handle nested timings correctly
const keys = Object.keys(this._timingStarts).reverse();
for (const key of keys) {
const start = this._timingStarts[key];
const elapsed = (this.perfCounter() - start) / 1000; // Convert to seconds
timings[key] = this.round((timings[key] || 0.0) + elapsed);
}
return timings;
}
toList(backOutStack: boolean = true): QueryTiming[] {
const timingDict = backOutStack ? this.toDict() : this.timings;
return Object.entries(timingDict).map(([key, time]) => ({
key: key,
time: this.round(time),
}));
}
/**
* Round to specified decimal places (milliseconds precision)
*/
private round(value: number): number {
return (
Math.round(value * Math.pow(10, TIMING_DECIMAL_PLACES)) / Math.pow(10, TIMING_DECIMAL_PLACES)
);
}
}
internal-packages/tsql/src/query/validator.test.ts
+128
View File
@@ -0,0 +1,128 @@
import { describe, it, expect } from "vitest";
import { validateQuery } from "./validator.js";
import { parseTSQLSelect } from "../index.js";
import { column, type TableSchema } from "./schema.js";
const runsSchema: TableSchema = {
name: "runs",
clickhouseName: "trigger_dev.task_runs_v2",
columns: {
id: { name: "id", ...column("String") },
status: {
name: "status",
...column("String", {
allowedValues: ["PENDING", "COMPLETED", "FAILED"],
}),
},
task_id: { name: "task_id", ...column("String") },
created_at: { name: "created_at", ...column("DateTime64") },
},
tenantColumns: {
organizationId: "organization_id",
projectId: "project_id",
environmentId: "environment_id",
},
};
function validateSQL(query: string, schema: TableSchema[] = [runsSchema]) {
const ast = parseTSQLSelect(query);
return validateQuery(ast, schema);
}
describe("validateQuery", () => {
describe("SELECT aliases", () => {
it("should allow ORDER BY to reference aliased columns", () => {
const result = validateSQL(
"SELECT status, count(*) as count FROM runs GROUP BY status ORDER BY count DESC"
);
expect(result.valid).toBe(true);
expect(result.issues).toHaveLength(0);
});
it("should allow ORDER BY to reference multiple aliased columns", () => {
const result = validateSQL(
"SELECT status, count(*) as total, avg(created_at) as avg_time FROM runs GROUP BY status ORDER BY total DESC, avg_time ASC"
);
expect(result.valid).toBe(true);
expect(result.issues).toHaveLength(0);
});
it("should report unknown columns that are not aliases as errors", () => {
const result = validateSQL(
"SELECT status, count(*) as count FROM runs GROUP BY status ORDER BY unknown_col DESC"
);
expect(result.valid).toBe(false); // unknown column is now an error
expect(result.issues).toHaveLength(1);
expect(result.issues[0].type).toBe("unknown_column");
expect(result.issues[0].severity).toBe("error");
expect(result.issues[0].columnName).toBe("unknown_col");
});
it("should allow ORDER BY to reference both aliases and real columns", () => {
const result = validateSQL(
"SELECT status, count(*) as count FROM runs GROUP BY status ORDER BY status ASC, count DESC"
);
expect(result.valid).toBe(true);
expect(result.issues).toHaveLength(0);
});
it("should allow HAVING to reference implicit column names from aggregations", () => {
const result = validateSQL(
"SELECT COUNT(), status FROM runs GROUP BY status HAVING count > 20"
);
expect(result.valid).toBe(true);
expect(result.issues).toHaveLength(0);
});
it("should allow HAVING to reference implicit names from multiple aggregations", () => {
const result = validateSQL(
"SELECT COUNT(), SUM(created_at), status FROM runs GROUP BY status HAVING count > 10 AND sum > 100"
);
expect(result.valid).toBe(true);
expect(result.issues).toHaveLength(0);
});
it("should allow ORDER BY to reference implicit column names", () => {
const result = validateSQL(
"SELECT COUNT(), status FROM runs GROUP BY status ORDER BY count DESC"
);
expect(result.valid).toBe(true);
expect(result.issues).toHaveLength(0);
});
});
describe("column validation", () => {
it("should validate known columns", () => {
const result = validateSQL("SELECT id, status FROM runs LIMIT 10");
expect(result.valid).toBe(true);
expect(result.issues).toHaveLength(0);
});
it("should error on unknown columns", () => {
const result = validateSQL("SELECT id, unknown_column FROM runs LIMIT 10");
expect(result.valid).toBe(false); // unknown columns are now errors
expect(result.issues).toHaveLength(1);
expect(result.issues[0].type).toBe("unknown_column");
expect(result.issues[0].severity).toBe("error");
expect(result.issues[0].columnName).toBe("unknown_column");
});
});
describe("enum validation", () => {
it("should validate enum values", () => {
const result = validateSQL("SELECT id, status FROM runs WHERE status = 'COMPLETED' LIMIT 10");
expect(result.valid).toBe(true);
expect(result.issues).toHaveLength(0);
});
it("should error on invalid enum values", () => {
const result = validateSQL(
"SELECT id, status FROM runs WHERE status = 'INVALID_STATUS' LIMIT 10"
);
expect(result.valid).toBe(false);
expect(result.issues).toHaveLength(1);
expect(result.issues[0].type).toBe("invalid_enum_value");
expect(result.issues[0].invalidValue).toBe("INVALID_STATUS");
});
});
});
internal-packages/tsql/src/query/validator.ts
+596
View File
@@ -0,0 +1,596 @@
// Schema validation for TSQL queries
// Validates column names and enum values against the schema
import type {
SelectQuery,
SelectSetQuery,
Expression,
Field,
CompareOperation,
Constant,
And,
Or,
Not,
Alias,
OrderExpr,
Call,
JoinExpr,
BetweenExpr,
Array as ASTArray,
ArithmeticOperation,
} from "./ast.js";
import type { TableSchema, ColumnSchema } from "./schema.js";
import { getAllowedUserValues, getCoreColumns, isValidUserValue } from "./schema.js";
import { CompareOperationOp, ArithmeticOperationOp } from "./ast.js";
/**
* Severity of a validation issue
*/
export type ValidationSeverity = "error" | "warning" | "info";
/**
* A validation issue found in the query
*/
export interface ValidationIssue {
/** The error/warning message */
message: string;
/** Severity of the issue */
severity: ValidationSeverity;
/** The type of issue */
type: "unknown_column" | "unknown_table" | "invalid_enum_value" | "select_star";
/** Optional: the column name that caused the issue */
columnName?: string;
/** Optional: the table name that caused the issue */
tableName?: string;
/** Optional: the invalid value */
invalidValue?: string;
/** Optional: list of allowed values */
allowedValues?: string[];
/** Optional: suggested columns to use instead (for select_star) */
suggestedColumns?: string[];
}
/**
* Result of validating a query
*/
export interface ValidationResult {
/** Whether the query is valid */
valid: boolean;
/** List of issues found */
issues: ValidationIssue[];
}
/**
* Context for tracking tables and columns during validation
*/
interface ValidationContext {
/** Map of table aliases/names to their schemas */
tables: Map<string, TableSchema>;
/** The schema array for lookups */
schema: TableSchema[];
/** Accumulated issues */
issues: ValidationIssue[];
/** Set of column aliases defined in the SELECT clause */
selectAliases: Set<string>;
}
/**
* Validate a parsed TSQL query against a schema
*
* @param ast - The parsed query AST
* @param schema - Array of table schemas to validate against
* @returns Validation result with any issues found
*/
export function validateQuery(
ast: SelectQuery | SelectSetQuery,
schema: TableSchema[]
): ValidationResult {
const context: ValidationContext = {
tables: new Map(),
schema,
issues: [],
selectAliases: new Set(),
};
if (ast.expression_type === "select_set_query") {
validateSelectSetQuery(ast, context);
} else {
validateSelectQuery(ast, context);
}
return {
valid: context.issues.filter((i) => i.severity === "error").length === 0,
issues: context.issues,
};
}
/**
* Get the implicit column name for an expression without an explicit alias.
* This matches the naming used in the printer for result columns.
*
* @param expr - The SELECT expression
* @returns The implicit name, or null if no implicit name applies
*/
function getImplicitName(expr: Expression): string | null {
// Handle Call (function/aggregation) - use lowercase function name
if ((expr as Call).expression_type === "call") {
const call = expr as Call;
return call.name.toLowerCase();
}
// Handle ArithmeticOperation - use operator function name
if ((expr as ArithmeticOperation).expression_type === "arithmetic_operation") {
const arith = expr as ArithmeticOperation;
switch (arith.op) {
case ArithmeticOperationOp.Add:
return "plus";
case ArithmeticOperationOp.Sub:
return "minus";
case ArithmeticOperationOp.Mult:
return "multiply";
case ArithmeticOperationOp.Div:
return "divide";
case ArithmeticOperationOp.Mod:
return "modulo";
default:
return "expression";
}
}
// Handle Constant - use string representation
if ((expr as Constant).expression_type === "constant") {
const constant = expr as Constant;
if (constant.value === null) {
return "NULL";
}
if (typeof constant.value === "string") {
return `'${constant.value}'`;
}
return String(constant.value);
}
// Field expressions don't get implicit names (they use the column name directly)
return null;
}
/**
* Validate a SELECT SET query (UNION, INTERSECT, etc.)
*/
function validateSelectSetQuery(node: SelectSetQuery, context: ValidationContext): void {
if (node.initial_select_query.expression_type === "select_set_query") {
validateSelectSetQuery(node.initial_select_query, context);
} else {
validateSelectQuery(node.initial_select_query, context);
}
for (const subsequent of node.subsequent_select_queries) {
if (subsequent.select_query.expression_type === "select_set_query") {
validateSelectSetQuery(subsequent.select_query as SelectSetQuery, context);
} else {
validateSelectQuery(subsequent.select_query as SelectQuery, context);
}
}
}
/**
* Check if an expression is a SELECT * (asterisk)
*/
function isSelectStar(expr: Expression): boolean {
if ((expr as Field).expression_type !== "field") return false;
const field = expr as Field;
// SELECT * or SELECT table.*
return (
(field.chain.length === 1 && field.chain[0] === "*") ||
(field.chain.length === 2 && field.chain[1] === "*")
);
}
/**
* Validate a SELECT query
*/
function validateSelectQuery(node: SelectQuery, context: ValidationContext): void {
// Save parent aliases and create fresh set for this query
const parentAliases = context.selectAliases;
context.selectAliases = new Set();
// First, extract tables from FROM clause to build context
if (node.select_from) {
extractTablesFromJoin(node.select_from, context);
}
// Check for SELECT * and emit warning
if (node.select) {
const hasSelectStar = node.select.some(isSelectStar);
if (hasSelectStar) {
// Collect core columns from all tables in context
const coreColumns: string[] = [];
for (const tableSchema of context.tables.values()) {
const tableCoreColumns = getCoreColumns(tableSchema);
coreColumns.push(...tableCoreColumns);
}
// Build info message about SELECT * behavior
let suggestionMsg = "SELECT * doesn't return all columns.";
if (coreColumns.length > 0) {
suggestionMsg += `It will return: ${coreColumns.join(", ")}. `;
}
suggestionMsg += "Specify columns explicitly to include other columns.";
context.issues.push({
message: suggestionMsg,
severity: "info",
type: "select_star",
suggestedColumns: coreColumns.length > 0 ? coreColumns : undefined,
});
}
}
// Extract column aliases from SELECT clause before validation
// This allows ORDER BY and HAVING to reference aliased columns
if (node.select) {
for (const expr of node.select) {
if ((expr as Alias).expression_type === "alias") {
// Explicit alias: SELECT ... AS name (stored lowercase for case-insensitive lookup)
context.selectAliases.add((expr as Alias).alias.toLowerCase());
} else {
// Check for implicit aliases from expressions without AS
const implicitName = getImplicitName(expr);
if (implicitName) {
context.selectAliases.add(implicitName);
}
}
}
}
// Validate SELECT columns
if (node.select) {
for (const expr of node.select) {
validateExpression(expr, context);
}
}
// Validate WHERE clause
if (node.where) {
validateExpression(node.where, context);
}
// Validate GROUP BY
if (node.group_by) {
for (const expr of node.group_by) {
validateExpression(expr, context);
}
}
// Validate HAVING
if (node.having) {
validateExpression(node.having, context);
}
// Validate ORDER BY
if (node.order_by) {
for (const expr of node.order_by) {
validateExpression(expr, context);
}
}
// Restore parent aliases
context.selectAliases = parentAliases;
}
/**
* Extract table schemas from JOIN expressions
*/
function extractTablesFromJoin(node: JoinExpr, context: ValidationContext): void {
if (node.table) {
const tableExpr = node.table;
if ((tableExpr as Field).expression_type === "field") {
const field = tableExpr as Field;
const tableName = field.chain[0];
if (typeof tableName === "string") {
// Find the table schema
const tableSchema = context.schema.find(
(t) => t.name.toLowerCase() === tableName.toLowerCase()
);
if (tableSchema) {
// Register with alias if provided, otherwise use table name
const key = node.alias || tableName;
context.tables.set(key.toLowerCase(), tableSchema);
} else {
// Unknown table
context.issues.push({
message: `Unknown table "${tableName}". Available tables: ${
context.schema.map((t) => t.name).join(", ") || "(none)"
}`,
severity: "warning",
type: "unknown_table",
tableName,
});
}
}
} else if (
(tableExpr as SelectQuery).expression_type === "select_query" ||
(tableExpr as SelectSetQuery).expression_type === "select_set_query"
) {
// Subquery - validate it recursively
if ((tableExpr as SelectSetQuery).expression_type === "select_set_query") {
validateSelectSetQuery(tableExpr as SelectSetQuery, context);
} else {
validateSelectQuery(tableExpr as SelectQuery, context);
}
}
}
// Process next join in chain
if (node.next_join) {
extractTablesFromJoin(node.next_join, context);
}
}
/**
* Validate an expression and its children
*/
function validateExpression(expr: Expression, context: ValidationContext): void {
if (!expr || typeof expr !== "object") return;
const exprType = expr.expression_type;
switch (exprType) {
case "field":
validateField(expr as Field, context);
break;
case "compare_operation":
validateCompareOperation(expr as CompareOperation, context);
break;
case "and":
for (const e of (expr as And).exprs) {
validateExpression(e, context);
}
break;
case "or":
for (const e of (expr as Or).exprs) {
validateExpression(e, context);
}
break;
case "not":
validateExpression((expr as Not).expr, context);
break;
case "alias":
validateExpression((expr as Alias).expr, context);
break;
case "order_expr":
validateExpression((expr as OrderExpr).expr, context);
break;
case "call":
for (const arg of (expr as Call).args) {
validateExpression(arg, context);
}
break;
case "between_expr":
validateExpression((expr as BetweenExpr).expr, context);
validateExpression((expr as BetweenExpr).low, context);
validateExpression((expr as BetweenExpr).high, context);
break;
case "array":
for (const e of (expr as ASTArray).exprs) {
validateExpression(e, context);
}
break;
// Other expression types that we don't need to deeply validate
case "constant":
case "select_query":
case "select_set_query":
// Skip - constants don't need validation, subqueries are handled separately
break;
}
}
/**
* Validate a field reference
*/
function validateField(field: Field, context: ValidationContext): void {
const chain = field.chain;
if (chain.length === 0) return;
// Handle asterisk
if (chain[0] === "*") return;
if (chain.length === 2 && chain[1] === "*") return;
const firstPart = chain[0];
if (typeof firstPart !== "string") return;
// Case 1: Qualified reference like table.column
if (chain.length >= 2) {
const tableAlias = firstPart.toLowerCase();
const columnName = chain[1];
if (typeof columnName !== "string") return;
const tableSchema = context.tables.get(tableAlias);
if (tableSchema) {
// Check if column exists
if (!tableSchema.columns[columnName]) {
const availableColumns = Object.keys(tableSchema.columns).join(", ");
context.issues.push({
message: `Unknown column "${columnName}" on table "${tableAlias}". Available columns: ${availableColumns}`,
severity: "error",
type: "unknown_column",
columnName,
tableName: tableAlias,
});
}
}
return;
}
// Case 2: Unqualified reference - try to find in any table or SELECT alias
const columnName = firstPart;
// Check if it's a SELECT alias (e.g., from "count(*) as count")
// Case-insensitive: aliases are stored lowercase
if (context.selectAliases.has(columnName.toLowerCase())) {
return;
}
let found = false;
for (const tableSchema of context.tables.values()) {
if (tableSchema.columns[columnName]) {
found = true;
break;
}
}
if (!found && context.tables.size > 0) {
// Only report if we have tables to check against
const allColumns = new Set<string>();
for (const tableSchema of context.tables.values()) {
for (const col of Object.keys(tableSchema.columns)) {
allColumns.add(col);
}
}
context.issues.push({
message: `Unknown column "${columnName}". Available columns: ${Array.from(allColumns).join(
", "
)}`,
severity: "error",
type: "unknown_column",
columnName,
});
}
}
/**
* Validate a comparison operation, including enum value checks
*/
function validateCompareOperation(op: CompareOperation, context: ValidationContext): void {
// Validate both sides recursively
validateExpression(op.left, context);
validateExpression(op.right, context);
// Check for enum value validation
// We look for patterns like: column = 'value' or column IN ('value1', 'value2')
const columnInfo = extractColumnFromExpression(op.left, context);
if (!columnInfo) return;
const { columnSchema, columnName, tableName } = columnInfo;
// Only validate if the column has allowedValues or valueMap
const allowedValues = getAllowedUserValues(columnSchema);
if (allowedValues.length === 0) return;
// Check the comparison type
switch (op.op) {
case CompareOperationOp.Eq:
case CompareOperationOp.NotEq:
// Single value comparison
validateEnumValue(op.right, columnSchema, columnName, tableName, context);
break;
case CompareOperationOp.In:
case CompareOperationOp.NotIn:
case CompareOperationOp.GlobalIn:
case CompareOperationOp.GlobalNotIn:
// Array of values
if ((op.right as ASTArray).expression_type === "array") {
for (const elem of (op.right as ASTArray).exprs) {
validateEnumValue(elem, columnSchema, columnName, tableName, context);
}
}
break;
}
}
/**
* Extract column information from an expression if it's a simple column reference
*/
function extractColumnFromExpression(
expr: Expression,
context: ValidationContext
): { columnSchema: ColumnSchema; columnName: string; tableName?: string } | null {
if ((expr as Field).expression_type !== "field") return null;
const field = expr as Field;
const chain = field.chain;
if (chain.length === 0) return null;
const firstPart = chain[0];
if (typeof firstPart !== "string") return null;
// Qualified reference: table.column
if (chain.length >= 2) {
const tableAlias = firstPart.toLowerCase();
const columnName = chain[1];
if (typeof columnName !== "string") return null;
const tableSchema = context.tables.get(tableAlias);
if (!tableSchema) return null;
const columnSchema = tableSchema.columns[columnName];
if (!columnSchema) return null;
return { columnSchema, columnName, tableName: tableAlias };
}
// Unqualified reference
const columnName = firstPart;
for (const [tableName, tableSchema] of context.tables.entries()) {
const columnSchema = tableSchema.columns[columnName];
if (columnSchema) {
return { columnSchema, columnName, tableName };
}
}
return null;
}
/**
* Validate that a value matches the allowed enum values for a column
* Supports both allowedValues and valueMap, with case-insensitive matching
*/
function validateEnumValue(
expr: Expression,
columnSchema: ColumnSchema,
columnName: string,
tableName: string | undefined,
context: ValidationContext
): void {
if ((expr as Constant).expression_type !== "constant") return;
const constant = expr as Constant;
if (typeof constant.value !== "string") return;
const value = constant.value;
// Use isValidUserValue for case-insensitive validation against user-friendly values
if (!isValidUserValue(columnSchema, value)) {
const columnRef = tableName ? `${tableName}.${columnName}` : columnName;
// Show user-friendly values in the error message
const allowedValues = getAllowedUserValues(columnSchema);
context.issues.push({
message: `Invalid value "${value}" for column "${columnRef}". Allowed values: ${allowedValues.join(
", "
)}`,
severity: "error",
type: "invalid_enum_value",
columnName,
tableName,
invalidValue: value,
allowedValues,
});
}
}
internal-packages/tsql/tsconfig.json
+25
View File
@@ -0,0 +1,25 @@
{
"compilerOptions": {
"target": "ES2019",
"lib": ["ES2019", "DOM", "DOM.Iterable", "DOM.AsyncIterable"],
"module": "ESNext",
"moduleResolution": "node",
"moduleDetection": "force",
"verbatimModuleSyntax": false,
"types": ["vitest/globals"],
"esModuleInterop": true,
"forceConsistentCasingInFileNames": true,
"isolatedModules": true,
"preserveWatchOutput": true,
"skipLibCheck": true,
"noEmit": true,
"strict": true,
"paths": {
"@trigger.dev/core": ["../../packages/core/src/index"],
"@trigger.dev/core/*": ["../../packages/core/src/*"],
"@internal/clickhouse": ["../clickhouse/src/index"],
"@internal/clickhouse/*": ["../clickhouse/src/*"]
}
},
"exclude": ["node_modules"]
}
internal-packages/tsql/vitest.config.ts
+14
View File
@@ -0,0 +1,14 @@
import { defineConfig } from "vitest/config";
export default defineConfig({
test: {
include: ["**/*.test.ts"],
globals: true,
isolate: true,
fileParallelism: false,
testTimeout: 60_000,
coverage: {
provider: "v8",
},
},
});