diff --git a/CLAUDE.md b/CLAUDE.md index dfce6cfb08..f21fe118e5 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -120,8 +120,8 @@ A case is a directory holding the inputs and the expected output. `exec.json` names the command and its arguments — omit it and the case runs `generate`, comparing the generated files against the ones committed alongside; give it `{"command": "analyze", "args": [...]}` and the case compares the command's -stdout against `stdout.txt`. A case that is expected to fail commits its -`stderr.txt`. Regenerate a golden by running the command in its directory and +stdout against `output.json` (or `stdout.txt` for a command that does not +print JSON). A case that is expected to fail commits its `stderr.txt`. Regenerate a golden by running the command in its directory and writing the output back over the committed file. `TestReplay` runs the whole corpus once per *context*. `base` runs each case as @@ -147,8 +147,11 @@ the run early. Run a subset to get past one (`-run 'TestReplay/core/^select'`). The dialect seeds under `/internal/engine//dialect/` are generated from a live database by `/internal/goldeneye`, a nested module, and its tests -verify the committed files against one byte for byte. Engines whose database -is not available skip. +verify the committed files against one byte for byte. The same module checks +the `analyze_*` cases under `/internal/endtoend/testdata/` against what the +database itself reports for them, so a `fixture.sql` next to a case's schema +gives the queries rows to run against. Engines whose database is not +available skip. ```bash cd internal/goldeneye @@ -239,8 +242,10 @@ MYSQL_SERVER_URI="root:mysecretpassword@tcp(127.0.0.1:3306)/mysql?multiStatement JSONL read by `/internal/core/seed`; the generated parts come from `/internal/goldeneye` - `/internal/goldeneye/` - Nested module that generates the dialect seeds - under `/internal/engine//dialect/` from a live database and checks - the committed ones against it, one package per engine; see its README + under `/internal/engine//dialect/` from a live database, checks + the committed ones against it, and checks the analyze cases under + `/internal/endtoend/testdata/` against what the database reports, one + package per engine; see its README - `/internal/core/` - The analysis core: catalog, analyzer and dialect seeds - `/internal/compiler/` - Query compilation logic - `/internal/codegen/` - Code generation for different languages diff --git a/docs/howto/analyze.md b/docs/howto/analyze.md index 87541d093e..325acbd6f6 100644 --- a/docs/howto/analyze.md +++ b/docs/howto/analyze.md @@ -70,23 +70,24 @@ reports the result columns and parameters: "columns": [ { "name": "id", - "data_type": "bigserial", - "not_null": true, - "is_array": false, + "type": { + "name": "bigserial" + }, "table": "authors" }, { "name": "name", - "data_type": "text", - "not_null": true, - "is_array": false, + "type": { + "name": "text" + }, "table": "authors" }, { "name": "bio", - "data_type": "text", - "not_null": false, - "is_array": false, + "type": { + "name": "text", + "nullable": true + }, "table": "authors" } ], @@ -95,9 +96,9 @@ reports the result columns and parameters: "number": 1, "column": { "name": "id", - "data_type": "bigserial", - "not_null": true, - "is_array": false, + "type": { + "name": "bigserial" + }, "table": "authors" } } @@ -106,6 +107,13 @@ reports the result columns and parameters: ] ``` +A column's `type` is written as a call expression: a `name` applied to +`args`, each of which carries an optional `label` and exactly one of `type`, +`int`, `bool` or `string`, with `nullable` set at whatever depth it applies. +An array of text is `array` applied to `text`; a `Map(String, Nullable(UInt8))` +in ClickHouse is `map` applied to `string` and a nullable `uint8`. Names are +recorded as the engine reports them. + Pass `--ast` to also include each statement's parsed AST under an `ast` key. It has the same shape as the output of [`parse`](parse.md), with every node tagged by type. diff --git a/internal/cmd/analyze.go b/internal/cmd/analyze.go index df71bae0b4..ba278498b6 100644 --- a/internal/cmd/analyze.go +++ b/internal/cmd/analyze.go @@ -11,6 +11,7 @@ import ( "github.com/sqlc-dev/sqlc/internal/compiler" "github.com/sqlc-dev/sqlc/internal/config" + "github.com/sqlc-dev/sqlc/internal/core" "github.com/sqlc-dev/sqlc/internal/multierr" "github.com/sqlc-dev/sqlc/internal/opts" "github.com/sqlc-dev/sqlc/internal/sql/ast" @@ -204,11 +205,9 @@ type analyzedQuery struct { } type analyzedColumn struct { - Name string `json:"name"` - DataType string `json:"data_type"` - NotNull bool `json:"not_null"` - IsArray bool `json:"is_array"` - Table string `json:"table,omitempty"` + Name string `json:"name"` + Type *core.TypeExpr `json:"type,omitempty"` + Table string `json:"table,omitempty"` } type analyzedParam struct { @@ -243,13 +242,34 @@ func newAnalyzedColumn(col *compiler.Column) analyzedColumn { return analyzedColumn{} } ac := analyzedColumn{ - Name: col.Name, - DataType: col.DataType, - NotNull: col.NotNull, - IsArray: col.IsArray, + Name: col.Name, + Type: newAnalyzedType(col), } if col.Table != nil { ac.Table = col.Table.Name } return ac } + +// newAnalyzedType is the column's type as an expression: the one the +// analysis core wrote when it did, otherwise the flat description the +// compiler holds, which is the data type wrapped in one array node per +// dimension with the column's nullability on the outermost node. +func newAnalyzedType(col *compiler.Column) *core.TypeExpr { + if col.TypeExpr != nil { + return col.TypeExpr + } + if col.DataType == "" { + return nil + } + t := core.ParseTypeExpr(col.DataType) + dims := col.ArrayDims + if col.IsArray && dims == 0 { + dims = 1 + } + for i := 0; i < dims; i++ { + t = &core.TypeExpr{Name: "array", Args: []core.TypeArg{{Type: t}}} + } + t.Nullable = !col.NotNull + return t +} diff --git a/internal/compiler/parse_core.go b/internal/compiler/parse_core.go index 6cae7586c3..7e5685a962 100644 --- a/internal/compiler/parse_core.go +++ b/internal/compiler/parse_core.go @@ -105,6 +105,7 @@ func coreColumn(c core.Column) *Column { DataType: c.DataType, NotNull: c.NotNull, IsArray: c.IsArray, + TypeExpr: c.Type, } // The core reports arrays without dimensions, and codegen renders one // "[]" per dimension. @@ -129,6 +130,7 @@ func coreParamColumn(p core.Parameter, params *named.ParamSet) *Column { DataType: p.DataType, NotNull: p.NotNull, IsArray: p.IsArray, + TypeExpr: p.Type, } if p.IsArray { col.ArrayDims = 1 diff --git a/internal/compiler/query.go b/internal/compiler/query.go index b3cf9d6154..8753d5a7cc 100644 --- a/internal/compiler/query.go +++ b/internal/compiler/query.go @@ -1,6 +1,7 @@ package compiler import ( + "github.com/sqlc-dev/sqlc/internal/core" "github.com/sqlc-dev/sqlc/internal/metadata" "github.com/sqlc-dev/sqlc/internal/sql/ast" "github.com/sqlc-dev/sqlc/internal/sql/catalog" @@ -37,6 +38,11 @@ type Column struct { Type *ast.TypeName EmbedTable *ast.TableName + // TypeExpr is the type as the analysis core wrote it, with the + // arguments and nesting DataType and IsArray flatten away. It is unset + // on the legacy path. + TypeExpr *core.TypeExpr + IsSqlcSlice bool // is this sqlc.slice() skipTableRequiredCheck bool diff --git a/internal/core/analysis.go b/internal/core/analysis.go index 822c12c411..7542d12b1e 100644 --- a/internal/core/analysis.go +++ b/internal/core/analysis.go @@ -53,8 +53,11 @@ type ColumnSource struct { } type Column struct { - Name string `json:"name"` - DataType string `json:"data_type"` + Name string `json:"name"` + DataType string `json:"data_type"` + // Type is the column's type as an expression, carrying what DataType + // and IsArray flatten away: arguments, nesting and inner nullability. + Type *TypeExpr `json:"type,omitempty"` TypeOID int64 `json:"type_oid,omitempty"` NotNull bool `json:"not_null"` IsArray bool `json:"is_array,omitempty"` @@ -73,6 +76,7 @@ type Parameter struct { Number int `json:"number"` Name string `json:"name,omitempty"` DataType string `json:"data_type,omitempty"` + Type *TypeExpr `json:"type,omitempty"` TypeOID int64 `json:"type_oid,omitempty"` NotNull bool `json:"not_null"` IsArray bool `json:"is_array,omitempty"` diff --git a/internal/core/analyzer/analyzer.go b/internal/core/analyzer/analyzer.go index 4608ff1bdc..8bc92bf5b3 100644 --- a/internal/core/analyzer/analyzer.go +++ b/internal/core/analyzer/analyzer.go @@ -119,6 +119,20 @@ func derivedRel(alias string, cols []core.Column) scopeRel { } func (a *analyzer) result() core.PrepareResult { + // A placeholder nothing constrained takes the dialect's type for one, when + // the dialect has such a type. + if oid, ok := a.cat.UntypedTypeOID(); ok { + for n, p := range a.params { + if p.TypeOID == 0 && p.DataType == "" { + t := exprType{typeOID: oid, nullable: true} + p.TypeOID = oid + p.DataType, p.IsArray = a.typeNameOf(t) + p.NotNull = false + p.Type = a.typeExprOf(t, "") + a.params[n] = p + } + } + } res := core.PrepareResult{ Command: a.command, Columns: a.columns, @@ -213,9 +227,39 @@ func (a *analyzer) analyzeSelect(s *ast.SelectStmt) error { return err } } + for _, item := range listItems(s.SortClause) { + if sb, ok := item.(*ast.SortBy); ok { + if _, err := a.typeExpr(sb.Node); err != nil { + return fmt.Errorf("order by: %w", err) + } + } + } + for _, n := range []ast.Node{s.LimitCount, s.LimitOffset} { + if err := a.typeLimit(n); err != nil { + return fmt.Errorf("limit: %w", err) + } + } return nil } +// typeLimit types a LIMIT or OFFSET count. A bare placeholder there holds +// whatever the dialect counts rows in. +func (a *analyzer) typeLimit(n ast.Node) error { + if n == nil { + return nil + } + if pr, ok := n.(*ast.ParamRef); ok { + oid, err := a.cat.LimitTypeOID() + if err != nil { + return err + } + a.inferParam(pr.Number, exprType{typeOID: oid}) + return nil + } + _, err := a.typeExpr(n) + return err +} + func (a *analyzer) typeValuesLists(l *ast.List) error { for _, row := range listItems(l) { if _, err := a.typeExpr(row); err != nil { diff --git a/internal/core/analyzer/expr.go b/internal/core/analyzer/expr.go index a36cf25cbf..d034432662 100644 --- a/internal/core/analyzer/expr.go +++ b/internal/core/analyzer/expr.go @@ -134,7 +134,7 @@ func (a *analyzer) typeConst(c *ast.A_Const) (exprType, error) { } func (a *analyzer) boolType(nullable bool) (exprType, error) { - oid, err := a.cat.ConstTypeOID(core.ConstBool) + oid, err := a.cat.BoolTypeOID() if err != nil { return exprType{}, err } @@ -207,10 +207,12 @@ func (a *analyzer) inferParam(number int, t exprType) { if !ok { cur = core.Parameter{Number: number} } - if cur.TypeOID == 0 && cur.DataType == "" && (t.typeOID != 0 || t.typeName != "") { + typed := cur.TypeOID == 0 && cur.DataType == "" && (t.typeOID != 0 || t.typeName != "") + if typed { cur.TypeOID = t.typeOID cur.DataType, cur.IsArray = a.typeNameOf(t) cur.NotNull = !t.nullable + cur.Type = a.typeExprOf(t, "") } if cur.Source == nil && t.sourceAttributeOID != 0 { ad, err := a.cat.LookupAttribute(t.sourceAttributeOID) @@ -221,11 +223,29 @@ func (a *analyzer) inferParam(number int, t exprType) { TableAlias: t.sourceTableAlias, Column: ad.Column, } + if typed { + cur.Type = a.typeExprOf(t, ad.DeclType) + } } } a.params[number] = cur } +// nameParamAfter names a placeholder compared with a function call after +// the function, the way a placeholder compared with a column is named after +// the column. +func (a *analyzer) nameParamAfter(number int, other ast.Node) { + fc, ok := other.(*ast.FuncCall) + if !ok { + return + } + cur := a.params[number] + if cur.Name == "" && cur.Source == nil { + cur.Name = funcCallName(fc) + a.params[number] = cur + } +} + func (a *analyzer) typeAExpr(e *ast.A_Expr) (exprType, error) { // Not every engine classifies its operators, so a zero kind is a plain // operator application rather than an unset field. LIKE and its relatives @@ -272,10 +292,12 @@ func (a *analyzer) typeAExpr(e *ast.A_Expr) (exprType, error) { if pr, ok := e.Lexpr.(*ast.ParamRef); ok && rightT.typeOID != 0 { a.inferParam(pr.Number, rightT) + a.nameParamAfter(pr.Number, e.Rexpr) leftT = rightT } if pr, ok := e.Rexpr.(*ast.ParamRef); ok && leftT.typeOID != 0 { a.inferParam(pr.Number, leftT) + a.nameParamAfter(pr.Number, e.Lexpr) rightT = leftT } @@ -335,6 +357,19 @@ func (a *analyzer) typeIn(e *ast.In) (exprType, error) { return exprType{}, err } } + // "x IN (SELECT ...)" compares x against the subquery's column, and the + // subquery's own placeholders are reported with the rest. + if sel, ok := e.Sel.(*ast.SelectStmt); ok { + cols, err := a.subqueryColumns(sel) + if err != nil { + return exprType{}, err + } + if len(cols) > 0 { + if err := a.typeOperands(e.Expr, exprType{typeOID: cols[0].TypeOID, nullable: !cols[0].NotNull}); err != nil { + return exprType{}, err + } + } + } return a.boolType(false) } @@ -385,10 +420,26 @@ func (a *analyzer) typeCase(e *ast.CaseExpr) (exprType, error) { return t, nil } -// typeCoalesce types COALESCE, which is its first argument's type and is null -// only when every argument is. +// typeCoalesce types COALESCE, which is its first typed argument's type and +// is null only when every argument is. func (a *analyzer) typeCoalesce(e *ast.CoalesceExpr) (exprType, error) { - return a.typeFirstOf(listItems(e.Args), false) + var out exprType + found := false + nullable := true + for _, n := range listItems(e.Args) { + t, err := a.typeExpr(n) + if err != nil { + return exprType{}, err + } + if !found && t.typeOID != 0 { + // The result is an expression's, not the column's it came from. + out = exprType{typeOID: t.typeOID, typeName: t.typeName} + found = true + } + nullable = nullable && t.nullable + } + out.nullable = nullable + return out, nil } // typeFirstOf types a set of alternative results, taking the first one that has @@ -591,7 +642,7 @@ func (a *analyzer) resolveOperator(name string, leftOID, rightOID int64) (core.O // operator's name implies: a comparison yields a boolean and anything // else yields the type it was applied to. if a.cat.IsComparisonOperator(name) { - boolOID, err := a.cat.ConstTypeOID(core.ConstBool) + boolOID, err := a.cat.BoolTypeOID() if err != nil { return core.OperatorOverload{}, err } @@ -631,13 +682,17 @@ func (a *analyzer) typeFuncCall(f *ast.FuncCall) (exprType, error) { } args := listItems(f.Args) - argTypes := make([]int64, 0, len(args)) + argTypes := make([]exprType, 0, len(args)) + argOIDs := make([]int64, 0, len(args)) + anyNullable := false for _, arg := range args { t, err := a.typeExpr(arg) if err != nil { return exprType{}, err } - argTypes = append(argTypes, t.typeOID) + argTypes = append(argTypes, t) + argOIDs = append(argOIDs, t.typeOID) + anyNullable = anyNullable || t.nullable } overloads, err := a.cat.FindProcs(name, nil) @@ -650,30 +705,84 @@ func (a *analyzer) typeFuncCall(f *ast.FuncCall) (exprType, error) { // rather than failing the query. return exprType{nullable: true}, nil } - p := pickOverload(overloads, argTypes) - // An argument that is a bare placeholder takes the parameter's type. + p := a.pickOverload(overloads, argOIDs) + // An argument that is a bare placeholder takes the parameter's type, + // unless the parameter is polymorphic and says nothing. for i, arg := range args { if i >= len(p.ArgTypes) { break } + if a.isPolymorphicOID(p.ArgTypes[i]) { + continue + } if err := a.typeOperands(arg, exprType{typeOID: p.ArgTypes[i]}); err != nil { return exprType{}, err } } - return exprType{typeOID: a.returnType(p, argTypes), nullable: p.ReturnNullable}, nil + ret := a.returnType(p, argTypes) + ret.nullable = p.ReturnNullable + if !p.NeverNull && anyNullable && a.cat.PropagatesNullable() { + ret.nullable = true + } + return ret, nil } -// returnType resolves a polymorphic return type — max(anyelement) and its -// like — to the type the call was made with. -func (a *analyzer) returnType(p core.ProcOverload, argTypes []int64) int64 { - if p.ReturnTypeOID == 0 || len(argTypes) == 0 || argTypes[0] == 0 { - return p.ReturnTypeOID +// returnType resolves a polymorphic return type — max(anyelement), or a +// seed's "$2" for the type of the second argument — to the type the call was +// made with. +func (a *analyzer) returnType(p core.ProcOverload, argTypes []exprType) exprType { + if p.ReturnTypeOID == 0 || len(argTypes) == 0 { + return exprType{typeOID: p.ReturnTypeOID} } name, err := a.cat.TypeName(p.ReturnTypeOID) - if err != nil || !isPolymorphic(name) { - return p.ReturnTypeOID + if err != nil { + return exprType{typeOID: p.ReturnTypeOID} + } + if n, ok := argIndex(name); ok { + if n < len(argTypes) { + return exprType{typeOID: argTypes[n].typeOID, typeName: argTypes[n].typeName} + } + return exprType{} + } + if isPolymorphic(name) && argTypes[0].typeOID != 0 { + return exprType{typeOID: argTypes[0].typeOID} + } + return exprType{typeOID: p.ReturnTypeOID} +} + +// argIndex reads a seed's "$n" pseudo-type as the zero-based index of the +// argument whose type it stands for. +func argIndex(typeName string) (int, bool) { + rest, ok := strings.CutPrefix(typeName, "$") + if !ok { + return 0, false } - return argTypes[0] + n := 0 + for _, r := range rest { + if r < '0' || r > '9' { + return 0, false + } + n = n*10 + int(r-'0') + } + if n == 0 { + return 0, false + } + return n - 1, true +} + +// isPolymorphicOID reports whether a parameter type accepts any argument. +func (a *analyzer) isPolymorphicOID(oid int64) bool { + if oid == 0 { + return true + } + name, err := a.cat.TypeName(oid) + if err != nil { + return false + } + if _, ok := argIndex(name); ok { + return true + } + return isPolymorphic(name) } func isPolymorphic(typeName string) bool { @@ -687,30 +796,32 @@ func isPolymorphic(typeName string) bool { } // pickOverload chooses the overload whose parameters the call's arguments -// match, preferring an exact match on types over one on arity alone. -func pickOverload(overloads []core.ProcOverload, argTypes []int64) core.ProcOverload { - var byArity *core.ProcOverload +// match best: an exact type match on a parameter beats a polymorphic one, +// which beats a mismatch, and any overload of the right arity beats one of +// the wrong arity. +func (a *analyzer) pickOverload(overloads []core.ProcOverload, argTypes []int64) core.ProcOverload { + best := -1 + bestScore := -1 for i := range overloads { ov := &overloads[i] if len(ov.ArgTypes) != len(argTypes) { continue } - if byArity == nil { - byArity = ov - } - exact := true + score := 0 for j, oid := range argTypes { - if oid != ov.ArgTypes[j] { - exact = false - break + switch { + case oid != 0 && oid == ov.ArgTypes[j]: + score += 2 + case a.isPolymorphicOID(ov.ArgTypes[j]): + score += 1 } } - if exact { - return *ov + if score > bestScore { + best, bestScore = i, score } } - if byArity != nil { - return *byArity + if best >= 0 { + return overloads[best] } return overloads[0] } diff --git a/internal/core/analyzer/projection.go b/internal/core/analyzer/projection.go index 64c37f7251..f3293dcbc6 100644 --- a/internal/core/analyzer/projection.go +++ b/internal/core/analyzer/projection.go @@ -23,6 +23,21 @@ func (a *analyzer) projectTarget(rt *ast.ResTarget) error { if err != nil { return err } + // A placeholder selected directly is named by its alias and, when nothing + // constrains it, typed as the dialect types such a placeholder. + if pr, ok := rt.Val.(*ast.ParamRef); ok { + if rt.Name != nil && *rt.Name != "" { + if p := a.params[pr.Number]; p.Name == "" { + p.Name = *rt.Name + a.params[pr.Number] = p + } + } + if t.typeOID == 0 && t.typeName == "" { + if oid, ok := a.cat.UntypedTypeOID(); ok { + t = exprType{typeOID: oid, nullable: true} + } + } + } col := core.Column{ Name: targetName(rt, fields), TypeOID: t.typeOID, @@ -32,10 +47,65 @@ func (a *analyzer) projectTarget(rt *ast.ResTarget) error { } col.DataType, col.IsArray = a.typeNameOf(t) a.decorateSource(&col, t.sourceAttributeOID, t.sourceTableAlias) + col.Type = a.typeExprOf(t, col.DeclType) + if rt.Name == nil || *rt.Name == "" { + a.qualifyDuplicate(&col, t.sourceTableAlias) + } a.columns = append(a.columns, col) return nil } +// qualifyDuplicate names a column after its relation when an earlier result +// column from another relation already has its name, in a dialect that +// does so. +func (a *analyzer) qualifyDuplicate(col *core.Column, alias string) { + if alias == "" || !a.cat.QualifiesDuplicateColumns() { + return + } + for _, prev := range a.columns { + if prev.Name != col.Name { + continue + } + prevAlias := "" + if prev.Source != nil { + prevAlias = prev.Source.TableAlias + } + if prevAlias != alias { + col.Name = alias + "." + col.Name + return + } + } +} + +// typeExprOf writes a type as an expression. A source column's declared +// spelling carries what the catalog's flat name cannot, so it is parsed +// when there is one; otherwise the expression is the type's name, wrapped +// in an array when the type is one. Nullability comes from the spelling +// when the spelling says anything about it, and from the analysis +// otherwise. +func (a *analyzer) typeExprOf(t exprType, declType string) *core.TypeExpr { + name, isArray := a.typeNameOf(t) + if name == "" && declType == "" { + return nil + } + var expr *core.TypeExpr + if declType != "" { + expr = core.ParseTypeExpr(declType) + if isArray && expr.Name != "array" { + expr = &core.TypeExpr{Name: "array", Args: []core.TypeArg{{Type: expr}}} + } + } else { + expr = core.ParseTypeExpr(name) + if isArray { + expr = &core.TypeExpr{Name: "array", Args: []core.TypeArg{{Type: expr}}} + } + } + if !expr.HasNullable() { + expr.Nullable = t.nullable + } + return expr +} + func (a *analyzer) decorateSource(col *core.Column, attOID int64, tableAlias string) { if attOID == 0 { return @@ -109,6 +179,8 @@ func (a *analyzer) emitStar(rt *ast.ResTarget, fields []string) { } col.DataType, col.IsArray = a.typeNameOf(exprType{typeOID: c.TypeOID}) a.decorateSource(&col, c.AttOID, rel.alias) + col.Type = a.typeExprOf(exprType{typeOID: c.TypeOID, nullable: !c.NotNull}, col.DeclType) + a.qualifyDuplicate(&col, rel.alias) a.columns = append(a.columns, col) star.Columns = append(star.Columns, core.StarColumn{ Relation: rel.alias, diff --git a/internal/core/dialect.go b/internal/core/dialect.go index 5b3286e901..bad80dedcb 100644 --- a/internal/core/dialect.go +++ b/internal/core/dialect.go @@ -54,6 +54,29 @@ func (c *Catalog) DialectFlag(dialectOID int64, key string) (string, error) { // same ones the seeded types have. const FlagComparisonOperators = "operators.comparison" +// FlagBoolType holds the type a comparison or predicate yields, when it is +// not the type a boolean literal has: ClickHouse compares to UInt8 while +// writing true as Bool. +const FlagBoolType = "types.bool" + +// FlagLimitType holds the type a LIMIT or OFFSET count has, which is what a +// placeholder in one is typed as. +const FlagLimitType = "types.limit" + +// FlagUntypedType holds the type a placeholder takes when nothing in the +// query constrains it, for a dialect that gives such a placeholder one. +const FlagUntypedType = "types.untyped" + +// FlagPropagateNullable is set when a function's result is nullable whenever +// one of its arguments is, the way ClickHouse's ordinary functions behave. +const FlagPropagateNullable = "functions.propagate_nullable" + +// FlagQualifyDuplicateColumns is set for a dialect that names a result +// column after its relation when an earlier result column from another +// relation has the same name, as ClickHouse names the second id of a join +// e.id. +const FlagQualifyDuplicateColumns = "columns.qualify_duplicates" + // FlagCastCategories holds the categories whose types are all implicitly // castable to one another, as the dialect's seed declared them, so that a type // arriving after the seed — an extension's, say — can join its category. @@ -109,3 +132,61 @@ func (c *Catalog) ConstTypeOID(kind string) (int64, error) { } return c.TypeOID(name) } + +// BoolTypeOID returns the type a comparison or predicate yields. +func (c *Catalog) BoolTypeOID() (int64, error) { + if c.dialectOID != 0 { + if name, _ := c.DialectFlag(c.dialectOID, FlagBoolType); name != "" { + return c.TypeOID(name) + } + } + return c.ConstTypeOID(ConstBool) +} + +// LimitTypeOID returns the type a LIMIT or OFFSET count has, falling back to +// the type of an integer literal. +func (c *Catalog) LimitTypeOID() (int64, error) { + if c.dialectOID != 0 { + if name, _ := c.DialectFlag(c.dialectOID, FlagLimitType); name != "" { + return c.TypeOID(name) + } + } + return c.ConstTypeOID(ConstInteger) +} + +// UntypedTypeOID returns the type an unconstrained placeholder takes, and +// whether the dialect gives it one at all. +func (c *Catalog) UntypedTypeOID() (int64, bool) { + if c.dialectOID == 0 { + return 0, false + } + name, _ := c.DialectFlag(c.dialectOID, FlagUntypedType) + if name == "" { + return 0, false + } + oid, err := c.TypeOID(name) + if err != nil { + return 0, false + } + return oid, true +} + +// PropagatesNullable reports whether a function's result is nullable +// whenever one of its arguments is. +func (c *Catalog) PropagatesNullable() bool { + if c.dialectOID == 0 { + return false + } + v, _ := c.DialectFlag(c.dialectOID, FlagPropagateNullable) + return v == "true" +} + +// QualifiesDuplicateColumns reports whether a result column that repeats an +// earlier one's name from another relation is named after its relation. +func (c *Catalog) QualifiesDuplicateColumns() bool { + if c.dialectOID == 0 { + return false + } + v, _ := c.DialectFlag(c.dialectOID, FlagQualifyDuplicateColumns) + return v == "true" +} diff --git a/internal/core/proc.go b/internal/core/proc.go index baee74e132..2409a4c4af 100644 --- a/internal/core/proc.go +++ b/internal/core/proc.go @@ -17,11 +17,22 @@ type ProcSpec struct { ReturnTypeOID int64 ReturnSet bool ReturnNullable bool - Strict bool - VariadicKind string - Args []ProcArg + // NeverNull marks a function whose result is never NULL even when an + // argument is, in a dialect that otherwise propagates nullability. + NeverNull bool + Strict bool + VariadicKind string + Args []ProcArg } +// The proc table stores nullability as one integer: 0 leaves it to the +// dialect's rule, 1 is always nullable and 2 is never nullable. +const ( + nullableDefault int64 = 0 + nullableAlways int64 = 1 + nullableNever int64 = 2 +) + type ProcArg struct { Name string TypeOID int64 @@ -44,7 +55,7 @@ func (c *Catalog) CreateProc(p ProcSpec) (int64, error) { Kind: p.Kind, ReturnTypeOid: p.ReturnTypeOID, ReturnSet: boolToInt64(p.ReturnSet), - ReturnNullable: boolToInt64(p.ReturnNullable), + ReturnNullable: returnNullable(p), Strict: boolToInt64(p.Strict), VariadicKind: p.VariadicKind, }) @@ -71,12 +82,23 @@ func (c *Catalog) CreateProc(p ProcSpec) (int64, error) { return procOID, nil } +func returnNullable(p ProcSpec) int64 { + switch { + case p.NeverNull: + return nullableNever + case p.ReturnNullable: + return nullableAlways + } + return nullableDefault +} + type ProcOverload struct { OID int64 Name string Kind string ReturnTypeOID int64 ReturnNullable bool + NeverNull bool ArgTypes []int64 } @@ -99,7 +121,8 @@ func (c *Catalog) FindProcs(name string, namespaceOIDs []int64) ([]ProcOverload, Name: r.Name, Kind: r.Kind, ReturnTypeOID: r.ReturnTypeOid, - ReturnNullable: r.ReturnNullable != 0, + ReturnNullable: r.ReturnNullable == nullableAlways, + NeverNull: r.ReturnNullable == nullableNever, }) } } else { @@ -121,7 +144,8 @@ func (c *Catalog) FindProcs(name string, namespaceOIDs []int64) ([]ProcOverload, Name: r.Name, Kind: r.Kind, ReturnTypeOID: r.ReturnTypeOid, - ReturnNullable: r.ReturnNullable != 0, + ReturnNullable: r.ReturnNullable == nullableAlways, + NeverNull: r.ReturnNullable == nullableNever, }) } } diff --git a/internal/core/schema/schema.go b/internal/core/schema/schema.go index 19dc3cc827..470d6716b5 100644 --- a/internal/core/schema/schema.go +++ b/internal/core/schema/schema.go @@ -162,7 +162,7 @@ func applyCreateTable(cat *core.Catalog, stmt *ast.CreateTableStmt) error { Num: i + 1, NotNull: col.IsNotNull || col.PrimaryKey, IsPrimaryKey: col.PrimaryKey, - DeclType: col.TypeName.Name, + DeclType: declType(col.TypeName), Hidden: col.IsHidden, }); err != nil { return fmt.Errorf("attr %s.%s: %w", stmt.Name.Name, col.Colname, err) @@ -234,7 +234,7 @@ func applyAlterTable(cat *core.Catalog, stmt *ast.AlterTableStmt) error { Num: num, NotNull: cmd.Def.IsNotNull || cmd.Def.PrimaryKey, IsPrimaryKey: cmd.Def.PrimaryKey, - DeclType: cmd.Def.TypeName.Name, + DeclType: declType(cmd.Def.TypeName), }); err != nil { return err } @@ -427,3 +427,15 @@ func columnTypeOID(cat *core.Catalog, col *ast.ColumnDef) (int64, error) { } return cat.ResolveTypeName(name) } + +// declType is the type as the schema spelled it: an engine that folds or +// reduces the name for the catalog keeps the full spelling alongside. +func declType(tn *ast.TypeName) string { + if tn == nil { + return "" + } + if tn.Spelling != "" { + return tn.Spelling + } + return tn.Name +} diff --git a/internal/core/seed/seed.go b/internal/core/seed/seed.go index cc0c5a7977..4dc7525092 100644 --- a/internal/core/seed/seed.go +++ b/internal/core/seed/seed.go @@ -63,6 +63,24 @@ type Settings struct { // Bool names the type comparisons return. Bool string `json:"bool,omitempty"` + // Limit names the type a LIMIT or OFFSET count has, which is what a + // placeholder in one is typed as. Unset, it is the integer literal's. + Limit string `json:"limit,omitempty"` + + // Untyped names the type a placeholder takes when nothing in the query + // constrains it. Unset, such a placeholder stays untyped. + Untyped string `json:"untyped,omitempty"` + + // PropagateNullable makes a function's result nullable whenever one of + // its arguments is, the way ClickHouse's ordinary functions behave, + // unless the function is seeded as never null. + PropagateNullable bool `json:"propagate_nullable,omitempty"` + + // QualifyDuplicateColumns names a result column after its relation when + // an earlier result column from another relation has the same name, as + // ClickHouse names the second id of a join e.id. + QualifyDuplicateColumns bool `json:"qualify_duplicate_columns,omitempty"` + // Comparison operators are registered as (T, T) -> Bool for every type in // ComparisonCategories. Comparison []string `json:"comparison,omitempty"` @@ -108,11 +126,16 @@ type Cast struct { // Function is a function the dialect ships with. Kind is 'f'unction, // 'a'ggregate, 'w'indow or 'p'rocedure. type Function struct { - Name string `json:"name"` - Kind string `json:"kind,omitempty"` - Args []Arg `json:"args,omitempty"` + Name string `json:"name,omitempty"` + Kind string `json:"kind,omitempty"` + Args []Arg `json:"args,omitempty"` + // Returns names the result type, or "$1", "$2"... for the type of that + // argument. Returns string `json:"returns"` Nullable bool `json:"nullable,omitempty"` + // NeverNull marks a result that is never NULL even when an argument + // is, in a dialect that propagates nullability. + NeverNull bool `json:"never_null,omitempty"` } // Relation is a table or view the dialect ships with, such as one of @@ -450,6 +473,31 @@ func (b *builder) consts() error { return err } } + for key, name := range map[string]string{ + core.FlagBoolType: b.settings.Bool, + core.FlagLimitType: b.settings.Limit, + core.FlagUntypedType: b.settings.Untyped, + } { + if name == "" { + continue + } + if _, ok := b.oids[strings.ToLower(name)]; !ok { + return fmt.Errorf("seed %s: %s names unknown type %q", b.settings.Dialect, key, name) + } + if err := b.cat.SetDialectFlag(b.dialectOID, key, name); err != nil { + return err + } + } + if b.settings.PropagateNullable { + if err := b.cat.SetDialectFlag(b.dialectOID, core.FlagPropagateNullable, "true"); err != nil { + return err + } + } + if b.settings.QualifyDuplicateColumns { + if err := b.cat.SetDialectFlag(b.dialectOID, core.FlagQualifyDuplicateColumns, "true"); err != nil { + return err + } + } // A schema declares types the seed knows nothing about — enums, domains, // arrays, a SQLite column typed whatever the author felt like. Recording // the comparison operators lets the catalog give those types the same ones. @@ -602,6 +650,7 @@ func (b *builder) addFunction(fn Function) error { Kind: fn.Kind, ReturnTypeOID: returnOID, ReturnNullable: fn.Nullable, + NeverNull: fn.NeverNull, Args: args, }) if err != nil { diff --git a/internal/core/typeexpr.go b/internal/core/typeexpr.go new file mode 100644 index 0000000000..bebb7e83ce --- /dev/null +++ b/internal/core/typeexpr.go @@ -0,0 +1,198 @@ +package core + +import ( + "strconv" + "strings" +) + +// TypeExpr is a type written as a call expression: a lowercased name applied +// to arguments that are other types, integers, booleans or strings, each +// with an optional label, and a nullable flag at whatever depth it applies. +// Nothing about a nested type is special-cased, so an array of nullable +// strings is array(string nullable), a map is map(string, uint32) and a +// named tuple is tuple(lat: float64, lon: float64). The catalog resolves the +// names; the expression only records what was declared or inferred. +type TypeExpr struct { + Name string `json:"name"` + Nullable bool `json:"nullable,omitempty"` + Args []TypeArg `json:"args,omitempty"` +} + +// TypeArg is one argument of a TypeExpr: exactly one of Type, Int, Bool or +// String is set. +type TypeArg struct { + Label string `json:"label,omitempty"` + Type *TypeExpr `json:"type,omitempty"` + Int *int64 `json:"int,omitempty"` + Bool *bool `json:"bool,omitempty"` + String *string `json:"string,omitempty"` +} + +// ParseTypeExpr reads a type spelled the way every dialect spells one, as a +// name optionally applied to a parenthesised, comma-separated argument list +// whose entries may be labelled (`lat Float64` in a tuple, `'a' = 1` in an +// enum). A Nullable(T) wrapper becomes T with Nullable set, and a trailing +// [] becomes an array of the element. +func ParseTypeExpr(s string) *TypeExpr { + s = strings.TrimSpace(s) + if element, ok := strings.CutSuffix(s, ArraySuffix); ok { + return &TypeExpr{Name: "array", Args: []TypeArg{{Type: ParseTypeExpr(element)}}} + } + name, args := splitTypeArgs(s) + name = strings.ToLower(name) + if name == "nullable" && len(args) == 1 { + t := ParseTypeExpr(args[0]) + t.Nullable = true + return t + } + t := &TypeExpr{Name: name} + for _, a := range args { + t.Args = append(t.Args, parseTypeArg(a)) + } + return t +} + +func parseTypeArg(a string) TypeArg { + a = strings.TrimSpace(a) + if strings.HasPrefix(a, "'") { + end := quotedEnd(a) + lit := strings.ReplaceAll(strings.ReplaceAll(a[1:end-1], `\'`, `'`), `''`, `'`) + if rest := strings.TrimSpace(a[end:]); strings.HasPrefix(rest, "=") { + arg := parseTypeArg(rest[1:]) + arg.Label = lit + return arg + } + return TypeArg{String: &lit} + } + if n, err := strconv.ParseInt(a, 10, 64); err == nil { + return TypeArg{Int: &n} + } + switch strings.ToLower(a) { + case "true", "false": + b := strings.EqualFold(a, "true") + return TypeArg{Bool: &b} + } + // A label is a word before a space that comes before any parenthesis, + // as in `lat Float64` or `tags Array(String)`. + head := a + if p := strings.IndexByte(a, '('); p >= 0 { + head = a[:p] + } + if i := strings.IndexByte(head, ' '); i > 0 { + arg := parseTypeArg(a[i+1:]) + arg.Label = a[:i] + return arg + } + return TypeArg{Type: ParseTypeExpr(a)} +} + +// quotedEnd returns the index just past the single-quoted literal starting +// at the beginning of s, honouring backslash escapes and doubled quotes. +func quotedEnd(s string) int { + for i := 1; i < len(s); i++ { + switch { + case s[i] == '\\' && i+1 < len(s): + i++ + case s[i] == '\'' && i+1 < len(s) && s[i+1] == '\'': + i++ + case s[i] == '\'': + return i + 1 + } + } + return len(s) +} + +// splitTypeArgs splits `Base(arg, arg)` into its base name and top-level +// arguments, leaving nested parentheses and quoted strings intact. +func splitTypeArgs(t string) (string, []string) { + open := strings.IndexByte(t, '(') + if open < 0 || !strings.HasSuffix(t, ")") { + return t, nil + } + base := strings.TrimSpace(t[:open]) + inner := t[open+1 : len(t)-1] + var ( + args []string + depth int + quote byte + start int + ) + for i := 0; i < len(inner); i++ { + c := inner[i] + switch { + case quote != 0: + if c == '\\' { + i++ + } else if c == quote { + quote = 0 + } + case c == '\'' || c == '"' || c == '`': + quote = c + case c == '(': + depth++ + case c == ')': + depth-- + case c == ',' && depth == 0: + args = append(args, strings.TrimSpace(inner[start:i])) + start = i + 1 + } + } + if last := strings.TrimSpace(inner[start:]); last != "" || len(args) > 0 { + args = append(args, last) + } + return base, args +} + +// HasNullable reports whether the expression marks nullability anywhere, +// which tells whether the spelling it came from said so itself. +func (t *TypeExpr) HasNullable() bool { + if t == nil { + return false + } + if t.Nullable { + return true + } + for _, a := range t.Args { + if a.Type.HasNullable() { + return true + } + } + return false +} + +// String renders the expression in its canonical text form, with a trailing +// "nullable" marking a nullable type. +func (t *TypeExpr) String() string { + if t == nil { + return "" + } + var b strings.Builder + b.WriteString(t.Name) + if len(t.Args) > 0 { + b.WriteByte('(') + for i, a := range t.Args { + if i > 0 { + b.WriteString(", ") + } + if a.Label != "" { + b.WriteString(a.Label) + b.WriteString(": ") + } + switch { + case a.Type != nil: + b.WriteString(a.Type.String()) + case a.Int != nil: + b.WriteString(strconv.FormatInt(*a.Int, 10)) + case a.Bool != nil: + b.WriteString(strconv.FormatBool(*a.Bool)) + case a.String != nil: + b.WriteString("'" + strings.ReplaceAll(*a.String, "'", `\'`) + "'") + } + } + b.WriteByte(')') + } + if t.Nullable { + b.WriteString(" nullable") + } + return b.String() +} diff --git a/internal/endtoend/case_test.go b/internal/endtoend/case_test.go index 183b965a2a..0fb5e8f100 100644 --- a/internal/endtoend/case_test.go +++ b/internal/endtoend/case_test.go @@ -52,17 +52,22 @@ func parseStderr(t *testing.T, dir, testctx string) []byte { return nil } +// parseStdout reads the command's expected output: output.json for a +// command that prints JSON, so editors highlight it, otherwise stdout.txt. func parseStdout(t *testing.T, dir string) []byte { t.Helper() - path := filepath.Join(dir, "stdout.txt") - if _, err := os.Stat(path); os.IsNotExist(err) { - return nil - } - blob, err := os.ReadFile(path) - if err != nil { - t.Fatal(err) + for _, name := range []string{"output.json", "stdout.txt"} { + path := filepath.Join(dir, name) + if _, err := os.Stat(path); os.IsNotExist(err) { + continue + } + blob, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + return blob } - return blob + return nil } // hasSQLCConfig reports whether dir contains an sqlc configuration file. diff --git a/internal/endtoend/testdata/analyze_ast/postgresql/stdout.txt b/internal/endtoend/testdata/analyze_ast/postgresql/output.json similarity index 93% rename from internal/endtoend/testdata/analyze_ast/postgresql/stdout.txt rename to internal/endtoend/testdata/analyze_ast/postgresql/output.json index 1473a99cc5..427fe5af92 100644 --- a/internal/endtoend/testdata/analyze_ast/postgresql/stdout.txt +++ b/internal/endtoend/testdata/analyze_ast/postgresql/output.json @@ -5,9 +5,9 @@ "columns": [ { "name": "name", - "data_type": "text", - "not_null": true, - "is_array": false, + "type": { + "name": "text" + }, "table": "authors" } ], @@ -16,9 +16,9 @@ "number": 1, "column": { "name": "id", - "data_type": "bigserial", - "not_null": true, - "is_array": false, + "type": { + "name": "bigserial" + }, "table": "authors" } } diff --git a/internal/endtoend/testdata/analyze_basic/clickhouse/fixture.sql b/internal/endtoend/testdata/analyze_basic/clickhouse/fixture.sql new file mode 100644 index 0000000000..88c30875e9 --- /dev/null +++ b/internal/endtoend/testdata/analyze_basic/clickhouse/fixture.sql @@ -0,0 +1 @@ +INSERT INTO events (id, name, tag, amount, created) VALUES (1, 'signup', NULL, 9.5, '2024-01-01 00:00:00'); diff --git a/internal/endtoend/testdata/analyze_basic/clickhouse/stdout.txt b/internal/endtoend/testdata/analyze_basic/clickhouse/output.json similarity index 50% rename from internal/endtoend/testdata/analyze_basic/clickhouse/stdout.txt rename to internal/endtoend/testdata/analyze_basic/clickhouse/output.json index 11e3a90d5d..ad9963d7e7 100644 --- a/internal/endtoend/testdata/analyze_basic/clickhouse/stdout.txt +++ b/internal/endtoend/testdata/analyze_basic/clickhouse/output.json @@ -5,37 +5,38 @@ "columns": [ { "name": "id", - "data_type": "uint64", - "not_null": true, - "is_array": false, + "type": { + "name": "uint64" + }, "table": "events" }, { "name": "name", - "data_type": "string", - "not_null": true, - "is_array": false, + "type": { + "name": "string" + }, "table": "events" }, { "name": "tag", - "data_type": "string", - "not_null": false, - "is_array": false, + "type": { + "name": "string", + "nullable": true + }, "table": "events" }, { "name": "amount", - "data_type": "float64", - "not_null": true, - "is_array": false, + "type": { + "name": "float64" + }, "table": "events" }, { "name": "created", - "data_type": "datetime", - "not_null": true, - "is_array": false, + "type": { + "name": "datetime" + }, "table": "events" } ], diff --git a/internal/endtoend/testdata/analyze_basic/duckdb/stdout.txt b/internal/endtoend/testdata/analyze_basic/duckdb/output.json similarity index 51% rename from internal/endtoend/testdata/analyze_basic/duckdb/stdout.txt rename to internal/endtoend/testdata/analyze_basic/duckdb/output.json index 597820269b..6b7631c5de 100644 --- a/internal/endtoend/testdata/analyze_basic/duckdb/stdout.txt +++ b/internal/endtoend/testdata/analyze_basic/duckdb/output.json @@ -5,37 +5,38 @@ "columns": [ { "name": "id", - "data_type": "bigint", - "not_null": true, - "is_array": false, + "type": { + "name": "bigint" + }, "table": "authors" }, { "name": "name", - "data_type": "text", - "not_null": true, - "is_array": false, + "type": { + "name": "text" + }, "table": "authors" }, { "name": "bio", - "data_type": "text", - "not_null": false, - "is_array": false, + "type": { + "name": "text", + "nullable": true + }, "table": "authors" }, { "name": "royalties", - "data_type": "decimal", - "not_null": true, - "is_array": false, + "type": { + "name": "decimal" + }, "table": "authors" }, { "name": "created", - "data_type": "timestamp", - "not_null": true, - "is_array": false, + "type": { + "name": "timestamp" + }, "table": "authors" } ], diff --git a/internal/endtoend/testdata/analyze_basic/googlesql/stdout.txt b/internal/endtoend/testdata/analyze_basic/googlesql/output.json similarity index 57% rename from internal/endtoend/testdata/analyze_basic/googlesql/stdout.txt rename to internal/endtoend/testdata/analyze_basic/googlesql/output.json index 4075f80992..b14d4a249e 100644 --- a/internal/endtoend/testdata/analyze_basic/googlesql/stdout.txt +++ b/internal/endtoend/testdata/analyze_basic/googlesql/output.json @@ -5,16 +5,16 @@ "columns": [ { "name": "id", - "data_type": "int64", - "not_null": true, - "is_array": false, + "type": { + "name": "int64" + }, "table": "users" }, { "name": "name", - "data_type": "string", - "not_null": true, - "is_array": false, + "type": { + "name": "string" + }, "table": "users" } ], @@ -23,9 +23,9 @@ "number": 1, "column": { "name": "id", - "data_type": "int64", - "not_null": true, - "is_array": false, + "type": { + "name": "int64" + }, "table": "users" } } diff --git a/internal/endtoend/testdata/analyze_basic/mssql/stdout.txt b/internal/endtoend/testdata/analyze_basic/mssql/output.json similarity index 50% rename from internal/endtoend/testdata/analyze_basic/mssql/stdout.txt rename to internal/endtoend/testdata/analyze_basic/mssql/output.json index 7226354b09..181660561a 100644 --- a/internal/endtoend/testdata/analyze_basic/mssql/stdout.txt +++ b/internal/endtoend/testdata/analyze_basic/mssql/output.json @@ -5,37 +5,38 @@ "columns": [ { "name": "id", - "data_type": "bigint", - "not_null": true, - "is_array": false, + "type": { + "name": "bigint" + }, "table": "authors" }, { "name": "name", - "data_type": "nvarchar", - "not_null": true, - "is_array": false, + "type": { + "name": "nvarchar" + }, "table": "authors" }, { "name": "bio", - "data_type": "nvarchar", - "not_null": false, - "is_array": false, + "type": { + "name": "nvarchar", + "nullable": true + }, "table": "authors" }, { "name": "royalties", - "data_type": "decimal", - "not_null": true, - "is_array": false, + "type": { + "name": "decimal" + }, "table": "authors" }, { "name": "created", - "data_type": "datetime2", - "not_null": true, - "is_array": false, + "type": { + "name": "datetime2" + }, "table": "authors" } ], diff --git a/internal/endtoend/testdata/analyze_basic/sqlite/stdout.txt b/internal/endtoend/testdata/analyze_basic/mysql/output.json similarity index 57% rename from internal/endtoend/testdata/analyze_basic/sqlite/stdout.txt rename to internal/endtoend/testdata/analyze_basic/mysql/output.json index aa26281658..83dde3e339 100644 --- a/internal/endtoend/testdata/analyze_basic/sqlite/stdout.txt +++ b/internal/endtoend/testdata/analyze_basic/mysql/output.json @@ -5,16 +5,16 @@ "columns": [ { "name": "id", - "data_type": "integer", - "not_null": true, - "is_array": false, + "type": { + "name": "bigint" + }, "table": "users" }, { "name": "name", - "data_type": "text", - "not_null": true, - "is_array": false, + "type": { + "name": "varchar" + }, "table": "users" } ], @@ -23,9 +23,9 @@ "number": 1, "column": { "name": "id", - "data_type": "integer", - "not_null": true, - "is_array": false, + "type": { + "name": "bigint" + }, "table": "users" } } diff --git a/internal/endtoend/testdata/analyze_basic/postgresql/stdout.txt b/internal/endtoend/testdata/analyze_basic/postgresql/output.json similarity index 55% rename from internal/endtoend/testdata/analyze_basic/postgresql/stdout.txt rename to internal/endtoend/testdata/analyze_basic/postgresql/output.json index b93421c32a..36356b73c3 100644 --- a/internal/endtoend/testdata/analyze_basic/postgresql/stdout.txt +++ b/internal/endtoend/testdata/analyze_basic/postgresql/output.json @@ -5,23 +5,24 @@ "columns": [ { "name": "id", - "data_type": "bigserial", - "not_null": true, - "is_array": false, + "type": { + "name": "bigserial" + }, "table": "authors" }, { "name": "name", - "data_type": "text", - "not_null": true, - "is_array": false, + "type": { + "name": "text" + }, "table": "authors" }, { "name": "bio", - "data_type": "text", - "not_null": false, - "is_array": false, + "type": { + "name": "text", + "nullable": true + }, "table": "authors" } ], @@ -30,9 +31,9 @@ "number": 1, "column": { "name": "id", - "data_type": "bigserial", - "not_null": true, - "is_array": false, + "type": { + "name": "bigserial" + }, "table": "authors" } } diff --git a/internal/endtoend/testdata/analyze_basic/mysql/stdout.txt b/internal/endtoend/testdata/analyze_basic/sqlite/output.json similarity index 57% rename from internal/endtoend/testdata/analyze_basic/mysql/stdout.txt rename to internal/endtoend/testdata/analyze_basic/sqlite/output.json index e599e249aa..8799d17310 100644 --- a/internal/endtoend/testdata/analyze_basic/mysql/stdout.txt +++ b/internal/endtoend/testdata/analyze_basic/sqlite/output.json @@ -5,16 +5,16 @@ "columns": [ { "name": "id", - "data_type": "bigint", - "not_null": true, - "is_array": false, + "type": { + "name": "integer" + }, "table": "users" }, { "name": "name", - "data_type": "varchar", - "not_null": true, - "is_array": false, + "type": { + "name": "text" + }, "table": "users" } ], @@ -23,9 +23,9 @@ "number": 1, "column": { "name": "id", - "data_type": "bigint", - "not_null": true, - "is_array": false, + "type": { + "name": "integer" + }, "table": "users" } } diff --git a/internal/endtoend/testdata/analyze_dml/duckdb/stdout.txt b/internal/endtoend/testdata/analyze_dml/duckdb/output.json similarity index 62% rename from internal/endtoend/testdata/analyze_dml/duckdb/stdout.txt rename to internal/endtoend/testdata/analyze_dml/duckdb/output.json index 1ac3372918..afdd96cfd7 100644 --- a/internal/endtoend/testdata/analyze_dml/duckdb/stdout.txt +++ b/internal/endtoend/testdata/analyze_dml/duckdb/output.json @@ -5,9 +5,9 @@ "columns": [ { "name": "id", - "data_type": "bigint", - "not_null": true, - "is_array": false, + "type": { + "name": "bigint" + }, "table": "authors" } ], @@ -16,9 +16,9 @@ "number": 1, "column": { "name": "id", - "data_type": "bigint", - "not_null": true, - "is_array": false, + "type": { + "name": "bigint" + }, "table": "authors" } }, @@ -26,9 +26,9 @@ "number": 2, "column": { "name": "name", - "data_type": "text", - "not_null": true, - "is_array": false, + "type": { + "name": "text" + }, "table": "authors" } }, @@ -36,9 +36,10 @@ "number": 3, "column": { "name": "bio", - "data_type": "text", - "not_null": false, - "is_array": false, + "type": { + "name": "text", + "nullable": true + }, "table": "authors" } } @@ -53,9 +54,9 @@ "number": 1, "column": { "name": "name", - "data_type": "text", - "not_null": true, - "is_array": false, + "type": { + "name": "text" + }, "table": "authors" } }, @@ -63,9 +64,10 @@ "number": 2, "column": { "name": "bio", - "data_type": "text", - "not_null": false, - "is_array": false, + "type": { + "name": "text", + "nullable": true + }, "table": "authors" } }, @@ -73,9 +75,9 @@ "number": 3, "column": { "name": "id", - "data_type": "bigint", - "not_null": true, - "is_array": false, + "type": { + "name": "bigint" + }, "table": "authors" } } @@ -90,9 +92,9 @@ "number": 1, "column": { "name": "name", - "data_type": "text", - "not_null": true, - "is_array": false, + "type": { + "name": "text" + }, "table": "authors" } } @@ -104,16 +106,16 @@ "columns": [ { "name": "id", - "data_type": "bigint", - "not_null": true, - "is_array": false, + "type": { + "name": "bigint" + }, "table": "books" }, { "name": "title", - "data_type": "text", - "not_null": true, - "is_array": false, + "type": { + "name": "text" + }, "table": "books" } ], @@ -122,9 +124,10 @@ "number": 1, "column": { "name": "price", - "data_type": "decimal", - "not_null": false, - "is_array": false, + "type": { + "name": "decimal", + "nullable": true + }, "table": "books" } } diff --git a/internal/endtoend/testdata/analyze_dml/googlesql/stdout.txt b/internal/endtoend/testdata/analyze_dml/googlesql/output.json similarity index 61% rename from internal/endtoend/testdata/analyze_dml/googlesql/stdout.txt rename to internal/endtoend/testdata/analyze_dml/googlesql/output.json index 5148457d6d..4e696578c2 100644 --- a/internal/endtoend/testdata/analyze_dml/googlesql/stdout.txt +++ b/internal/endtoend/testdata/analyze_dml/googlesql/output.json @@ -8,9 +8,9 @@ "number": 1, "column": { "name": "id", - "data_type": "int64", - "not_null": true, - "is_array": false, + "type": { + "name": "int64" + }, "table": "users" } }, @@ -18,9 +18,9 @@ "number": 2, "column": { "name": "name", - "data_type": "string", - "not_null": true, - "is_array": false, + "type": { + "name": "string" + }, "table": "users" } }, @@ -28,9 +28,10 @@ "number": 3, "column": { "name": "bio", - "data_type": "string", - "not_null": false, - "is_array": false, + "type": { + "name": "string", + "nullable": true + }, "table": "users" } } @@ -42,16 +43,16 @@ "columns": [ { "name": "id", - "data_type": "int64", - "not_null": true, - "is_array": false, + "type": { + "name": "int64" + }, "table": "users" }, { "name": "name", - "data_type": "string", - "not_null": true, - "is_array": false, + "type": { + "name": "string" + }, "table": "users" } ], @@ -60,9 +61,9 @@ "number": 1, "column": { "name": "id", - "data_type": "int64", - "not_null": true, - "is_array": false, + "type": { + "name": "int64" + }, "table": "users" } }, @@ -70,9 +71,9 @@ "number": 2, "column": { "name": "name", - "data_type": "string", - "not_null": true, - "is_array": false, + "type": { + "name": "string" + }, "table": "users" } } @@ -87,9 +88,10 @@ "number": 1, "column": { "name": "bio", - "data_type": "string", - "not_null": false, - "is_array": false, + "type": { + "name": "string", + "nullable": true + }, "table": "users" } }, @@ -97,9 +99,9 @@ "number": 2, "column": { "name": "id", - "data_type": "int64", - "not_null": true, - "is_array": false, + "type": { + "name": "int64" + }, "table": "users" } } @@ -111,16 +113,16 @@ "columns": [ { "name": "id", - "data_type": "int64", - "not_null": true, - "is_array": false, + "type": { + "name": "int64" + }, "table": "users" }, { "name": "name", - "data_type": "string", - "not_null": true, - "is_array": false, + "type": { + "name": "string" + }, "table": "users" } ], @@ -129,9 +131,9 @@ "number": 1, "column": { "name": "name", - "data_type": "string", - "not_null": true, - "is_array": false, + "type": { + "name": "string" + }, "table": "users" } }, @@ -139,9 +141,9 @@ "number": 2, "column": { "name": "id", - "data_type": "int64", - "not_null": true, - "is_array": false, + "type": { + "name": "int64" + }, "table": "users" } } @@ -156,9 +158,9 @@ "number": 1, "column": { "name": "id", - "data_type": "int64", - "not_null": true, - "is_array": false, + "type": { + "name": "int64" + }, "table": "users" } } @@ -170,9 +172,9 @@ "columns": [ { "name": "id", - "data_type": "int64", - "not_null": true, - "is_array": false, + "type": { + "name": "int64" + }, "table": "users" } ], @@ -181,9 +183,9 @@ "number": 1, "column": { "name": "id", - "data_type": "int64", - "not_null": true, - "is_array": false, + "type": { + "name": "int64" + }, "table": "users" } } diff --git a/internal/endtoend/testdata/analyze_dml/mssql/stdout.txt b/internal/endtoend/testdata/analyze_dml/mssql/output.json similarity index 64% rename from internal/endtoend/testdata/analyze_dml/mssql/stdout.txt rename to internal/endtoend/testdata/analyze_dml/mssql/output.json index d2ed5f27e0..24e2efb3e4 100644 --- a/internal/endtoend/testdata/analyze_dml/mssql/stdout.txt +++ b/internal/endtoend/testdata/analyze_dml/mssql/output.json @@ -5,9 +5,9 @@ "columns": [ { "name": "id", - "data_type": "bigint", - "not_null": true, - "is_array": false, + "type": { + "name": "bigint" + }, "table": "authors" } ], @@ -16,9 +16,9 @@ "number": 1, "column": { "name": "name", - "data_type": "nvarchar", - "not_null": true, - "is_array": false, + "type": { + "name": "nvarchar" + }, "table": "authors" } }, @@ -26,9 +26,10 @@ "number": 2, "column": { "name": "bio", - "data_type": "nvarchar", - "not_null": false, - "is_array": false, + "type": { + "name": "nvarchar", + "nullable": true + }, "table": "authors" } } @@ -43,9 +44,9 @@ "number": 1, "column": { "name": "name", - "data_type": "nvarchar", - "not_null": true, - "is_array": false, + "type": { + "name": "nvarchar" + }, "table": "authors" } }, @@ -53,9 +54,9 @@ "number": 2, "column": { "name": "id", - "data_type": "bigint", - "not_null": true, - "is_array": false, + "type": { + "name": "bigint" + }, "table": "authors" } } @@ -70,9 +71,10 @@ "number": 1, "column": { "name": "price", - "data_type": "decimal", - "not_null": false, - "is_array": false, + "type": { + "name": "decimal", + "nullable": true + }, "table": "books" } }, @@ -80,9 +82,9 @@ "number": 2, "column": { "name": "name", - "data_type": "nvarchar", - "not_null": true, - "is_array": false, + "type": { + "name": "nvarchar" + }, "table": "authors" } } @@ -97,9 +99,9 @@ "number": 1, "column": { "name": "name", - "data_type": "nvarchar", - "not_null": true, - "is_array": false, + "type": { + "name": "nvarchar" + }, "table": "authors" } } diff --git a/internal/endtoend/testdata/analyze_dml/mysql/stdout.txt b/internal/endtoend/testdata/analyze_dml/mysql/output.json similarity index 66% rename from internal/endtoend/testdata/analyze_dml/mysql/stdout.txt rename to internal/endtoend/testdata/analyze_dml/mysql/output.json index abaa295663..e474c04a04 100644 --- a/internal/endtoend/testdata/analyze_dml/mysql/stdout.txt +++ b/internal/endtoend/testdata/analyze_dml/mysql/output.json @@ -8,9 +8,9 @@ "number": 1, "column": { "name": "id", - "data_type": "bigint", - "not_null": true, - "is_array": false, + "type": { + "name": "bigint" + }, "table": "users" } }, @@ -18,9 +18,9 @@ "number": 2, "column": { "name": "name", - "data_type": "varchar", - "not_null": true, - "is_array": false, + "type": { + "name": "varchar" + }, "table": "users" } }, @@ -28,9 +28,10 @@ "number": 3, "column": { "name": "bio", - "data_type": "text", - "not_null": false, - "is_array": false, + "type": { + "name": "text", + "nullable": true + }, "table": "users" } } @@ -45,9 +46,10 @@ "number": 1, "column": { "name": "bio", - "data_type": "text", - "not_null": false, - "is_array": false, + "type": { + "name": "text", + "nullable": true + }, "table": "users" } }, @@ -55,9 +57,9 @@ "number": 2, "column": { "name": "id", - "data_type": "bigint", - "not_null": true, - "is_array": false, + "type": { + "name": "bigint" + }, "table": "users" } } @@ -72,9 +74,9 @@ "number": 1, "column": { "name": "id", - "data_type": "bigint", - "not_null": true, - "is_array": false, + "type": { + "name": "bigint" + }, "table": "users" } } @@ -89,9 +91,9 @@ "number": 1, "column": { "name": "name", - "data_type": "varchar", - "not_null": true, - "is_array": false, + "type": { + "name": "varchar" + }, "table": "users" } } diff --git a/internal/endtoend/testdata/analyze_dml/postgresql/stdout.txt b/internal/endtoend/testdata/analyze_dml/postgresql/output.json similarity index 60% rename from internal/endtoend/testdata/analyze_dml/postgresql/stdout.txt rename to internal/endtoend/testdata/analyze_dml/postgresql/output.json index bcfce7b15a..876a1ed726 100644 --- a/internal/endtoend/testdata/analyze_dml/postgresql/stdout.txt +++ b/internal/endtoend/testdata/analyze_dml/postgresql/output.json @@ -5,23 +5,24 @@ "columns": [ { "name": "id", - "data_type": "bigserial", - "not_null": true, - "is_array": false, + "type": { + "name": "bigserial" + }, "table": "users" }, { "name": "name", - "data_type": "text", - "not_null": true, - "is_array": false, + "type": { + "name": "text" + }, "table": "users" }, { "name": "bio", - "data_type": "text", - "not_null": false, - "is_array": false, + "type": { + "name": "text", + "nullable": true + }, "table": "users" } ], @@ -30,9 +31,9 @@ "number": 1, "column": { "name": "name", - "data_type": "text", - "not_null": true, - "is_array": false, + "type": { + "name": "text" + }, "table": "users" } }, @@ -40,9 +41,10 @@ "number": 2, "column": { "name": "bio", - "data_type": "text", - "not_null": false, - "is_array": false, + "type": { + "name": "text", + "nullable": true + }, "table": "users" } } @@ -57,9 +59,10 @@ "number": 1, "column": { "name": "bio", - "data_type": "text", - "not_null": false, - "is_array": false, + "type": { + "name": "text", + "nullable": true + }, "table": "users" } }, @@ -67,9 +70,9 @@ "number": 2, "column": { "name": "id", - "data_type": "bigserial", - "not_null": true, - "is_array": false, + "type": { + "name": "bigserial" + }, "table": "users" } } @@ -81,16 +84,16 @@ "columns": [ { "name": "id", - "data_type": "bigserial", - "not_null": true, - "is_array": false, + "type": { + "name": "bigserial" + }, "table": "users" }, { "name": "name", - "data_type": "text", - "not_null": true, - "is_array": false, + "type": { + "name": "text" + }, "table": "users" } ], @@ -99,9 +102,9 @@ "number": 1, "column": { "name": "name", - "data_type": "text", - "not_null": true, - "is_array": false, + "type": { + "name": "text" + }, "table": "users" } }, @@ -109,9 +112,9 @@ "number": 2, "column": { "name": "id", - "data_type": "bigserial", - "not_null": true, - "is_array": false, + "type": { + "name": "bigserial" + }, "table": "users" } } @@ -126,9 +129,9 @@ "number": 1, "column": { "name": "id", - "data_type": "bigserial", - "not_null": true, - "is_array": false, + "type": { + "name": "bigserial" + }, "table": "users" } } @@ -140,9 +143,9 @@ "columns": [ { "name": "id", - "data_type": "bigserial", - "not_null": true, - "is_array": false, + "type": { + "name": "bigserial" + }, "table": "users" } ], @@ -151,9 +154,9 @@ "number": 1, "column": { "name": "id", - "data_type": "bigserial", - "not_null": true, - "is_array": false, + "type": { + "name": "bigserial" + }, "table": "users" } } diff --git a/internal/endtoend/testdata/analyze_dml/sqlite/stdout.txt b/internal/endtoend/testdata/analyze_dml/sqlite/output.json similarity index 60% rename from internal/endtoend/testdata/analyze_dml/sqlite/stdout.txt rename to internal/endtoend/testdata/analyze_dml/sqlite/output.json index 78d512d3e7..06863367fa 100644 --- a/internal/endtoend/testdata/analyze_dml/sqlite/stdout.txt +++ b/internal/endtoend/testdata/analyze_dml/sqlite/output.json @@ -5,23 +5,24 @@ "columns": [ { "name": "id", - "data_type": "integer", - "not_null": true, - "is_array": false, + "type": { + "name": "integer" + }, "table": "users" }, { "name": "name", - "data_type": "text", - "not_null": true, - "is_array": false, + "type": { + "name": "text" + }, "table": "users" }, { "name": "bio", - "data_type": "text", - "not_null": false, - "is_array": false, + "type": { + "name": "text", + "nullable": true + }, "table": "users" } ], @@ -30,9 +31,9 @@ "number": 1, "column": { "name": "name", - "data_type": "text", - "not_null": true, - "is_array": false, + "type": { + "name": "text" + }, "table": "users" } }, @@ -40,9 +41,10 @@ "number": 2, "column": { "name": "bio", - "data_type": "text", - "not_null": false, - "is_array": false, + "type": { + "name": "text", + "nullable": true + }, "table": "users" } } @@ -57,9 +59,10 @@ "number": 1, "column": { "name": "bio", - "data_type": "text", - "not_null": false, - "is_array": false, + "type": { + "name": "text", + "nullable": true + }, "table": "users" } }, @@ -67,9 +70,9 @@ "number": 2, "column": { "name": "id", - "data_type": "integer", - "not_null": true, - "is_array": false, + "type": { + "name": "integer" + }, "table": "users" } } @@ -84,9 +87,9 @@ "number": 1, "column": { "name": "id", - "data_type": "integer", - "not_null": true, - "is_array": false, + "type": { + "name": "integer" + }, "table": "users" } } diff --git a/internal/endtoend/testdata/analyze_exec/clickhouse/exec.json b/internal/endtoend/testdata/analyze_exec/clickhouse/exec.json new file mode 100644 index 0000000000..c92bfa3dab --- /dev/null +++ b/internal/endtoend/testdata/analyze_exec/clickhouse/exec.json @@ -0,0 +1,5 @@ +{ + "command": "analyze", + "args": ["--dialect", "clickhouse", "--schema", "schema.sql", "query.sql"], + "contexts": ["base"] +} diff --git a/internal/endtoend/testdata/analyze_exec/clickhouse/fixture.sql b/internal/endtoend/testdata/analyze_exec/clickhouse/fixture.sql new file mode 100644 index 0000000000..9adc5fa938 --- /dev/null +++ b/internal/endtoend/testdata/analyze_exec/clickhouse/fixture.sql @@ -0,0 +1,2 @@ +INSERT INTO events VALUES (1, 'signup', NULL, 9.5), (2, 'login', 'web', 0.5); +INSERT INTO users VALUES (1, 'a@example.com'); diff --git a/internal/endtoend/testdata/analyze_exec/clickhouse/output.json b/internal/endtoend/testdata/analyze_exec/clickhouse/output.json new file mode 100644 index 0000000000..3709c4909b --- /dev/null +++ b/internal/endtoend/testdata/analyze_exec/clickhouse/output.json @@ -0,0 +1,130 @@ +[ + { + "name": "CreateUser", + "cmd": ":exec", + "columns": [], + "params": [ + { + "number": 1, + "column": { + "name": "id", + "type": { + "name": "uint64" + }, + "table": "users" + } + }, + { + "number": 2, + "column": { + "name": "email", + "type": { + "name": "string" + }, + "table": "users" + } + } + ] + }, + { + "name": "CreateEvent", + "cmd": ":exec", + "columns": [], + "params": [ + { + "number": 1, + "column": { + "name": "id", + "type": { + "name": "uint64" + }, + "table": "events" + } + }, + { + "number": 2, + "column": { + "name": "name", + "type": { + "name": "string" + }, + "table": "events" + } + }, + { + "number": 3, + "column": { + "name": "tag", + "type": { + "name": "string", + "nullable": true + }, + "table": "events" + } + }, + { + "number": 4, + "column": { + "name": "amount", + "type": { + "name": "float64" + }, + "table": "events" + } + } + ] + }, + { + "name": "CreateEvents", + "cmd": ":exec", + "columns": [], + "params": [ + { + "number": 1, + "column": { + "name": "id", + "type": { + "name": "uint64" + }, + "table": "events" + } + }, + { + "number": 2, + "column": { + "name": "name", + "type": { + "name": "string" + }, + "table": "events" + } + }, + { + "number": 3, + "column": { + "name": "id", + "type": { + "name": "uint64" + }, + "table": "events" + } + }, + { + "number": 4, + "column": { + "name": "name", + "type": { + "name": "string" + }, + "table": "events" + } + } + ] + }, + { + "name": "DropUsers", + "cmd": ":exec", + "columns": [], + "params": [] + } +] diff --git a/internal/endtoend/testdata/analyze_exec/clickhouse/query.sql b/internal/endtoend/testdata/analyze_exec/clickhouse/query.sql new file mode 100644 index 0000000000..a27579ef0f --- /dev/null +++ b/internal/endtoend/testdata/analyze_exec/clickhouse/query.sql @@ -0,0 +1,11 @@ +-- name: CreateUser :exec +INSERT INTO users (id, email) VALUES (?, ?); + +-- name: CreateEvent :exec +INSERT INTO events VALUES (?, ?, ?, ?); + +-- name: CreateEvents :exec +INSERT INTO events (id, name) VALUES (?, ?), (?, ?); + +-- name: DropUsers :exec +TRUNCATE TABLE users; diff --git a/internal/endtoend/testdata/analyze_exec/clickhouse/schema.sql b/internal/endtoend/testdata/analyze_exec/clickhouse/schema.sql new file mode 100644 index 0000000000..9c04e10685 --- /dev/null +++ b/internal/endtoend/testdata/analyze_exec/clickhouse/schema.sql @@ -0,0 +1,11 @@ +CREATE TABLE events ( + id UInt64, + name String, + tag Nullable(String), + amount Float64 +) ENGINE = MergeTree ORDER BY id; + +CREATE TABLE users ( + id UInt64, + email String +) ENGINE = MergeTree ORDER BY id; diff --git a/internal/endtoend/testdata/analyze_expressions/clickhouse/exec.json b/internal/endtoend/testdata/analyze_expressions/clickhouse/exec.json new file mode 100644 index 0000000000..c92bfa3dab --- /dev/null +++ b/internal/endtoend/testdata/analyze_expressions/clickhouse/exec.json @@ -0,0 +1,5 @@ +{ + "command": "analyze", + "args": ["--dialect", "clickhouse", "--schema", "schema.sql", "query.sql"], + "contexts": ["base"] +} diff --git a/internal/endtoend/testdata/analyze_expressions/clickhouse/fixture.sql b/internal/endtoend/testdata/analyze_expressions/clickhouse/fixture.sql new file mode 100644 index 0000000000..9adc5fa938 --- /dev/null +++ b/internal/endtoend/testdata/analyze_expressions/clickhouse/fixture.sql @@ -0,0 +1,2 @@ +INSERT INTO events VALUES (1, 'signup', NULL, 9.5), (2, 'login', 'web', 0.5); +INSERT INTO users VALUES (1, 'a@example.com'); diff --git a/internal/endtoend/testdata/analyze_expressions/clickhouse/output.json b/internal/endtoend/testdata/analyze_expressions/clickhouse/output.json new file mode 100644 index 0000000000..e3e4716696 --- /dev/null +++ b/internal/endtoend/testdata/analyze_expressions/clickhouse/output.json @@ -0,0 +1,342 @@ +[ + { + "name": "Aggregates", + "cmd": ":one", + "columns": [ + { + "name": "n", + "type": { + "name": "uint64" + } + }, + { + "name": "top", + "type": { + "name": "float64" + } + }, + { + "name": "sum(amount)", + "type": { + "name": "float64" + } + }, + { + "name": "first_tag", + "type": { + "name": "string", + "nullable": true + } + }, + { + "name": "uniq(name)", + "type": { + "name": "uint64" + } + } + ], + "params": [] + }, + { + "name": "Expressions", + "cmd": ":many", + "columns": [ + { + "name": "next_id", + "type": { + "name": "uint64" + } + }, + { + "name": "maybe", + "type": { + "name": "string", + "nullable": true + } + }, + { + "name": "tag_or_none", + "type": { + "name": "string" + } + }, + { + "name": "lower(name)", + "type": { + "name": "string" + } + }, + { + "name": "today", + "type": { + "name": "date" + } + }, + { + "name": "big", + "type": { + "name": "uint8" + } + }, + { + "name": "word", + "type": { + "name": "string", + "nullable": true + } + } + ], + "params": [] + }, + { + "name": "LeftJoinDefaults", + "cmd": ":many", + "columns": [ + { + "name": "id", + "type": { + "name": "uint64" + }, + "table": "events" + }, + { + "name": "name", + "type": { + "name": "string" + }, + "table": "events" + }, + { + "name": "email", + "type": { + "name": "string" + }, + "table": "users" + } + ], + "params": [] + }, + { + "name": "Positional", + "cmd": ":many", + "columns": [ + { + "name": "id", + "type": { + "name": "uint64" + }, + "table": "events" + }, + { + "name": "name", + "type": { + "name": "string" + }, + "table": "events" + } + ], + "params": [ + { + "number": 1, + "column": { + "name": "name", + "type": { + "name": "string" + }, + "table": "events" + } + }, + { + "number": 2, + "column": { + "name": "amount", + "type": { + "name": "float64" + }, + "table": "events" + } + }, + { + "number": 3, + "column": { + "name": "tag", + "type": { + "name": "string", + "nullable": true + }, + "table": "events" + } + } + ] + }, + { + "name": "Named", + "cmd": ":many", + "columns": [ + { + "name": "id", + "type": { + "name": "uint64" + }, + "table": "events" + }, + { + "name": "name", + "type": { + "name": "string" + }, + "table": "events" + } + ], + "params": [ + { + "number": 1, + "column": { + "name": "name", + "type": { + "name": "string" + }, + "table": "events" + } + }, + { + "number": 2, + "column": { + "name": "tag", + "type": { + "name": "string", + "nullable": true + }, + "table": "events" + } + }, + { + "number": 3, + "column": { + "name": "amount", + "type": { + "name": "float64" + }, + "table": "events" + } + }, + { + "number": 4, + "column": { + "name": "amount", + "type": { + "name": "float64" + }, + "table": "events" + } + } + ] + }, + { + "name": "Functions", + "cmd": ":many", + "columns": [ + { + "name": "id", + "type": { + "name": "uint64" + }, + "table": "events" + } + ], + "params": [ + { + "number": 1, + "column": { + "name": "lower", + "type": { + "name": "string" + } + } + }, + { + "number": 2, + "column": { + "name": "id", + "type": { + "name": "uint64" + }, + "table": "events" + } + }, + { + "number": 3, + "column": { + "name": "toDate", + "type": { + "name": "date" + } + } + } + ] + }, + { + "name": "Paging", + "cmd": ":many", + "columns": [ + { + "name": "id", + "type": { + "name": "uint64" + }, + "table": "events" + } + ], + "params": [ + { + "number": 1, + "column": { + "name": "", + "type": { + "name": "uint64" + } + } + }, + { + "number": 2, + "column": { + "name": "", + "type": { + "name": "uint64" + } + } + } + ] + }, + { + "name": "Projected", + "cmd": ":one", + "columns": [ + { + "name": "echo", + "type": { + "name": "nothing", + "nullable": true + } + }, + { + "name": "lit", + "type": { + "name": "string" + } + } + ], + "params": [ + { + "number": 1, + "column": { + "name": "echo", + "type": { + "name": "nothing", + "nullable": true + } + } + } + ] + } +] diff --git a/internal/endtoend/testdata/analyze_expressions/clickhouse/query.sql b/internal/endtoend/testdata/analyze_expressions/clickhouse/query.sql new file mode 100644 index 0000000000..dd5604e452 --- /dev/null +++ b/internal/endtoend/testdata/analyze_expressions/clickhouse/query.sql @@ -0,0 +1,29 @@ +-- name: Aggregates :one +SELECT count() AS n, max(amount) AS top, sum(amount), min(tag) AS first_tag, uniq(name) +FROM events; + +-- name: Expressions :many +SELECT id + 1 AS next_id, nullIf(name, '') AS maybe, coalesce(tag, 'none') AS tag_or_none, + lower(name), toDate(now()) AS today, id > 1 AS big, if(id = 1, 'one', NULL) AS word +FROM events; + +-- name: LeftJoinDefaults :many +SELECT e.id, e.name, u.email +FROM events e +LEFT JOIN users u ON u.id = e.id; + +-- name: Positional :many +SELECT id, name FROM events WHERE name = ? AND amount > ? AND tag = ?; + +-- name: Named :many +SELECT id, name FROM events +WHERE name = sqlc.arg(name) AND tag = sqlc.narg(tag) AND (amount > sqlc.arg(amount) OR amount < sqlc.arg(amount)); + +-- name: Functions :many +SELECT id FROM events WHERE lower(name) = ? AND id IN (?) AND toDate(now()) > ?; + +-- name: Paging :many +SELECT id FROM events ORDER BY id LIMIT ? OFFSET ?; + +-- name: Projected :one +SELECT ? AS echo, 'literal' AS lit; diff --git a/internal/endtoend/testdata/analyze_expressions/clickhouse/schema.sql b/internal/endtoend/testdata/analyze_expressions/clickhouse/schema.sql new file mode 100644 index 0000000000..9c04e10685 --- /dev/null +++ b/internal/endtoend/testdata/analyze_expressions/clickhouse/schema.sql @@ -0,0 +1,11 @@ +CREATE TABLE events ( + id UInt64, + name String, + tag Nullable(String), + amount Float64 +) ENGINE = MergeTree ORDER BY id; + +CREATE TABLE users ( + id UInt64, + email String +) ENGINE = MergeTree ORDER BY id; diff --git a/internal/endtoend/testdata/analyze_extension/postgresql/stdout.txt b/internal/endtoend/testdata/analyze_extension/postgresql/output.json similarity index 57% rename from internal/endtoend/testdata/analyze_extension/postgresql/stdout.txt rename to internal/endtoend/testdata/analyze_extension/postgresql/output.json index 9bf76757ac..af10afec06 100644 --- a/internal/endtoend/testdata/analyze_extension/postgresql/stdout.txt +++ b/internal/endtoend/testdata/analyze_extension/postgresql/output.json @@ -5,16 +5,16 @@ "columns": [ { "name": "id", - "data_type": "bigserial", - "not_null": true, - "is_array": false, + "type": { + "name": "bigserial" + }, "table": "users" }, { "name": "email", - "data_type": "citext", - "not_null": true, - "is_array": false, + "type": { + "name": "citext" + }, "table": "users" } ], @@ -23,9 +23,9 @@ "number": 1, "column": { "name": "email", - "data_type": "citext", - "not_null": true, - "is_array": false, + "type": { + "name": "citext" + }, "table": "users" } } @@ -37,9 +37,9 @@ "columns": [ { "name": "id", - "data_type": "bigserial", - "not_null": true, - "is_array": false, + "type": { + "name": "bigserial" + }, "table": "users" } ], @@ -51,16 +51,16 @@ "columns": [ { "name": "id", - "data_type": "bigserial", - "not_null": true, - "is_array": false, + "type": { + "name": "bigserial" + }, "table": "users" }, { "name": "score", - "data_type": "real", - "not_null": true, - "is_array": false + "type": { + "name": "real" + } } ], "params": [ @@ -68,9 +68,9 @@ "number": 1, "column": { "name": "", - "data_type": "text", - "not_null": true, - "is_array": false + "type": { + "name": "text" + } } } ] diff --git a/internal/endtoend/testdata/analyze_params/clickhouse/fixture.sql b/internal/endtoend/testdata/analyze_params/clickhouse/fixture.sql new file mode 100644 index 0000000000..f13ad59a8d --- /dev/null +++ b/internal/endtoend/testdata/analyze_params/clickhouse/fixture.sql @@ -0,0 +1,2 @@ +INSERT INTO events (id, name, tag, amount, created) VALUES (1, 'signup', NULL, 9.5, '2024-01-01 00:00:00'); +INSERT INTO events (id, name, tag, amount, created) VALUES (2, 'login', 'web', 0.5, '2024-01-02 00:00:00'); diff --git a/internal/endtoend/testdata/analyze_params/clickhouse/stdout.txt b/internal/endtoend/testdata/analyze_params/clickhouse/output.json similarity index 57% rename from internal/endtoend/testdata/analyze_params/clickhouse/stdout.txt rename to internal/endtoend/testdata/analyze_params/clickhouse/output.json index 6fb0cc654b..548bb8b7b1 100644 --- a/internal/endtoend/testdata/analyze_params/clickhouse/stdout.txt +++ b/internal/endtoend/testdata/analyze_params/clickhouse/output.json @@ -5,16 +5,16 @@ "columns": [ { "name": "id", - "data_type": "uint64", - "not_null": true, - "is_array": false, + "type": { + "name": "uint64" + }, "table": "events" }, { "name": "name", - "data_type": "string", - "not_null": true, - "is_array": false, + "type": { + "name": "string" + }, "table": "events" } ], @@ -23,9 +23,9 @@ "number": 1, "column": { "name": "id", - "data_type": "uint64", - "not_null": true, - "is_array": false, + "type": { + "name": "uint64" + }, "table": "events" } } @@ -37,16 +37,16 @@ "columns": [ { "name": "id", - "data_type": "uint64", - "not_null": true, - "is_array": false, + "type": { + "name": "uint64" + }, "table": "events" }, { "name": "name", - "data_type": "string", - "not_null": true, - "is_array": false, + "type": { + "name": "string" + }, "table": "events" } ], @@ -55,9 +55,9 @@ "number": 1, "column": { "name": "name", - "data_type": "string", - "not_null": true, - "is_array": false, + "type": { + "name": "string" + }, "table": "events" } }, @@ -65,9 +65,9 @@ "number": 2, "column": { "name": "amount", - "data_type": "float64", - "not_null": true, - "is_array": false, + "type": { + "name": "float64" + }, "table": "events" } } diff --git a/internal/endtoend/testdata/analyze_params/duckdb/stdout.txt b/internal/endtoend/testdata/analyze_params/duckdb/output.json similarity index 58% rename from internal/endtoend/testdata/analyze_params/duckdb/stdout.txt rename to internal/endtoend/testdata/analyze_params/duckdb/output.json index baacd76bbc..0a85a39358 100644 --- a/internal/endtoend/testdata/analyze_params/duckdb/stdout.txt +++ b/internal/endtoend/testdata/analyze_params/duckdb/output.json @@ -5,23 +5,24 @@ "columns": [ { "name": "id", - "data_type": "bigint", - "not_null": true, - "is_array": false, + "type": { + "name": "bigint" + }, "table": "authors" }, { "name": "name", - "data_type": "text", - "not_null": true, - "is_array": false, + "type": { + "name": "text" + }, "table": "authors" }, { "name": "bio", - "data_type": "text", - "not_null": false, - "is_array": false, + "type": { + "name": "text", + "nullable": true + }, "table": "authors" } ], @@ -30,9 +31,9 @@ "number": 1, "column": { "name": "id", - "data_type": "bigint", - "not_null": true, - "is_array": false, + "type": { + "name": "bigint" + }, "table": "authors" } } @@ -44,16 +45,16 @@ "columns": [ { "name": "id", - "data_type": "bigint", - "not_null": true, - "is_array": false, + "type": { + "name": "bigint" + }, "table": "authors" }, { "name": "name", - "data_type": "text", - "not_null": true, - "is_array": false, + "type": { + "name": "text" + }, "table": "authors" } ], @@ -62,9 +63,9 @@ "number": 1, "column": { "name": "name", - "data_type": "text", - "not_null": true, - "is_array": false, + "type": { + "name": "text" + }, "table": "authors" } }, @@ -72,9 +73,9 @@ "number": 2, "column": { "name": "royalties", - "data_type": "decimal", - "not_null": true, - "is_array": false, + "type": { + "name": "decimal" + }, "table": "authors" } } @@ -86,9 +87,9 @@ "columns": [ { "name": "id", - "data_type": "bigint", - "not_null": true, - "is_array": false, + "type": { + "name": "bigint" + }, "table": "authors" } ], @@ -97,9 +98,9 @@ "number": 1, "column": { "name": "name", - "data_type": "text", - "not_null": true, - "is_array": false, + "type": { + "name": "text" + }, "table": "authors" } } diff --git a/internal/endtoend/testdata/analyze_params/mssql/stdout.txt b/internal/endtoend/testdata/analyze_params/mssql/output.json similarity index 56% rename from internal/endtoend/testdata/analyze_params/mssql/stdout.txt rename to internal/endtoend/testdata/analyze_params/mssql/output.json index c5765c94c3..1483213369 100644 --- a/internal/endtoend/testdata/analyze_params/mssql/stdout.txt +++ b/internal/endtoend/testdata/analyze_params/mssql/output.json @@ -5,23 +5,24 @@ "columns": [ { "name": "id", - "data_type": "bigint", - "not_null": true, - "is_array": false, + "type": { + "name": "bigint" + }, "table": "authors" }, { "name": "name", - "data_type": "nvarchar", - "not_null": true, - "is_array": false, + "type": { + "name": "nvarchar" + }, "table": "authors" }, { "name": "bio", - "data_type": "nvarchar", - "not_null": false, - "is_array": false, + "type": { + "name": "nvarchar", + "nullable": true + }, "table": "authors" } ], @@ -30,9 +31,9 @@ "number": 1, "column": { "name": "id", - "data_type": "bigint", - "not_null": true, - "is_array": false, + "type": { + "name": "bigint" + }, "table": "authors" } } @@ -44,16 +45,16 @@ "columns": [ { "name": "id", - "data_type": "bigint", - "not_null": true, - "is_array": false, + "type": { + "name": "bigint" + }, "table": "authors" }, { "name": "name", - "data_type": "nvarchar", - "not_null": true, - "is_array": false, + "type": { + "name": "nvarchar" + }, "table": "authors" } ], @@ -62,9 +63,9 @@ "number": 1, "column": { "name": "name", - "data_type": "nvarchar", - "not_null": true, - "is_array": false, + "type": { + "name": "nvarchar" + }, "table": "authors" } }, @@ -72,9 +73,9 @@ "number": 2, "column": { "name": "royalties", - "data_type": "decimal", - "not_null": true, - "is_array": false, + "type": { + "name": "decimal" + }, "table": "authors" } } diff --git a/internal/endtoend/testdata/analyze_params/mysql/stdout.txt b/internal/endtoend/testdata/analyze_params/mysql/output.json similarity index 57% rename from internal/endtoend/testdata/analyze_params/mysql/stdout.txt rename to internal/endtoend/testdata/analyze_params/mysql/output.json index e5ef23ba6f..7ff80429a7 100644 --- a/internal/endtoend/testdata/analyze_params/mysql/stdout.txt +++ b/internal/endtoend/testdata/analyze_params/mysql/output.json @@ -5,16 +5,16 @@ "columns": [ { "name": "id", - "data_type": "bigint", - "not_null": true, - "is_array": false, + "type": { + "name": "bigint" + }, "table": "users" }, { "name": "name", - "data_type": "varchar", - "not_null": true, - "is_array": false, + "type": { + "name": "varchar" + }, "table": "users" } ], @@ -23,9 +23,9 @@ "number": 1, "column": { "name": "id", - "data_type": "bigint", - "not_null": true, - "is_array": false, + "type": { + "name": "bigint" + }, "table": "users" } } @@ -37,16 +37,16 @@ "columns": [ { "name": "id", - "data_type": "bigint", - "not_null": true, - "is_array": false, + "type": { + "name": "bigint" + }, "table": "users" }, { "name": "name", - "data_type": "varchar", - "not_null": true, - "is_array": false, + "type": { + "name": "varchar" + }, "table": "users" } ], @@ -55,9 +55,9 @@ "number": 1, "column": { "name": "name", - "data_type": "varchar", - "not_null": true, - "is_array": false, + "type": { + "name": "varchar" + }, "table": "users" } }, @@ -65,9 +65,10 @@ "number": 2, "column": { "name": "bio", - "data_type": "text", - "not_null": false, - "is_array": false, + "type": { + "name": "text", + "nullable": true + }, "table": "users" } } @@ -79,16 +80,16 @@ "columns": [ { "name": "id", - "data_type": "bigint", - "not_null": true, - "is_array": false, + "type": { + "name": "bigint" + }, "table": "users" }, { "name": "name", - "data_type": "varchar", - "not_null": true, - "is_array": false, + "type": { + "name": "varchar" + }, "table": "users" } ], @@ -97,9 +98,9 @@ "number": 1, "column": { "name": "ids", - "data_type": "bigint", - "not_null": true, - "is_array": false, + "type": { + "name": "bigint" + }, "table": "users" } } diff --git a/internal/endtoend/testdata/analyze_params/postgresql/stdout.txt b/internal/endtoend/testdata/analyze_params/postgresql/output.json similarity index 57% rename from internal/endtoend/testdata/analyze_params/postgresql/stdout.txt rename to internal/endtoend/testdata/analyze_params/postgresql/output.json index 6bbcf899ff..3d98c7ff65 100644 --- a/internal/endtoend/testdata/analyze_params/postgresql/stdout.txt +++ b/internal/endtoend/testdata/analyze_params/postgresql/output.json @@ -5,16 +5,16 @@ "columns": [ { "name": "id", - "data_type": "bigserial", - "not_null": true, - "is_array": false, + "type": { + "name": "bigserial" + }, "table": "users" }, { "name": "name", - "data_type": "text", - "not_null": true, - "is_array": false, + "type": { + "name": "text" + }, "table": "users" } ], @@ -23,9 +23,9 @@ "number": 1, "column": { "name": "id", - "data_type": "bigserial", - "not_null": true, - "is_array": false, + "type": { + "name": "bigserial" + }, "table": "users" } } @@ -37,16 +37,16 @@ "columns": [ { "name": "id", - "data_type": "bigserial", - "not_null": true, - "is_array": false, + "type": { + "name": "bigserial" + }, "table": "users" }, { "name": "name", - "data_type": "text", - "not_null": true, - "is_array": false, + "type": { + "name": "text" + }, "table": "users" } ], @@ -55,9 +55,9 @@ "number": 1, "column": { "name": "name", - "data_type": "text", - "not_null": true, - "is_array": false, + "type": { + "name": "text" + }, "table": "users" } }, @@ -65,9 +65,10 @@ "number": 2, "column": { "name": "bio", - "data_type": "text", - "not_null": false, - "is_array": false, + "type": { + "name": "text", + "nullable": true + }, "table": "users" } } @@ -79,16 +80,16 @@ "columns": [ { "name": "id", - "data_type": "bigserial", - "not_null": true, - "is_array": false, + "type": { + "name": "bigserial" + }, "table": "users" }, { "name": "name", - "data_type": "text", - "not_null": true, - "is_array": false, + "type": { + "name": "text" + }, "table": "users" } ], @@ -97,9 +98,9 @@ "number": 1, "column": { "name": "ids", - "data_type": "bigserial", - "not_null": true, - "is_array": false, + "type": { + "name": "bigserial" + }, "table": "users" } } @@ -111,16 +112,17 @@ "columns": [ { "name": "id", - "data_type": "bigserial", - "not_null": true, - "is_array": false, + "type": { + "name": "bigserial" + }, "table": "posts" }, { "name": "title", - "data_type": "varchar", - "not_null": false, - "is_array": false, + "type": { + "name": "varchar", + "nullable": true + }, "table": "posts" } ], @@ -129,9 +131,9 @@ "number": 1, "column": { "name": "created", - "data_type": "timestamptz", - "not_null": true, - "is_array": false, + "type": { + "name": "timestamptz" + }, "table": "posts" } }, @@ -139,9 +141,9 @@ "number": 2, "column": { "name": "created", - "data_type": "timestamptz", - "not_null": true, - "is_array": false, + "type": { + "name": "timestamptz" + }, "table": "posts" } } diff --git a/internal/endtoend/testdata/analyze_params/sqlite/stdout.txt b/internal/endtoend/testdata/analyze_params/sqlite/output.json similarity index 58% rename from internal/endtoend/testdata/analyze_params/sqlite/stdout.txt rename to internal/endtoend/testdata/analyze_params/sqlite/output.json index 067fe197be..fd0e721573 100644 --- a/internal/endtoend/testdata/analyze_params/sqlite/stdout.txt +++ b/internal/endtoend/testdata/analyze_params/sqlite/output.json @@ -5,16 +5,16 @@ "columns": [ { "name": "id", - "data_type": "integer", - "not_null": true, - "is_array": false, + "type": { + "name": "integer" + }, "table": "users" }, { "name": "name", - "data_type": "text", - "not_null": true, - "is_array": false, + "type": { + "name": "text" + }, "table": "users" } ], @@ -23,9 +23,9 @@ "number": 1, "column": { "name": "id", - "data_type": "integer", - "not_null": true, - "is_array": false, + "type": { + "name": "integer" + }, "table": "users" } } @@ -37,16 +37,16 @@ "columns": [ { "name": "id", - "data_type": "integer", - "not_null": true, - "is_array": false, + "type": { + "name": "integer" + }, "table": "users" }, { "name": "name", - "data_type": "text", - "not_null": true, - "is_array": false, + "type": { + "name": "text" + }, "table": "users" } ], @@ -55,9 +55,9 @@ "number": 1, "column": { "name": "name", - "data_type": "text", - "not_null": true, - "is_array": false, + "type": { + "name": "text" + }, "table": "users" } }, @@ -65,9 +65,10 @@ "number": 2, "column": { "name": "bio", - "data_type": "text", - "not_null": false, - "is_array": false, + "type": { + "name": "text", + "nullable": true + }, "table": "users" } } @@ -79,9 +80,9 @@ "columns": [ { "name": "total", - "data_type": "integer", - "not_null": true, - "is_array": false + "type": { + "name": "integer" + } } ], "params": [ @@ -89,9 +90,9 @@ "number": 1, "column": { "name": "user_id", - "data_type": "integer", - "not_null": true, - "is_array": false, + "type": { + "name": "integer" + }, "table": "posts" } } diff --git a/internal/endtoend/testdata/analyze_select/duckdb/stdout.txt b/internal/endtoend/testdata/analyze_select/duckdb/output.json similarity index 54% rename from internal/endtoend/testdata/analyze_select/duckdb/stdout.txt rename to internal/endtoend/testdata/analyze_select/duckdb/output.json index 88d31141e5..2b0e161be2 100644 --- a/internal/endtoend/testdata/analyze_select/duckdb/stdout.txt +++ b/internal/endtoend/testdata/analyze_select/duckdb/output.json @@ -5,23 +5,24 @@ "columns": [ { "name": "id", - "data_type": "bigint", - "not_null": true, - "is_array": false, + "type": { + "name": "bigint" + }, "table": "users" }, { "name": "name", - "data_type": "text", - "not_null": true, - "is_array": false, + "type": { + "name": "text" + }, "table": "users" }, { "name": "age", - "data_type": "integer", - "not_null": false, - "is_array": false, + "type": { + "name": "integer", + "nullable": true + }, "table": "users" } ], @@ -33,9 +34,9 @@ "columns": [ { "name": "total", - "data_type": "bigint", - "not_null": true, - "is_array": false + "type": { + "name": "bigint" + } } ], "params": [] @@ -46,37 +47,38 @@ "columns": [ { "name": "name", - "data_type": "text", - "not_null": true, - "is_array": false, + "type": { + "name": "text" + }, "table": "users" }, { "name": "id", - "data_type": "bigint", - "not_null": true, - "is_array": false, + "type": { + "name": "bigint" + }, "table": "posts" }, { "name": "user_id", - "data_type": "bigint", - "not_null": true, - "is_array": false, + "type": { + "name": "bigint" + }, "table": "posts" }, { "name": "title", - "data_type": "text", - "not_null": true, - "is_array": false, + "type": { + "name": "text" + }, "table": "posts" }, { "name": "body", - "data_type": "text", - "not_null": false, - "is_array": false, + "type": { + "name": "text", + "nullable": true + }, "table": "posts" } ], @@ -85,9 +87,9 @@ "number": 1, "column": { "name": "name", - "data_type": "text", - "not_null": true, - "is_array": false, + "type": { + "name": "text" + }, "table": "users" } } @@ -99,16 +101,16 @@ "columns": [ { "name": "name", - "data_type": "text", - "not_null": true, - "is_array": false, + "type": { + "name": "text" + }, "table": "users" }, { "name": "n", - "data_type": "bigint", - "not_null": true, - "is_array": false + "type": { + "name": "bigint" + } } ], "params": [] diff --git a/internal/endtoend/testdata/analyze_select/googlesql/stdout.txt b/internal/endtoend/testdata/analyze_select/googlesql/output.json similarity index 53% rename from internal/endtoend/testdata/analyze_select/googlesql/stdout.txt rename to internal/endtoend/testdata/analyze_select/googlesql/output.json index b2b19ce73f..0bd71ba16a 100644 --- a/internal/endtoend/testdata/analyze_select/googlesql/stdout.txt +++ b/internal/endtoend/testdata/analyze_select/googlesql/output.json @@ -5,23 +5,24 @@ "columns": [ { "name": "id", - "data_type": "int64", - "not_null": true, - "is_array": false, + "type": { + "name": "int64" + }, "table": "users" }, { "name": "name", - "data_type": "string", - "not_null": true, - "is_array": false, + "type": { + "name": "string" + }, "table": "users" }, { "name": "bio", - "data_type": "string", - "not_null": false, - "is_array": false, + "type": { + "name": "string", + "nullable": true + }, "table": "users" } ], @@ -33,9 +34,9 @@ "columns": [ { "name": "total", - "data_type": "int64", - "not_null": true, - "is_array": false + "type": { + "name": "int64" + } } ], "params": [] @@ -46,37 +47,38 @@ "columns": [ { "name": "name", - "data_type": "string", - "not_null": true, - "is_array": false, + "type": { + "name": "string" + }, "table": "users" }, { "name": "id", - "data_type": "int64", - "not_null": true, - "is_array": false, + "type": { + "name": "int64" + }, "table": "posts" }, { "name": "user_id", - "data_type": "int64", - "not_null": true, - "is_array": false, + "type": { + "name": "int64" + }, "table": "posts" }, { "name": "title", - "data_type": "string", - "not_null": false, - "is_array": false, + "type": { + "name": "string", + "nullable": true + }, "table": "posts" }, { "name": "created", - "data_type": "timestamp", - "not_null": true, - "is_array": false, + "type": { + "name": "timestamp" + }, "table": "posts" } ], @@ -85,9 +87,9 @@ "number": 1, "column": { "name": "name", - "data_type": "string", - "not_null": true, - "is_array": false, + "type": { + "name": "string" + }, "table": "users" } } diff --git a/internal/endtoend/testdata/analyze_select/mysql/stdout.txt b/internal/endtoend/testdata/analyze_select/mysql/output.json similarity index 54% rename from internal/endtoend/testdata/analyze_select/mysql/stdout.txt rename to internal/endtoend/testdata/analyze_select/mysql/output.json index 14fac17e42..2ef9380633 100644 --- a/internal/endtoend/testdata/analyze_select/mysql/stdout.txt +++ b/internal/endtoend/testdata/analyze_select/mysql/output.json @@ -5,23 +5,24 @@ "columns": [ { "name": "id", - "data_type": "bigint", - "not_null": true, - "is_array": false, + "type": { + "name": "bigint" + }, "table": "users" }, { "name": "name", - "data_type": "varchar", - "not_null": true, - "is_array": false, + "type": { + "name": "varchar" + }, "table": "users" }, { "name": "bio", - "data_type": "text", - "not_null": false, - "is_array": false, + "type": { + "name": "text", + "nullable": true + }, "table": "users" } ], @@ -33,9 +34,9 @@ "columns": [ { "name": "total", - "data_type": "bigint", - "not_null": true, - "is_array": false + "type": { + "name": "bigint" + } } ], "params": [] @@ -46,37 +47,38 @@ "columns": [ { "name": "name", - "data_type": "varchar", - "not_null": true, - "is_array": false, + "type": { + "name": "varchar" + }, "table": "users" }, { "name": "id", - "data_type": "bigint", - "not_null": true, - "is_array": false, + "type": { + "name": "bigint" + }, "table": "posts" }, { "name": "user_id", - "data_type": "bigint", - "not_null": true, - "is_array": false, + "type": { + "name": "bigint" + }, "table": "posts" }, { "name": "title", - "data_type": "varchar", - "not_null": false, - "is_array": false, + "type": { + "name": "varchar", + "nullable": true + }, "table": "posts" }, { "name": "created", - "data_type": "datetime", - "not_null": true, - "is_array": false, + "type": { + "name": "datetime" + }, "table": "posts" } ], @@ -85,9 +87,9 @@ "number": 1, "column": { "name": "name", - "data_type": "varchar", - "not_null": true, - "is_array": false, + "type": { + "name": "varchar" + }, "table": "users" } } @@ -99,16 +101,17 @@ "columns": [ { "name": "id", - "data_type": "bigint", - "not_null": true, - "is_array": false, + "type": { + "name": "bigint" + }, "table": "posts" }, { "name": "title", - "data_type": "varchar", - "not_null": false, - "is_array": false, + "type": { + "name": "varchar", + "nullable": true + }, "table": "posts" } ], @@ -117,9 +120,10 @@ "number": 1, "column": { "name": "title", - "data_type": "varchar", - "not_null": false, - "is_array": false, + "type": { + "name": "varchar", + "nullable": true + }, "table": "posts" } }, @@ -127,9 +131,9 @@ "number": 2, "column": { "name": "created", - "data_type": "datetime", - "not_null": true, - "is_array": false, + "type": { + "name": "datetime" + }, "table": "posts" } } @@ -141,16 +145,16 @@ "columns": [ { "name": "name", - "data_type": "varchar", - "not_null": true, - "is_array": false, + "type": { + "name": "varchar" + }, "table": "users" }, { "name": "created", - "data_type": "datetime", - "not_null": true, - "is_array": false + "type": { + "name": "datetime" + } } ], "params": [] diff --git a/internal/endtoend/testdata/analyze_select/postgresql/stdout.txt b/internal/endtoend/testdata/analyze_select/postgresql/output.json similarity index 54% rename from internal/endtoend/testdata/analyze_select/postgresql/stdout.txt rename to internal/endtoend/testdata/analyze_select/postgresql/output.json index d424775aa1..75ae4486c3 100644 --- a/internal/endtoend/testdata/analyze_select/postgresql/stdout.txt +++ b/internal/endtoend/testdata/analyze_select/postgresql/output.json @@ -5,23 +5,24 @@ "columns": [ { "name": "id", - "data_type": "bigserial", - "not_null": true, - "is_array": false, + "type": { + "name": "bigserial" + }, "table": "users" }, { "name": "name", - "data_type": "text", - "not_null": true, - "is_array": false, + "type": { + "name": "text" + }, "table": "users" }, { "name": "bio", - "data_type": "text", - "not_null": false, - "is_array": false, + "type": { + "name": "text", + "nullable": true + }, "table": "users" } ], @@ -33,9 +34,9 @@ "columns": [ { "name": "total", - "data_type": "bigint", - "not_null": true, - "is_array": false + "type": { + "name": "bigint" + } } ], "params": [] @@ -46,37 +47,38 @@ "columns": [ { "name": "name", - "data_type": "text", - "not_null": true, - "is_array": false, + "type": { + "name": "text" + }, "table": "users" }, { "name": "id", - "data_type": "bigserial", - "not_null": true, - "is_array": false, + "type": { + "name": "bigserial" + }, "table": "posts" }, { "name": "user_id", - "data_type": "int8", - "not_null": true, - "is_array": false, + "type": { + "name": "int8" + }, "table": "posts" }, { "name": "title", - "data_type": "varchar", - "not_null": false, - "is_array": false, + "type": { + "name": "varchar", + "nullable": true + }, "table": "posts" }, { "name": "created", - "data_type": "timestamptz", - "not_null": true, - "is_array": false, + "type": { + "name": "timestamptz" + }, "table": "posts" } ], @@ -85,9 +87,9 @@ "number": 1, "column": { "name": "name", - "data_type": "text", - "not_null": true, - "is_array": false, + "type": { + "name": "text" + }, "table": "users" } } @@ -99,16 +101,17 @@ "columns": [ { "name": "id", - "data_type": "bigserial", - "not_null": true, - "is_array": false, + "type": { + "name": "bigserial" + }, "table": "posts" }, { "name": "title", - "data_type": "varchar", - "not_null": false, - "is_array": false, + "type": { + "name": "varchar", + "nullable": true + }, "table": "posts" } ], @@ -117,9 +120,10 @@ "number": 1, "column": { "name": "title", - "data_type": "varchar", - "not_null": false, - "is_array": false, + "type": { + "name": "varchar", + "nullable": true + }, "table": "posts" } }, @@ -127,9 +131,9 @@ "number": 2, "column": { "name": "created", - "data_type": "timestamptz", - "not_null": true, - "is_array": false, + "type": { + "name": "timestamptz" + }, "table": "posts" } } @@ -141,16 +145,16 @@ "columns": [ { "name": "name", - "data_type": "text", - "not_null": true, - "is_array": false, + "type": { + "name": "text" + }, "table": "users" }, { "name": "created", - "data_type": "timestamptz", - "not_null": true, - "is_array": false + "type": { + "name": "timestamptz" + } } ], "params": [] diff --git a/internal/endtoend/testdata/analyze_select/sqlite/stdout.txt b/internal/endtoend/testdata/analyze_select/sqlite/output.json similarity index 55% rename from internal/endtoend/testdata/analyze_select/sqlite/stdout.txt rename to internal/endtoend/testdata/analyze_select/sqlite/output.json index 7c525df07c..032d947426 100644 --- a/internal/endtoend/testdata/analyze_select/sqlite/stdout.txt +++ b/internal/endtoend/testdata/analyze_select/sqlite/output.json @@ -5,23 +5,24 @@ "columns": [ { "name": "id", - "data_type": "integer", - "not_null": true, - "is_array": false, + "type": { + "name": "integer" + }, "table": "users" }, { "name": "name", - "data_type": "text", - "not_null": true, - "is_array": false, + "type": { + "name": "text" + }, "table": "users" }, { "name": "bio", - "data_type": "text", - "not_null": false, - "is_array": false, + "type": { + "name": "text", + "nullable": true + }, "table": "users" } ], @@ -33,9 +34,9 @@ "columns": [ { "name": "total", - "data_type": "integer", - "not_null": true, - "is_array": false + "type": { + "name": "integer" + } } ], "params": [] @@ -46,37 +47,38 @@ "columns": [ { "name": "name", - "data_type": "text", - "not_null": true, - "is_array": false, + "type": { + "name": "text" + }, "table": "users" }, { "name": "id", - "data_type": "integer", - "not_null": true, - "is_array": false, + "type": { + "name": "integer" + }, "table": "posts" }, { "name": "user_id", - "data_type": "integer", - "not_null": true, - "is_array": false, + "type": { + "name": "integer" + }, "table": "posts" }, { "name": "title", - "data_type": "text", - "not_null": false, - "is_array": false, + "type": { + "name": "text", + "nullable": true + }, "table": "posts" }, { "name": "created", - "data_type": "text", - "not_null": true, - "is_array": false, + "type": { + "name": "text" + }, "table": "posts" } ], @@ -85,9 +87,9 @@ "number": 1, "column": { "name": "name", - "data_type": "text", - "not_null": true, - "is_array": false, + "type": { + "name": "text" + }, "table": "users" } } @@ -99,16 +101,17 @@ "columns": [ { "name": "id", - "data_type": "integer", - "not_null": true, - "is_array": false, + "type": { + "name": "integer" + }, "table": "posts" }, { "name": "title", - "data_type": "text", - "not_null": false, - "is_array": false, + "type": { + "name": "text", + "nullable": true + }, "table": "posts" } ], @@ -117,9 +120,10 @@ "number": 1, "column": { "name": "title", - "data_type": "text", - "not_null": false, - "is_array": false, + "type": { + "name": "text", + "nullable": true + }, "table": "posts" } }, @@ -127,9 +131,9 @@ "number": 2, "column": { "name": "created", - "data_type": "text", - "not_null": true, - "is_array": false, + "type": { + "name": "text" + }, "table": "posts" } } diff --git a/internal/endtoend/testdata/analyze_subqueries/clickhouse/exec.json b/internal/endtoend/testdata/analyze_subqueries/clickhouse/exec.json new file mode 100644 index 0000000000..c92bfa3dab --- /dev/null +++ b/internal/endtoend/testdata/analyze_subqueries/clickhouse/exec.json @@ -0,0 +1,5 @@ +{ + "command": "analyze", + "args": ["--dialect", "clickhouse", "--schema", "schema.sql", "query.sql"], + "contexts": ["base"] +} diff --git a/internal/endtoend/testdata/analyze_subqueries/clickhouse/fixture.sql b/internal/endtoend/testdata/analyze_subqueries/clickhouse/fixture.sql new file mode 100644 index 0000000000..9adc5fa938 --- /dev/null +++ b/internal/endtoend/testdata/analyze_subqueries/clickhouse/fixture.sql @@ -0,0 +1,2 @@ +INSERT INTO events VALUES (1, 'signup', NULL, 9.5), (2, 'login', 'web', 0.5); +INSERT INTO users VALUES (1, 'a@example.com'); diff --git a/internal/endtoend/testdata/analyze_subqueries/clickhouse/output.json b/internal/endtoend/testdata/analyze_subqueries/clickhouse/output.json new file mode 100644 index 0000000000..bc4f0ec8e9 --- /dev/null +++ b/internal/endtoend/testdata/analyze_subqueries/clickhouse/output.json @@ -0,0 +1,159 @@ +[ + { + "name": "Aliased", + "cmd": ":many", + "columns": [ + { + "name": "x", + "type": { + "name": "uint64" + }, + "table": "events" + }, + { + "name": "total", + "type": { + "name": "float64" + } + }, + { + "name": "email", + "type": { + "name": "string" + } + } + ], + "params": [] + }, + { + "name": "Union", + "cmd": ":many", + "columns": [ + { + "name": "id", + "type": { + "name": "uint64" + }, + "table": "events" + }, + { + "name": "name", + "type": { + "name": "string" + }, + "table": "events" + } + ], + "params": [] + }, + { + "name": "ScalarSubquery", + "cmd": ":many", + "columns": [ + { + "name": "id", + "type": { + "name": "uint64" + }, + "table": "events" + }, + { + "name": "user_count", + "type": { + "name": "uint64", + "nullable": true + } + } + ], + "params": [ + { + "number": 1, + "column": { + "name": "email", + "type": { + "name": "string" + }, + "table": "users" + } + } + ] + }, + { + "name": "Cte", + "cmd": ":many", + "columns": [ + { + "name": "id", + "type": { + "name": "uint64" + }, + "table": "events" + }, + { + "name": "tag", + "type": { + "name": "string", + "nullable": true + }, + "table": "events" + }, + { + "name": "cnt", + "type": { + "name": "uint64" + } + } + ], + "params": [] + }, + { + "name": "Star", + "cmd": ":many", + "columns": [ + { + "name": "id", + "type": { + "name": "uint64" + }, + "table": "users" + }, + { + "name": "email", + "type": { + "name": "string" + }, + "table": "users" + }, + { + "name": "e.id", + "type": { + "name": "uint64" + }, + "table": "events" + }, + { + "name": "name", + "type": { + "name": "string" + }, + "table": "events" + }, + { + "name": "tag", + "type": { + "name": "string", + "nullable": true + }, + "table": "events" + }, + { + "name": "amount", + "type": { + "name": "float64" + }, + "table": "events" + } + ], + "params": [] + } +] diff --git a/internal/endtoend/testdata/analyze_subqueries/clickhouse/query.sql b/internal/endtoend/testdata/analyze_subqueries/clickhouse/query.sql new file mode 100644 index 0000000000..601ba8e146 --- /dev/null +++ b/internal/endtoend/testdata/analyze_subqueries/clickhouse/query.sql @@ -0,0 +1,27 @@ +-- name: Aliased :many +SELECT s.x, s.total, s.email +FROM ( + SELECT e.id AS x, sum(e.amount) AS total, any(u.email) AS email + FROM events e JOIN users u ON u.id = e.id + GROUP BY e.id +) s; + +-- name: Union :many +SELECT id, name FROM events +UNION ALL +SELECT id, email FROM users; + +-- name: ScalarSubquery :many +SELECT id, (SELECT count() FROM users) AS user_count +FROM events +WHERE id IN (SELECT id FROM users WHERE email = ?); + + +-- name: Cte :many +WITH t AS (SELECT id, tag FROM events) +SELECT t.id, t.tag, s.cnt +FROM t +JOIN (SELECT id, count() AS cnt FROM events GROUP BY id) s ON s.id = t.id; + +-- name: Star :many +SELECT * FROM users u JOIN events e ON e.id = u.id; diff --git a/internal/endtoend/testdata/analyze_subqueries/clickhouse/schema.sql b/internal/endtoend/testdata/analyze_subqueries/clickhouse/schema.sql new file mode 100644 index 0000000000..9c04e10685 --- /dev/null +++ b/internal/endtoend/testdata/analyze_subqueries/clickhouse/schema.sql @@ -0,0 +1,11 @@ +CREATE TABLE events ( + id UInt64, + name String, + tag Nullable(String), + amount Float64 +) ENGINE = MergeTree ORDER BY id; + +CREATE TABLE users ( + id UInt64, + email String +) ENGINE = MergeTree ORDER BY id; diff --git a/internal/endtoend/testdata/analyze_system_catalog/postgresql/stdout.txt b/internal/endtoend/testdata/analyze_system_catalog/postgresql/output.json similarity index 56% rename from internal/endtoend/testdata/analyze_system_catalog/postgresql/stdout.txt rename to internal/endtoend/testdata/analyze_system_catalog/postgresql/output.json index ee5f980921..0432abd33e 100644 --- a/internal/endtoend/testdata/analyze_system_catalog/postgresql/stdout.txt +++ b/internal/endtoend/testdata/analyze_system_catalog/postgresql/output.json @@ -5,23 +5,26 @@ "columns": [ { "name": "table_name", - "data_type": "sql_identifier", - "not_null": false, - "is_array": false, + "type": { + "name": "sql_identifier", + "nullable": true + }, "table": "columns" }, { "name": "column_name", - "data_type": "sql_identifier", - "not_null": false, - "is_array": false, + "type": { + "name": "sql_identifier", + "nullable": true + }, "table": "columns" }, { "name": "data_type", - "data_type": "character_data", - "not_null": false, - "is_array": false, + "type": { + "name": "character_data", + "nullable": true + }, "table": "columns" } ], @@ -30,9 +33,10 @@ "number": 1, "column": { "name": "table_schema", - "data_type": "sql_identifier", - "not_null": false, - "is_array": false, + "type": { + "name": "sql_identifier", + "nullable": true + }, "table": "columns" } } @@ -44,9 +48,9 @@ "columns": [ { "name": "total", - "data_type": "bigint", - "not_null": true, - "is_array": false + "type": { + "name": "bigint" + } } ], "params": [ @@ -54,9 +58,9 @@ "number": 1, "column": { "name": "relkind", - "data_type": "char", - "not_null": true, - "is_array": false, + "type": { + "name": "char" + }, "table": "pg_class" } } diff --git a/internal/endtoend/testdata/analyze_types/clickhouse/exec.json b/internal/endtoend/testdata/analyze_types/clickhouse/exec.json new file mode 100644 index 0000000000..c92bfa3dab --- /dev/null +++ b/internal/endtoend/testdata/analyze_types/clickhouse/exec.json @@ -0,0 +1,5 @@ +{ + "command": "analyze", + "args": ["--dialect", "clickhouse", "--schema", "schema.sql", "query.sql"], + "contexts": ["base"] +} diff --git a/internal/endtoend/testdata/analyze_types/clickhouse/fixture.sql b/internal/endtoend/testdata/analyze_types/clickhouse/fixture.sql new file mode 100644 index 0000000000..bb958af8a2 --- /dev/null +++ b/internal/endtoend/testdata/analyze_types/clickhouse/fixture.sql @@ -0,0 +1 @@ +INSERT INTO things VALUES (1, 'a', NULL, 1.5, ['x'], [NULL, 'y'], [[1, 2]], 'k', '2024-01-01 00:00:00', '2024-01-01 00:00:00.123', 1.25, 'active', {'k': 1}, (1.0, 2.0), (51.5, -0.1), {'a': NULL}, '127.0.0.1', '00000000-0000-0000-0000-000000000000', 'abcd', true); diff --git a/internal/endtoend/testdata/analyze_types/clickhouse/output.json b/internal/endtoend/testdata/analyze_types/clickhouse/output.json new file mode 100644 index 0000000000..5ef32d6b93 --- /dev/null +++ b/internal/endtoend/testdata/analyze_types/clickhouse/output.json @@ -0,0 +1,536 @@ +[ + { + "name": "AllTypes", + "cmd": ":many", + "columns": [ + { + "name": "id", + "type": { + "name": "uint64" + }, + "table": "things" + }, + { + "name": "name", + "type": { + "name": "string" + }, + "table": "things" + }, + { + "name": "tag", + "type": { + "name": "string", + "nullable": true + }, + "table": "things" + }, + { + "name": "amount", + "type": { + "name": "float64" + }, + "table": "things" + }, + { + "name": "tags", + "type": { + "name": "array", + "args": [ + { + "type": { + "name": "string" + } + } + ] + }, + "table": "things" + }, + { + "name": "labels", + "type": { + "name": "array", + "args": [ + { + "type": { + "name": "string", + "nullable": true + } + } + ] + }, + "table": "things" + }, + { + "name": "matrix", + "type": { + "name": "array", + "args": [ + { + "type": { + "name": "array", + "args": [ + { + "type": { + "name": "uint8" + } + } + ] + } + } + ] + }, + "table": "things" + }, + { + "name": "kind", + "type": { + "name": "lowcardinality", + "args": [ + { + "type": { + "name": "string", + "nullable": true + } + } + ] + }, + "table": "things" + }, + { + "name": "created", + "type": { + "name": "datetime" + }, + "table": "things" + }, + { + "name": "updated", + "type": { + "name": "datetime64", + "args": [ + { + "int": 3 + }, + { + "string": "UTC" + } + ] + }, + "table": "things" + }, + { + "name": "price", + "type": { + "name": "decimal", + "args": [ + { + "int": 10 + }, + { + "int": 2 + } + ] + }, + "table": "things" + }, + { + "name": "status", + "type": { + "name": "enum8", + "args": [ + { + "label": "active", + "int": 1 + }, + { + "label": "deleted", + "int": 2 + } + ] + }, + "table": "things" + }, + { + "name": "attrs", + "type": { + "name": "map", + "args": [ + { + "type": { + "name": "string" + } + }, + { + "type": { + "name": "uint32" + } + } + ] + }, + "table": "things" + }, + { + "name": "pos", + "type": { + "name": "tuple", + "args": [ + { + "type": { + "name": "float64" + } + }, + { + "type": { + "name": "float64" + } + } + ] + }, + "table": "things" + }, + { + "name": "geo", + "type": { + "name": "tuple", + "args": [ + { + "label": "lat", + "type": { + "name": "float64" + } + }, + { + "label": "lon", + "type": { + "name": "float64" + } + } + ] + }, + "table": "things" + }, + { + "name": "scores", + "type": { + "name": "map", + "args": [ + { + "type": { + "name": "string" + } + }, + { + "type": { + "name": "uint8", + "nullable": true + } + } + ] + }, + "table": "things" + }, + { + "name": "ip", + "type": { + "name": "ipv4" + }, + "table": "things" + }, + { + "name": "uid", + "type": { + "name": "uuid" + }, + "table": "things" + }, + { + "name": "fixed", + "type": { + "name": "fixedstring", + "args": [ + { + "int": 4 + } + ] + }, + "table": "things" + }, + { + "name": "flag", + "type": { + "name": "bool" + }, + "table": "things" + } + ], + "params": [] + }, + { + "name": "StarColumns", + "cmd": ":many", + "columns": [ + { + "name": "id", + "type": { + "name": "uint64" + }, + "table": "things" + }, + { + "name": "name", + "type": { + "name": "string" + }, + "table": "things" + }, + { + "name": "tag", + "type": { + "name": "string", + "nullable": true + }, + "table": "things" + }, + { + "name": "amount", + "type": { + "name": "float64" + }, + "table": "things" + }, + { + "name": "tags", + "type": { + "name": "array", + "args": [ + { + "type": { + "name": "string" + } + } + ] + }, + "table": "things" + }, + { + "name": "labels", + "type": { + "name": "array", + "args": [ + { + "type": { + "name": "string", + "nullable": true + } + } + ] + }, + "table": "things" + }, + { + "name": "matrix", + "type": { + "name": "array", + "args": [ + { + "type": { + "name": "array", + "args": [ + { + "type": { + "name": "uint8" + } + } + ] + } + } + ] + }, + "table": "things" + }, + { + "name": "kind", + "type": { + "name": "lowcardinality", + "args": [ + { + "type": { + "name": "string", + "nullable": true + } + } + ] + }, + "table": "things" + }, + { + "name": "created", + "type": { + "name": "datetime" + }, + "table": "things" + }, + { + "name": "updated", + "type": { + "name": "datetime64", + "args": [ + { + "int": 3 + }, + { + "string": "UTC" + } + ] + }, + "table": "things" + }, + { + "name": "price", + "type": { + "name": "decimal", + "args": [ + { + "int": 10 + }, + { + "int": 2 + } + ] + }, + "table": "things" + }, + { + "name": "status", + "type": { + "name": "enum8", + "args": [ + { + "label": "active", + "int": 1 + }, + { + "label": "deleted", + "int": 2 + } + ] + }, + "table": "things" + }, + { + "name": "attrs", + "type": { + "name": "map", + "args": [ + { + "type": { + "name": "string" + } + }, + { + "type": { + "name": "uint32" + } + } + ] + }, + "table": "things" + }, + { + "name": "pos", + "type": { + "name": "tuple", + "args": [ + { + "type": { + "name": "float64" + } + }, + { + "type": { + "name": "float64" + } + } + ] + }, + "table": "things" + }, + { + "name": "geo", + "type": { + "name": "tuple", + "args": [ + { + "label": "lat", + "type": { + "name": "float64" + } + }, + { + "label": "lon", + "type": { + "name": "float64" + } + } + ] + }, + "table": "things" + }, + { + "name": "scores", + "type": { + "name": "map", + "args": [ + { + "type": { + "name": "string" + } + }, + { + "type": { + "name": "uint8", + "nullable": true + } + } + ] + }, + "table": "things" + }, + { + "name": "ip", + "type": { + "name": "ipv4" + }, + "table": "things" + }, + { + "name": "uid", + "type": { + "name": "uuid" + }, + "table": "things" + }, + { + "name": "fixed", + "type": { + "name": "fixedstring", + "args": [ + { + "int": 4 + } + ] + }, + "table": "things" + }, + { + "name": "flag", + "type": { + "name": "bool" + }, + "table": "things" + } + ], + "params": [] + } +] diff --git a/internal/endtoend/testdata/analyze_types/clickhouse/query.sql b/internal/endtoend/testdata/analyze_types/clickhouse/query.sql new file mode 100644 index 0000000000..c55355e50d --- /dev/null +++ b/internal/endtoend/testdata/analyze_types/clickhouse/query.sql @@ -0,0 +1,5 @@ +-- name: AllTypes :many +SELECT * FROM things; + +-- name: StarColumns :many +SELECT id, name, tag, amount, tags, labels, matrix, kind, created, updated, price, status, attrs, pos, geo, scores, ip, uid, fixed, flag FROM things; diff --git a/internal/endtoend/testdata/analyze_types/clickhouse/schema.sql b/internal/endtoend/testdata/analyze_types/clickhouse/schema.sql new file mode 100644 index 0000000000..5f78064628 --- /dev/null +++ b/internal/endtoend/testdata/analyze_types/clickhouse/schema.sql @@ -0,0 +1,22 @@ +CREATE TABLE things ( + id UInt64, + name String, + tag Nullable(String), + amount Float64, + tags Array(String), + labels Array(Nullable(String)), + matrix Array(Array(UInt8)), + kind LowCardinality(Nullable(String)), + created DateTime, + updated DateTime64(3, 'UTC'), + price Decimal(10, 2), + status Enum8('active' = 1, 'deleted' = 2), + attrs Map(String, UInt32), + pos Tuple(Float64, Float64), + geo Tuple(lat Float64, lon Float64), + scores Map(String, Nullable(UInt8)), + ip IPv4, + uid UUID, + fixed FixedString(4), + flag Bool +) ENGINE = MergeTree ORDER BY id; diff --git a/internal/endtoend/testdata/parse_basic/clickhouse/stdout.txt b/internal/endtoend/testdata/parse_basic/clickhouse/stdout.txt index 82c35f9819..0344534966 100644 --- a/internal/endtoend/testdata/parse_basic/clickhouse/stdout.txt +++ b/internal/endtoend/testdata/parse_basic/clickhouse/stdout.txt @@ -11,15 +11,16 @@ "items": [ { "tag": "ResTarget", + "name": "1", "val": { "tag": "A_Const", "val": { "tag": "Integer", "ival": 1 }, - "location": 31 + "location": 30 }, - "location": 31 + "location": 30 } ] }, diff --git a/internal/engine/clickhouse/convert.go b/internal/engine/clickhouse/convert.go index ea4f342e2c..de2a1419bf 100644 --- a/internal/engine/clickhouse/convert.go +++ b/internal/engine/clickhouse/convert.go @@ -6,6 +6,7 @@ import ( "strings" chast "github.com/sqlc-dev/doubleclick/ast" + "github.com/sqlc-dev/doubleclick/token" "github.com/sqlc-dev/sqlc/internal/sql/ast" ) @@ -14,6 +15,18 @@ type cc struct { paramCount int } +// pos is a node's byte offset in the query. doubleclick counts positions +// from one; the rest of sqlc counts from zero. +func pos(n interface{ Pos() token.Position }) int { + if n == nil { + return 0 + } + if off := n.Pos().Offset; off > 0 { + return off - 1 + } + return 0 +} + func (c *cc) convert(node chast.Node) ast.Node { switch n := node.(type) { case *chast.SelectWithUnionQuery: @@ -153,19 +166,24 @@ func (c *cc) convertSelectQuery(n *chast.SelectQuery) *ast.SelectStmt { Ctes: &ast.List{}, } for _, cte := range n.With { - if aliased, ok := cte.(*chast.AliasedExpr); ok { - cteNode := &ast.CommonTableExpr{ - Ctename: &aliased.Alias, - } - // CTE expression may be a Subquery containing the actual SELECT - if subq, ok := aliased.Expr.(*chast.Subquery); ok { - cteNode.Ctequery = c.convert(subq.Query) - } else { - // Fallback: treat the expression itself as the query - cteNode.Ctequery = c.convertExpr(aliased.Expr) - } - stmt.WithClause.Ctes.Items = append(stmt.WithClause.Ctes.Items, cteNode) + var name string + var query chast.Expression + switch w := cte.(type) { + case *chast.WithElement: + // "name AS (SELECT ...)" or a scalar "(expr) AS name". + name, query = w.Name, w.Query + case *chast.AliasedExpr: + name, query = w.Alias, w.Expr + default: + continue + } + cteNode := &ast.CommonTableExpr{Ctename: &name} + if subq, ok := query.(*chast.Subquery); ok { + cteNode.Ctequery = c.convert(subq.Query) + } else { + cteNode.Ctequery = c.convertExpr(query) } + stmt.WithClause.Ctes.Items = append(stmt.WithClause.Ctes.Items, cteNode) } } @@ -174,7 +192,7 @@ func (c *cc) convertSelectQuery(n *chast.SelectQuery) *ast.SelectStmt { func (c *cc) convertToResTarget(expr chast.Expression) *ast.ResTarget { res := &ast.ResTarget{ - Location: expr.Pos().Offset, + Location: pos(expr), } switch e := expr.(type) { @@ -197,24 +215,160 @@ func (c *cc) convertToResTarget(expr chast.Expression) *ast.ResTarget { }, } } + return res case *chast.AliasedExpr: res.Name = &e.Alias res.Val = c.convertExpr(e.Expr) + return res case *chast.Identifier: if e.Alias != "" { res.Name = &e.Alias } res.Val = c.convertIdentifier(e) + return res + } + + res.Val = c.convertExpr(expr) + if alias := exprAlias(expr); alias != "" { + res.Name = &alias + } else if name := columnName(expr); name != "" { + // ClickHouse names an unaliased expression column after the + // expression itself, written in its canonical function form. + res.Name = &name + } + return res +} + +// exprAlias returns the alias the parser attached to an expression, for +// the node kinds that carry one directly. +func exprAlias(expr chast.Expression) string { + switch e := expr.(type) { + case *chast.AliasedExpr: + return e.Alias + case *chast.Identifier: + return e.Alias case *chast.FunctionCall: - if e.Alias != "" { - res.Name = &e.Alias + return e.Alias + case *chast.CaseExpr: + return e.Alias + case *chast.CastExpr: + return e.Alias + case *chast.LikeExpr: + return e.Alias + case *chast.ExtractExpr: + return e.Alias + case *chast.Subquery: + return e.Alias + } + return "" +} + +// binaryFunctions are the function names ClickHouse gives its operators when +// it names a column after an expression. +var binaryFunctions = map[string]string{ + "+": "plus", "-": "minus", "*": "multiply", "/": "divide", "%": "modulo", + "=": "equals", "==": "equals", "!=": "notEquals", "<>": "notEquals", + "<": "less", ">": "greater", "<=": "lessOrEquals", ">=": "greaterOrEquals", + "AND": "and", "OR": "or", "||": "concat", +} + +// columnName writes an expression the way ClickHouse names a result column +// that has no alias: a function call with its arguments, an operator as the +// function it stands for, an identifier or literal as written. It returns "" +// for an expression it cannot write, which then keeps sqlc's own name. +func columnName(expr chast.Expression) string { + switch e := expr.(type) { + case *chast.Identifier: + return e.Name() + case *chast.Literal: + return literalText(e) + case *chast.FunctionCall: + if e.Over != nil || e.Filter != nil || e.Distinct { + return "" } - res.Val = c.convertFunctionCall(e) - default: - res.Val = c.convertExpr(expr) + var params, args []string + for _, p := range e.Parameters { + s := columnName(p) + if s == "" { + return "" + } + params = append(params, s) + } + for _, a := range e.Arguments { + s := columnName(a) + if s == "" { + return "" + } + args = append(args, s) + } + name := e.Name + if len(e.Parameters) > 0 { + name += "(" + strings.Join(params, ", ") + ")" + } + return name + "(" + strings.Join(args, ", ") + ")" + case *chast.BinaryExpr: + fn, ok := binaryFunctions[strings.ToUpper(e.Op)] + if !ok { + return "" + } + left, right := columnName(e.Left), columnName(e.Right) + if left == "" || right == "" { + return "" + } + return fn + "(" + left + ", " + right + ")" + case *chast.UnaryExpr: + operand := columnName(e.Operand) + if operand == "" { + return "" + } + switch strings.ToUpper(e.Op) { + case "-": + return "negate(" + operand + ")" + case "NOT": + return "not(" + operand + ")" + } + return "" + case *chast.IsNullExpr: + arg := columnName(e.Expr) + if arg == "" { + return "" + } + if e.Not { + return "isNotNull(" + arg + ")" + } + return "isNull(" + arg + ")" + case *chast.TernaryExpr: + cond, then, els := columnName(e.Condition), columnName(e.Then), columnName(e.Else) + if cond == "" || then == "" || els == "" { + return "" + } + return "if(" + cond + ", " + then + ", " + els + ")" } + return "" +} - return res +// literalText writes a literal as ClickHouse prints it in a column name. +func literalText(l *chast.Literal) string { + switch l.Type { + case chast.LiteralString: + return quoteString(fmt.Sprint(l.Value)) + case chast.LiteralNull: + return "NULL" + case chast.LiteralBoolean: + return fmt.Sprint(l.Value) + case chast.LiteralInteger, chast.LiteralFloat: + if l.Source != "" { + return l.Source + } + return fmt.Sprint(l.Value) + } + return "" +} + +func quoteString(s string) string { + s = strings.ReplaceAll(s, `\`, `\\`) + s = strings.ReplaceAll(s, "'", `\'`) + return "'" + s + "'" } func (c *cc) convertTablesInSelectQuery(n *chast.TablesInSelectQuery) *ast.List { @@ -341,7 +495,23 @@ func (c *cc) convertExpr(expr chast.Expression) ast.Node { case *chast.BinaryExpr: return c.convertBinaryExpr(e) case *chast.FunctionCall: + switch strings.ToLower(e.Name) { + case "coalesce", "ifnull": + // COALESCE is null only when every argument is, which a seeded + // function signature cannot say. + args := &ast.List{} + for _, arg := range e.Arguments { + args.Items = append(args.Items, c.convertExpr(arg)) + } + return &ast.CoalesceExpr{Args: args, Location: pos(e)} + } return c.convertFunctionCall(e) + case *chast.ExistsExpr: + return &ast.SubLink{ + SubLinkType: ast.EXISTS_SUBLINK, + Subselect: c.convert(e.Query), + Location: pos(e), + } case *chast.AliasedExpr: return c.convertExpr(e.Expr) case *chast.Parameter: @@ -381,7 +551,7 @@ func (c *cc) convertIdentifier(n *chast.Identifier) *ast.ColumnRef { } return &ast.ColumnRef{ Fields: fields, - Location: n.Pos().Offset, + Location: pos(n), } } @@ -391,7 +561,7 @@ func (c *cc) convertLiteral(n *chast.Literal) *ast.A_Const { str := n.Value.(string) return &ast.A_Const{ Val: &ast.String{Str: str}, - Location: n.Pos().Offset, + Location: pos(n), } case chast.LiteralInteger: var ival int64 @@ -407,7 +577,7 @@ func (c *cc) convertLiteral(n *chast.Literal) *ast.A_Const { } return &ast.A_Const{ Val: &ast.Integer{Ival: ival}, - Location: n.Pos().Offset, + Location: pos(n), } case chast.LiteralFloat: var fval float64 @@ -420,7 +590,7 @@ func (c *cc) convertLiteral(n *chast.Literal) *ast.A_Const { str := strconv.FormatFloat(fval, 'f', -1, 64) return &ast.A_Const{ Val: &ast.Float{Str: str}, - Location: n.Pos().Offset, + Location: pos(n), } case chast.LiteralBoolean: // ClickHouse booleans are typically 0/1 @@ -428,21 +598,21 @@ func (c *cc) convertLiteral(n *chast.Literal) *ast.A_Const { if bval { return &ast.A_Const{ Val: &ast.Integer{Ival: 1}, - Location: n.Pos().Offset, + Location: pos(n), } } return &ast.A_Const{ Val: &ast.Integer{Ival: 0}, - Location: n.Pos().Offset, + Location: pos(n), } case chast.LiteralNull: return &ast.A_Const{ Val: &ast.Null{}, - Location: n.Pos().Offset, + Location: pos(n), } default: return &ast.A_Const{ - Location: n.Pos().Offset, + Location: pos(n), } } } @@ -466,7 +636,7 @@ func (c *cc) convertBinaryExpr(n *chast.BinaryExpr) ast.Node { c.convertExpr(n.Right), }, }, - Location: n.Pos().Offset, + Location: pos(n), } } @@ -478,7 +648,7 @@ func (c *cc) convertBinaryExpr(n *chast.BinaryExpr) ast.Node { }, Lexpr: c.convertExpr(n.Left), Rexpr: c.convertExpr(n.Right), - Location: n.Pos().Offset, + Location: pos(n), } } @@ -487,7 +657,7 @@ func (c *cc) convertFunctionCall(n *chast.FunctionCall) *ast.FuncCall { Funcname: &ast.List{ Items: []ast.Node{&ast.String{Str: n.Name}}, }, - Location: n.Pos().Offset, + Location: pos(n), AggDistinct: n.Distinct, } @@ -528,7 +698,7 @@ func (c *cc) convertParameter(n *chast.Parameter) ast.Node { } return &ast.ParamRef{ Number: c.paramCount, - Location: n.Pos().Offset, + Location: pos(n), } } @@ -540,13 +710,13 @@ func (c *cc) convertAsterisk(n *chast.Asterisk) *ast.ColumnRef { fields.Items = append(fields.Items, &ast.A_Star{}) return &ast.ColumnRef{ Fields: fields, - Location: n.Pos().Offset, + Location: pos(n), } } func (c *cc) convertCaseExpr(n *chast.CaseExpr) *ast.CaseExpr { ce := &ast.CaseExpr{ - Location: n.Pos().Offset, + Location: pos(n), } // Convert test expression (CASE expr WHEN ...) @@ -577,7 +747,7 @@ func (c *cc) convertCaseExpr(n *chast.CaseExpr) *ast.CaseExpr { func (c *cc) convertCastExpr(n *chast.CastExpr) *ast.TypeCast { tc := &ast.TypeCast{ Arg: c.convertExpr(n.Expr), - Location: n.Pos().Offset, + Location: pos(n), } if n.Type != nil { @@ -595,7 +765,7 @@ func (c *cc) convertBetweenExpr(n *chast.BetweenExpr) *ast.BetweenExpr { Left: c.convertExpr(n.Low), Right: c.convertExpr(n.High), Not: n.Not, - Location: n.Pos().Offset, + Location: pos(n), } } @@ -603,7 +773,7 @@ func (c *cc) convertInExpr(n *chast.InExpr) *ast.In { in := &ast.In{ Expr: c.convertExpr(n.Expr), Not: n.Not, - Location: n.Pos().Offset, + Location: pos(n), } // Convert the list @@ -625,7 +795,7 @@ func (c *cc) convertInExpr(n *chast.InExpr) *ast.In { func (c *cc) convertIsNullExpr(n *chast.IsNullExpr) *ast.NullTest { nullTest := &ast.NullTest{ Arg: c.convertExpr(n.Expr), - Location: n.Pos().Offset, + Location: pos(n), } if n.Not { nullTest.Nulltesttype = ast.NullTestTypeIsNotNull @@ -655,14 +825,17 @@ func (c *cc) convertLikeExpr(n *chast.LikeExpr) *ast.A_Expr { }, Lexpr: c.convertExpr(n.Expr), Rexpr: c.convertExpr(n.Pattern), - Location: n.Pos().Offset, + Location: pos(n), } } +// convertSubquery converts a subquery used as a value: a scalar subquery. +// EXISTS is its own node. func (c *cc) convertSubquery(n *chast.Subquery) *ast.SubLink { return &ast.SubLink{ - SubLinkType: ast.EXISTS_SUBLINK, + SubLinkType: ast.EXPR_SUBLINK, Subselect: c.convert(n.Query), + Location: pos(n), } } @@ -688,7 +861,7 @@ func (c *cc) convertUnaryExpr(n *chast.UnaryExpr) ast.Node { Args: &ast.List{ Items: []ast.Node{c.convertExpr(n.Operand)}, }, - Location: n.Pos().Offset, + Location: pos(n), } } @@ -698,14 +871,14 @@ func (c *cc) convertUnaryExpr(n *chast.UnaryExpr) ast.Node { Items: []ast.Node{&ast.String{Str: n.Op}}, }, Rexpr: c.convertExpr(n.Operand), - Location: n.Pos().Offset, + Location: pos(n), } } func (c *cc) convertOrderByElement(n *chast.OrderByElement) *ast.SortBy { sortBy := &ast.SortBy{ Node: c.convertExpr(n.Expression), - Location: n.Expression.Pos().Offset, + Location: pos(n.Expression), } if n.Descending { @@ -828,8 +1001,11 @@ func (c *cc) convertColumnDeclaration(n *chast.ColumnDeclaration) *ast.ColumnDef } if n.Type != nil { - base, isArray, nullable := unwrapTypeString(renderDataType(n.Type)) - colDef.TypeName = &ast.TypeName{Name: base} + spelling := renderDataType(n.Type) + base, isArray, nullable := unwrapTypeString(spelling) + // The catalog resolves the base type; the full spelling, with its + // arguments and nesting, is kept for the analysis to report. + colDef.TypeName = &ast.TypeName{Name: base, Spelling: spelling} colDef.IsArray = isArray if nullable { colDef.IsNotNull = false @@ -873,12 +1049,21 @@ func renderTypeParam(e chast.Expression) string { case *chast.DataType: return renderDataType(v) case *chast.Literal: + if v.Type == chast.LiteralString { + return quoteString(fmt.Sprint(v.Value)) + } if v.Source != "" { return v.Source } return fmt.Sprintf("%v", v.Value) case *chast.Identifier: return strings.Join(v.Parts, ".") + case *chast.NameTypePair: + // A named tuple or nested element: `lat Float64`. + return v.Name + " " + renderDataType(v.Type) + case *chast.BinaryExpr: + // An enum member: `'active' = 1`. + return renderTypeParam(v.Left) + " " + v.Op + " " + renderTypeParam(v.Right) default: return "" } diff --git a/internal/engine/clickhouse/dialect/dialect.json b/internal/engine/clickhouse/dialect/dialect.json index 3be4b1c4d2..fa7f2c5c73 100644 --- a/internal/engine/clickhouse/dialect/dialect.json +++ b/internal/engine/clickhouse/dialect/dialect.json @@ -6,9 +6,13 @@ "string": "String", "bool": "Bool" }, - "bool": "Bool", - "comparison": ["=", "<>", "!=", "<", "<=", ">", ">="], - "comparison_categories": "NBSD", - "arithmetic": ["+", "-", "*", "/"], + "bool": "UInt8", + "limit": "UInt64", + "untyped": "Nothing", + "propagate_nullable": true, + "qualify_duplicate_columns": true, + "comparison": ["=", "==", "<>", "!=", "<", "<=", ">", ">="], + "comparison_categories": "NBSDU", + "arithmetic": ["+", "-", "*", "/", "%"], "arithmetic_categories": "N" } diff --git a/internal/engine/clickhouse/dialect/functions.jsonl b/internal/engine/clickhouse/dialect/functions.jsonl index 3d0f936072..20898b8c9d 100644 --- a/internal/engine/clickhouse/dialect/functions.jsonl +++ b/internal/engine/clickhouse/dialect/functions.jsonl @@ -1 +1,581 @@ -{"name": "count", "kind": "a", "returns": "UInt64"} +{"name": "count", "kind": "a", "returns": "UInt64", "never_null": true} +{"name": "count", "kind": "a", "args": [{"type": "any"}], "returns": "UInt64", "never_null": true} +{"name": "countIf", "kind": "a", "args": [{"type": "any"}, {"type": "any"}], "returns": "UInt64", "never_null": true} +{"name": "sum", "kind": "a", "args": [{"type": "UInt8"}], "returns": "UInt64"} +{"name": "sum", "kind": "a", "args": [{"type": "UInt16"}], "returns": "UInt64"} +{"name": "sum", "kind": "a", "args": [{"type": "UInt32"}], "returns": "UInt64"} +{"name": "sum", "kind": "a", "args": [{"type": "Int8"}], "returns": "Int64"} +{"name": "sum", "kind": "a", "args": [{"type": "Int16"}], "returns": "Int64"} +{"name": "sum", "kind": "a", "args": [{"type": "Int32"}], "returns": "Int64"} +{"name": "sum", "kind": "a", "args": [{"type": "Float32"}], "returns": "Float64"} +{"name": "sum", "kind": "a", "args": [{"type": "any"}], "returns": "$1"} +{"name": "sumIf", "kind": "a", "args": [{"type": "any"}, {"type": "any"}], "returns": "$1"} +{"name": "avg", "kind": "a", "args": [{"type": "any"}], "returns": "Float64"} +{"name": "avgIf", "kind": "a", "args": [{"type": "any"}, {"type": "any"}], "returns": "Float64"} +{"name": "min", "kind": "a", "args": [{"type": "any"}], "returns": "$1"} +{"name": "max", "kind": "a", "args": [{"type": "any"}], "returns": "$1"} +{"name": "any", "kind": "a", "args": [{"type": "any"}], "returns": "$1"} +{"name": "anyLast", "kind": "a", "args": [{"type": "any"}], "returns": "$1"} +{"name": "minIf", "kind": "a", "args": [{"type": "any"}, {"type": "any"}], "returns": "$1"} +{"name": "maxIf", "kind": "a", "args": [{"type": "any"}, {"type": "any"}], "returns": "$1"} +{"name": "anyIf", "kind": "a", "args": [{"type": "any"}, {"type": "any"}], "returns": "$1"} +{"name": "uniq", "kind": "a", "args": [{"type": "any"}], "returns": "UInt64", "never_null": true} +{"name": "uniqExact", "kind": "a", "args": [{"type": "any"}], "returns": "UInt64", "never_null": true} +{"name": "uniqCombined", "kind": "a", "args": [{"type": "any"}], "returns": "UInt64", "never_null": true} +{"name": "uniqHLL12", "kind": "a", "args": [{"type": "any"}], "returns": "UInt64", "never_null": true} +{"name": "uniqIf", "kind": "a", "args": [{"type": "any"}, {"type": "any"}], "returns": "UInt64", "never_null": true} +{"name": "argMin", "kind": "a", "args": [{"type": "any"}, {"type": "any"}], "returns": "$1"} +{"name": "argMax", "kind": "a", "args": [{"type": "any"}, {"type": "any"}], "returns": "$1"} +{"name": "stddevPop", "kind": "a", "args": [{"type": "any"}], "returns": "Float64"} +{"name": "stddevSamp", "kind": "a", "args": [{"type": "any"}], "returns": "Float64"} +{"name": "varPop", "kind": "a", "args": [{"type": "any"}], "returns": "Float64"} +{"name": "varSamp", "kind": "a", "args": [{"type": "any"}], "returns": "Float64"} +{"name": "median", "kind": "a", "args": [{"type": "any"}], "returns": "Float64"} +{"name": "quantile", "kind": "a", "args": [{"type": "any"}], "returns": "Float64"} +{"name": "quantile", "kind": "a", "args": [{"type": "any"}, {"type": "any"}], "returns": "Float64"} +{"name": "row_number", "kind": "w", "returns": "UInt64", "never_null": true} +{"name": "rank", "kind": "w", "returns": "UInt64", "never_null": true} +{"name": "dense_rank", "kind": "w", "returns": "UInt64", "never_null": true} +{"name": "plus", "args": [{"type": "any"}, {"type": "any"}], "returns": "$1"} +{"name": "minus", "args": [{"type": "any"}, {"type": "any"}], "returns": "$1"} +{"name": "multiply", "args": [{"type": "any"}, {"type": "any"}], "returns": "$1"} +{"name": "intDiv", "args": [{"type": "any"}, {"type": "any"}], "returns": "$1"} +{"name": "intDivOrZero", "args": [{"type": "any"}, {"type": "any"}], "returns": "$1"} +{"name": "modulo", "args": [{"type": "any"}, {"type": "any"}], "returns": "$1"} +{"name": "moduloOrZero", "args": [{"type": "any"}, {"type": "any"}], "returns": "$1"} +{"name": "gcd", "args": [{"type": "any"}, {"type": "any"}], "returns": "$1"} +{"name": "lcm", "args": [{"type": "any"}, {"type": "any"}], "returns": "$1"} +{"name": "bitAnd", "args": [{"type": "any"}, {"type": "any"}], "returns": "$1"} +{"name": "bitOr", "args": [{"type": "any"}, {"type": "any"}], "returns": "$1"} +{"name": "bitXor", "args": [{"type": "any"}, {"type": "any"}], "returns": "$1"} +{"name": "bitShiftLeft", "args": [{"type": "any"}, {"type": "any"}], "returns": "$1"} +{"name": "bitShiftRight", "args": [{"type": "any"}, {"type": "any"}], "returns": "$1"} +{"name": "greatest", "args": [{"type": "any"}, {"type": "any"}], "returns": "$1"} +{"name": "least", "args": [{"type": "any"}, {"type": "any"}], "returns": "$1"} +{"name": "divide", "args": [{"type": "any"}, {"type": "any"}], "returns": "Float64"} +{"name": "negate", "args": [{"type": "any"}], "returns": "$1"} +{"name": "abs", "args": [{"type": "any"}], "returns": "$1"} +{"name": "round", "args": [{"type": "any"}], "returns": "$1"} +{"name": "floor", "args": [{"type": "any"}], "returns": "$1"} +{"name": "ceil", "args": [{"type": "any"}], "returns": "$1"} +{"name": "ceiling", "args": [{"type": "any"}], "returns": "$1"} +{"name": "trunc", "args": [{"type": "any"}], "returns": "$1"} +{"name": "truncate", "args": [{"type": "any"}], "returns": "$1"} +{"name": "identity", "args": [{"type": "any"}], "returns": "$1"} +{"name": "bitNot", "args": [{"type": "any"}], "returns": "$1"} +{"name": "round", "args": [{"type": "any"}, {"type": "any"}], "returns": "$1"} +{"name": "floor", "args": [{"type": "any"}, {"type": "any"}], "returns": "$1"} +{"name": "ceil", "args": [{"type": "any"}, {"type": "any"}], "returns": "$1"} +{"name": "trunc", "args": [{"type": "any"}, {"type": "any"}], "returns": "$1"} +{"name": "exp", "args": [{"type": "any"}], "returns": "Float64"} +{"name": "log", "args": [{"type": "any"}], "returns": "Float64"} +{"name": "ln", "args": [{"type": "any"}], "returns": "Float64"} +{"name": "exp2", "args": [{"type": "any"}], "returns": "Float64"} +{"name": "log2", "args": [{"type": "any"}], "returns": "Float64"} +{"name": "exp10", "args": [{"type": "any"}], "returns": "Float64"} +{"name": "log10", "args": [{"type": "any"}], "returns": "Float64"} +{"name": "sqrt", "args": [{"type": "any"}], "returns": "Float64"} +{"name": "cbrt", "args": [{"type": "any"}], "returns": "Float64"} +{"name": "sin", "args": [{"type": "any"}], "returns": "Float64"} +{"name": "cos", "args": [{"type": "any"}], "returns": "Float64"} +{"name": "tan", "args": [{"type": "any"}], "returns": "Float64"} +{"name": "asin", "args": [{"type": "any"}], "returns": "Float64"} +{"name": "acos", "args": [{"type": "any"}], "returns": "Float64"} +{"name": "atan", "args": [{"type": "any"}], "returns": "Float64"} +{"name": "sigmoid", "args": [{"type": "any"}], "returns": "Float64"} +{"name": "erf", "args": [{"type": "any"}], "returns": "Float64"} +{"name": "lgamma", "args": [{"type": "any"}], "returns": "Float64"} +{"name": "tgamma", "args": [{"type": "any"}], "returns": "Float64"} +{"name": "pow", "args": [{"type": "any"}, {"type": "any"}], "returns": "Float64"} +{"name": "power", "args": [{"type": "any"}, {"type": "any"}], "returns": "Float64"} +{"name": "atan2", "args": [{"type": "any"}, {"type": "any"}], "returns": "Float64"} +{"name": "hypot", "args": [{"type": "any"}, {"type": "any"}], "returns": "Float64"} +{"name": "e", "returns": "Float64", "never_null": true} +{"name": "pi", "returns": "Float64", "never_null": true} +{"name": "sign", "args": [{"type": "any"}], "returns": "Int8"} +{"name": "isFinite", "args": [{"type": "any"}], "returns": "UInt8"} +{"name": "isInfinite", "args": [{"type": "any"}], "returns": "UInt8"} +{"name": "isNaN", "args": [{"type": "any"}], "returns": "UInt8"} +{"name": "rand", "returns": "UInt32", "never_null": true} +{"name": "rand32", "returns": "UInt32", "never_null": true} +{"name": "rand64", "returns": "UInt64", "never_null": true} +{"name": "randCanonical", "returns": "Float64", "never_null": true} +{"name": "equals", "args": [{"type": "any"}, {"type": "any"}], "returns": "UInt8"} +{"name": "notEquals", "args": [{"type": "any"}, {"type": "any"}], "returns": "UInt8"} +{"name": "less", "args": [{"type": "any"}, {"type": "any"}], "returns": "UInt8"} +{"name": "greater", "args": [{"type": "any"}, {"type": "any"}], "returns": "UInt8"} +{"name": "lessOrEquals", "args": [{"type": "any"}, {"type": "any"}], "returns": "UInt8"} +{"name": "greaterOrEquals", "args": [{"type": "any"}, {"type": "any"}], "returns": "UInt8"} +{"name": "and", "args": [{"type": "any"}, {"type": "any"}], "returns": "UInt8"} +{"name": "or", "args": [{"type": "any"}, {"type": "any"}], "returns": "UInt8"} +{"name": "xor", "args": [{"type": "any"}, {"type": "any"}], "returns": "UInt8"} +{"name": "like", "args": [{"type": "any"}, {"type": "any"}], "returns": "UInt8"} +{"name": "notLike", "args": [{"type": "any"}, {"type": "any"}], "returns": "UInt8"} +{"name": "ilike", "args": [{"type": "any"}, {"type": "any"}], "returns": "UInt8"} +{"name": "notILike", "args": [{"type": "any"}, {"type": "any"}], "returns": "UInt8"} +{"name": "match", "args": [{"type": "any"}, {"type": "any"}], "returns": "UInt8"} +{"name": "startsWith", "args": [{"type": "any"}, {"type": "any"}], "returns": "UInt8"} +{"name": "endsWith", "args": [{"type": "any"}, {"type": "any"}], "returns": "UInt8"} +{"name": "has", "args": [{"type": "any"}, {"type": "any"}], "returns": "UInt8"} +{"name": "hasAll", "args": [{"type": "any"}, {"type": "any"}], "returns": "UInt8"} +{"name": "hasAny", "args": [{"type": "any"}, {"type": "any"}], "returns": "UInt8"} +{"name": "mapContains", "args": [{"type": "any"}, {"type": "any"}], "returns": "UInt8"} +{"name": "multiSearchAny", "args": [{"type": "any"}, {"type": "any"}], "returns": "UInt8"} +{"name": "in", "args": [{"type": "any"}, {"type": "any"}], "returns": "UInt8"} +{"name": "notIn", "args": [{"type": "any"}, {"type": "any"}], "returns": "UInt8"} +{"name": "not", "args": [{"type": "any"}], "returns": "UInt8"} +{"name": "isNull", "args": [{"type": "any"}], "returns": "UInt8", "never_null": true} +{"name": "isNotNull", "args": [{"type": "any"}], "returns": "UInt8", "never_null": true} +{"name": "isZeroOrNull", "args": [{"type": "any"}], "returns": "UInt8", "never_null": true} +{"name": "assumeNotNull", "args": [{"type": "any"}], "returns": "$1", "never_null": true} +{"name": "toNullable", "args": [{"type": "any"}], "returns": "$1", "nullable": true} +{"name": "nullIf", "args": [{"type": "any"}, {"type": "any"}], "returns": "$1", "nullable": true} +{"name": "if", "args": [{"type": "any"}, {"type": "any"}, {"type": "any"}], "returns": "$2"} +{"name": "empty", "args": [{"type": "any"}], "returns": "UInt8"} +{"name": "notEmpty", "args": [{"type": "any"}], "returns": "UInt8"} +{"name": "length", "args": [{"type": "any"}], "returns": "UInt64"} +{"name": "lengthUTF8", "args": [{"type": "any"}], "returns": "UInt64"} +{"name": "char_length", "args": [{"type": "any"}], "returns": "UInt64"} +{"name": "character_length", "args": [{"type": "any"}], "returns": "UInt64"} +{"name": "position", "args": [{"type": "any"}, {"type": "any"}], "returns": "UInt64"} +{"name": "positionUTF8", "args": [{"type": "any"}, {"type": "any"}], "returns": "UInt64"} +{"name": "positionCaseInsensitive", "args": [{"type": "any"}, {"type": "any"}], "returns": "UInt64"} +{"name": "countSubstrings", "args": [{"type": "any"}, {"type": "any"}], "returns": "UInt64"} +{"name": "indexOf", "args": [{"type": "any"}, {"type": "any"}], "returns": "UInt64"} +{"name": "countEqual", "args": [{"type": "any"}, {"type": "any"}], "returns": "UInt64"} +{"name": "ngramSearch", "args": [{"type": "any"}, {"type": "any"}], "returns": "UInt64"} +{"name": "lower", "args": [{"type": "any"}], "returns": "String"} +{"name": "upper", "args": [{"type": "any"}], "returns": "String"} +{"name": "lowerUTF8", "args": [{"type": "any"}], "returns": "String"} +{"name": "upperUTF8", "args": [{"type": "any"}], "returns": "String"} +{"name": "toString", "args": [{"type": "any"}], "returns": "String"} +{"name": "toValidUTF8", "args": [{"type": "any"}], "returns": "String"} +{"name": "trim", "args": [{"type": "any"}], "returns": "String"} +{"name": "trimLeft", "args": [{"type": "any"}], "returns": "String"} +{"name": "trimRight", "args": [{"type": "any"}], "returns": "String"} +{"name": "trimBoth", "args": [{"type": "any"}], "returns": "String"} +{"name": "ltrim", "args": [{"type": "any"}], "returns": "String"} +{"name": "rtrim", "args": [{"type": "any"}], "returns": "String"} +{"name": "hex", "args": [{"type": "any"}], "returns": "String"} +{"name": "unhex", "args": [{"type": "any"}], "returns": "String"} +{"name": "bin", "args": [{"type": "any"}], "returns": "String"} +{"name": "base64Encode", "args": [{"type": "any"}], "returns": "String"} +{"name": "base64Decode", "args": [{"type": "any"}], "returns": "String"} +{"name": "tryBase64Decode", "args": [{"type": "any"}], "returns": "String"} +{"name": "reverseUTF8", "args": [{"type": "any"}], "returns": "String"} +{"name": "initcap", "args": [{"type": "any"}], "returns": "String"} +{"name": "normalizeQuery", "args": [{"type": "any"}], "returns": "String"} +{"name": "typeName", "args": [{"type": "any"}], "returns": "String"} +{"name": "dumpColumnStructure", "args": [{"type": "any"}], "returns": "String"} +{"name": "urlDecode", "args": [{"type": "any"}], "returns": "String"} +{"name": "encodeURLComponent", "args": [{"type": "any"}], "returns": "String"} +{"name": "decodeURLComponent", "args": [{"type": "any"}], "returns": "String"} +{"name": "domain", "args": [{"type": "any"}], "returns": "String"} +{"name": "topLevelDomain", "args": [{"type": "any"}], "returns": "String"} +{"name": "path", "args": [{"type": "any"}], "returns": "String"} +{"name": "pathFull", "args": [{"type": "any"}], "returns": "String"} +{"name": "protocol", "args": [{"type": "any"}], "returns": "String"} +{"name": "queryString", "args": [{"type": "any"}], "returns": "String"} +{"name": "fragment", "args": [{"type": "any"}], "returns": "String"} +{"name": "cutQueryString", "args": [{"type": "any"}], "returns": "String"} +{"name": "cutFragment", "args": [{"type": "any"}], "returns": "String"} +{"name": "cutWWW", "args": [{"type": "any"}], "returns": "String"} +{"name": "firstSignificantSubdomain", "args": [{"type": "any"}], "returns": "String"} +{"name": "IPv4NumToString", "args": [{"type": "any"}], "returns": "String"} +{"name": "IPv6NumToString", "args": [{"type": "any"}], "returns": "String"} +{"name": "formatReadableSize", "args": [{"type": "any"}], "returns": "String"} +{"name": "formatReadableQuantity", "args": [{"type": "any"}], "returns": "String"} +{"name": "formatReadableDecimalSize", "args": [{"type": "any"}], "returns": "String"} +{"name": "formatReadableTimeDelta", "args": [{"type": "any"}], "returns": "String"} +{"name": "toDecimalString", "args": [{"type": "any"}], "returns": "String"} +{"name": "monthName", "args": [{"type": "any"}], "returns": "String"} +{"name": "toJSONString", "args": [{"type": "any"}], "returns": "String"} +{"name": "toTypeName", "args": [{"type": "any"}], "returns": "String", "never_null": true} +{"name": "hostName", "returns": "String", "never_null": true} +{"name": "version", "returns": "String", "never_null": true} +{"name": "currentDatabase", "returns": "String", "never_null": true} +{"name": "currentUser", "returns": "String", "never_null": true} +{"name": "FQDN", "returns": "String", "never_null": true} +{"name": "queryID", "returns": "String", "never_null": true} +{"name": "generateULID", "returns": "String", "never_null": true} +{"name": "concat", "args": [{"type": "any"}, {"type": "any"}], "returns": "String"} +{"name": "substring", "args": [{"type": "any"}, {"type": "any"}], "returns": "String"} +{"name": "substr", "args": [{"type": "any"}, {"type": "any"}], "returns": "String"} +{"name": "mid", "args": [{"type": "any"}, {"type": "any"}], "returns": "String"} +{"name": "substringUTF8", "args": [{"type": "any"}, {"type": "any"}], "returns": "String"} +{"name": "repeat", "args": [{"type": "any"}, {"type": "any"}], "returns": "String"} +{"name": "leftPad", "args": [{"type": "any"}, {"type": "any"}], "returns": "String"} +{"name": "rightPad", "args": [{"type": "any"}, {"type": "any"}], "returns": "String"} +{"name": "left", "args": [{"type": "any"}, {"type": "any"}], "returns": "String"} +{"name": "right", "args": [{"type": "any"}, {"type": "any"}], "returns": "String"} +{"name": "extract", "args": [{"type": "any"}, {"type": "any"}], "returns": "String"} +{"name": "formatDateTime", "args": [{"type": "any"}, {"type": "any"}], "returns": "String"} +{"name": "formatDateTimeInJodaSyntax", "args": [{"type": "any"}, {"type": "any"}], "returns": "String"} +{"name": "dateName", "args": [{"type": "any"}, {"type": "any"}], "returns": "String"} +{"name": "JSONExtractString", "args": [{"type": "any"}, {"type": "any"}], "returns": "String"} +{"name": "JSONExtractRaw", "args": [{"type": "any"}, {"type": "any"}], "returns": "String"} +{"name": "visitParamExtractString", "args": [{"type": "any"}, {"type": "any"}], "returns": "String"} +{"name": "simpleJSONExtractString", "args": [{"type": "any"}, {"type": "any"}], "returns": "String"} +{"name": "arrayStringConcat", "args": [{"type": "any"}, {"type": "any"}], "returns": "String"} +{"name": "regexpExtract", "args": [{"type": "any"}, {"type": "any"}], "returns": "String"} +{"name": "translate", "args": [{"type": "any"}, {"type": "any"}], "returns": "String"} +{"name": "format", "args": [{"type": "any"}, {"type": "any"}], "returns": "String"} +{"name": "toDecimalString", "args": [{"type": "any"}, {"type": "any"}], "returns": "String"} +{"name": "concat", "args": [{"type": "any"}, {"type": "any"}, {"type": "any"}], "returns": "String"} +{"name": "substring", "args": [{"type": "any"}, {"type": "any"}, {"type": "any"}], "returns": "String"} +{"name": "substr", "args": [{"type": "any"}, {"type": "any"}, {"type": "any"}], "returns": "String"} +{"name": "mid", "args": [{"type": "any"}, {"type": "any"}, {"type": "any"}], "returns": "String"} +{"name": "substringUTF8", "args": [{"type": "any"}, {"type": "any"}, {"type": "any"}], "returns": "String"} +{"name": "leftPad", "args": [{"type": "any"}, {"type": "any"}, {"type": "any"}], "returns": "String"} +{"name": "rightPad", "args": [{"type": "any"}, {"type": "any"}, {"type": "any"}], "returns": "String"} +{"name": "replaceOne", "args": [{"type": "any"}, {"type": "any"}, {"type": "any"}], "returns": "String"} +{"name": "replaceAll", "args": [{"type": "any"}, {"type": "any"}, {"type": "any"}], "returns": "String"} +{"name": "replaceRegexpOne", "args": [{"type": "any"}, {"type": "any"}, {"type": "any"}], "returns": "String"} +{"name": "replaceRegexpAll", "args": [{"type": "any"}, {"type": "any"}, {"type": "any"}], "returns": "String"} +{"name": "translate", "args": [{"type": "any"}, {"type": "any"}, {"type": "any"}], "returns": "String"} +{"name": "format", "args": [{"type": "any"}, {"type": "any"}, {"type": "any"}], "returns": "String"} +{"name": "concatWithSeparator", "args": [{"type": "any"}, {"type": "any"}, {"type": "any"}], "returns": "String"} +{"name": "regexpExtract", "args": [{"type": "any"}, {"type": "any"}, {"type": "any"}], "returns": "String"} +{"name": "formatDateTime", "args": [{"type": "any"}, {"type": "any"}, {"type": "any"}], "returns": "String"} +{"name": "JSONExtractString", "args": [{"type": "any"}, {"type": "any"}, {"type": "any"}], "returns": "String"} +{"name": "JSONExtractRaw", "args": [{"type": "any"}, {"type": "any"}, {"type": "any"}], "returns": "String"} +{"name": "concat", "args": [{"type": "any"}, {"type": "any"}, {"type": "any"}, {"type": "any"}], "returns": "String"} +{"name": "concat", "args": [{"type": "any"}, {"type": "any"}, {"type": "any"}, {"type": "any"}, {"type": "any"}], "returns": "String"} +{"name": "concat", "args": [{"type": "any"}, {"type": "any"}, {"type": "any"}, {"type": "any"}, {"type": "any"}, {"type": "any"}], "returns": "String"} +{"name": "arrayStringConcat", "args": [{"type": "any"}], "returns": "String"} +{"name": "splitByChar", "args": [{"type": "any"}, {"type": "any"}], "returns": "Array(String)"} +{"name": "splitByString", "args": [{"type": "any"}, {"type": "any"}], "returns": "Array(String)"} +{"name": "splitByRegexp", "args": [{"type": "any"}, {"type": "any"}], "returns": "Array(String)"} +{"name": "splitByWhitespace", "args": [{"type": "any"}, {"type": "any"}], "returns": "Array(String)"} +{"name": "alphaTokens", "args": [{"type": "any"}, {"type": "any"}], "returns": "Array(String)"} +{"name": "extractAll", "args": [{"type": "any"}, {"type": "any"}], "returns": "Array(String)"} +{"name": "tokens", "args": [{"type": "any"}, {"type": "any"}], "returns": "Array(String)"} +{"name": "splitByWhitespace", "args": [{"type": "any"}], "returns": "Array(String)"} +{"name": "alphaTokens", "args": [{"type": "any"}], "returns": "Array(String)"} +{"name": "tokens", "args": [{"type": "any"}], "returns": "Array(String)"} +{"name": "reverse", "args": [{"type": "any"}], "returns": "$1"} +{"name": "md5", "args": [{"type": "any"}], "returns": "FixedString(16)"} +{"name": "MD5", "args": [{"type": "any"}], "returns": "FixedString(16)"} +{"name": "sipHash128", "args": [{"type": "any"}], "returns": "FixedString(16)"} +{"name": "murmurHash3_128", "args": [{"type": "any"}], "returns": "FixedString(16)"} +{"name": "cityHash128", "args": [{"type": "any"}], "returns": "FixedString(16)"} +{"name": "SHA1", "args": [{"type": "any"}], "returns": "FixedString(20)"} +{"name": "SHA224", "args": [{"type": "any"}], "returns": "FixedString(28)"} +{"name": "SHA256", "args": [{"type": "any"}], "returns": "FixedString(32)"} +{"name": "SHA512", "args": [{"type": "any"}], "returns": "FixedString(64)"} +{"name": "sipHash64", "args": [{"type": "any"}], "returns": "UInt64", "never_null": true} +{"name": "cityHash64", "args": [{"type": "any"}], "returns": "UInt64", "never_null": true} +{"name": "xxHash64", "args": [{"type": "any"}], "returns": "UInt64", "never_null": true} +{"name": "halfMD5", "args": [{"type": "any"}], "returns": "UInt64", "never_null": true} +{"name": "farmHash64", "args": [{"type": "any"}], "returns": "UInt64", "never_null": true} +{"name": "javaHash", "args": [{"type": "any"}], "returns": "UInt64", "never_null": true} +{"name": "intHash64", "args": [{"type": "any"}], "returns": "UInt64", "never_null": true} +{"name": "murmurHash2_64", "args": [{"type": "any"}], "returns": "UInt64", "never_null": true} +{"name": "murmurHash3_64", "args": [{"type": "any"}], "returns": "UInt64", "never_null": true} +{"name": "metroHash64", "args": [{"type": "any"}], "returns": "UInt64", "never_null": true} +{"name": "xxh3", "args": [{"type": "any"}], "returns": "UInt64", "never_null": true} +{"name": "wyHash64", "args": [{"type": "any"}], "returns": "UInt64", "never_null": true} +{"name": "xxHash32", "args": [{"type": "any"}], "returns": "UInt32", "never_null": true} +{"name": "intHash32", "args": [{"type": "any"}], "returns": "UInt32", "never_null": true} +{"name": "murmurHash2_32", "args": [{"type": "any"}], "returns": "UInt32", "never_null": true} +{"name": "murmurHash3_32", "args": [{"type": "any"}], "returns": "UInt32", "never_null": true} +{"name": "crc32", "args": [{"type": "any"}], "returns": "UInt32", "never_null": true} +{"name": "CRC32", "args": [{"type": "any"}], "returns": "UInt32", "never_null": true} +{"name": "javaHash", "args": [{"type": "any"}], "returns": "UInt32", "never_null": true} +{"name": "levenshteinDistance", "args": [{"type": "any"}, {"type": "any"}], "returns": "UInt64"} +{"name": "editDistance", "args": [{"type": "any"}, {"type": "any"}], "returns": "UInt64"} +{"name": "ngramDistance", "args": [{"type": "any"}, {"type": "any"}], "returns": "Float32"} +{"name": "ngramSearch", "args": [{"type": "any"}, {"type": "any"}], "returns": "Float32"} +{"name": "jaroSimilarity", "args": [{"type": "any"}, {"type": "any"}], "returns": "Float32"} +{"name": "jaroWinklerSimilarity", "args": [{"type": "any"}, {"type": "any"}], "returns": "Float32"} +{"name": "toInt8", "args": [{"type": "any"}], "returns": "Int8"} +{"name": "toInt8OrZero", "args": [{"type": "any"}], "returns": "Int8"} +{"name": "toInt8OrNull", "args": [{"type": "any"}], "returns": "Int8", "nullable": true} +{"name": "toInt16", "args": [{"type": "any"}], "returns": "Int16"} +{"name": "toInt16OrZero", "args": [{"type": "any"}], "returns": "Int16"} +{"name": "toInt16OrNull", "args": [{"type": "any"}], "returns": "Int16", "nullable": true} +{"name": "toInt32", "args": [{"type": "any"}], "returns": "Int32"} +{"name": "toInt32OrZero", "args": [{"type": "any"}], "returns": "Int32"} +{"name": "toInt32OrNull", "args": [{"type": "any"}], "returns": "Int32", "nullable": true} +{"name": "toInt64", "args": [{"type": "any"}], "returns": "Int64"} +{"name": "toInt64OrZero", "args": [{"type": "any"}], "returns": "Int64"} +{"name": "toInt64OrNull", "args": [{"type": "any"}], "returns": "Int64", "nullable": true} +{"name": "toInt128", "args": [{"type": "any"}], "returns": "Int128"} +{"name": "toInt128OrZero", "args": [{"type": "any"}], "returns": "Int128"} +{"name": "toInt128OrNull", "args": [{"type": "any"}], "returns": "Int128", "nullable": true} +{"name": "toInt256", "args": [{"type": "any"}], "returns": "Int256"} +{"name": "toInt256OrZero", "args": [{"type": "any"}], "returns": "Int256"} +{"name": "toInt256OrNull", "args": [{"type": "any"}], "returns": "Int256", "nullable": true} +{"name": "toUInt8", "args": [{"type": "any"}], "returns": "UInt8"} +{"name": "toUInt8OrZero", "args": [{"type": "any"}], "returns": "UInt8"} +{"name": "toUInt8OrNull", "args": [{"type": "any"}], "returns": "UInt8", "nullable": true} +{"name": "toUInt16", "args": [{"type": "any"}], "returns": "UInt16"} +{"name": "toUInt16OrZero", "args": [{"type": "any"}], "returns": "UInt16"} +{"name": "toUInt16OrNull", "args": [{"type": "any"}], "returns": "UInt16", "nullable": true} +{"name": "toUInt32", "args": [{"type": "any"}], "returns": "UInt32"} +{"name": "toUInt32OrZero", "args": [{"type": "any"}], "returns": "UInt32"} +{"name": "toUInt32OrNull", "args": [{"type": "any"}], "returns": "UInt32", "nullable": true} +{"name": "toUInt64", "args": [{"type": "any"}], "returns": "UInt64"} +{"name": "toUInt64OrZero", "args": [{"type": "any"}], "returns": "UInt64"} +{"name": "toUInt64OrNull", "args": [{"type": "any"}], "returns": "UInt64", "nullable": true} +{"name": "toUInt128", "args": [{"type": "any"}], "returns": "UInt128"} +{"name": "toUInt128OrZero", "args": [{"type": "any"}], "returns": "UInt128"} +{"name": "toUInt128OrNull", "args": [{"type": "any"}], "returns": "UInt128", "nullable": true} +{"name": "toUInt256", "args": [{"type": "any"}], "returns": "UInt256"} +{"name": "toUInt256OrZero", "args": [{"type": "any"}], "returns": "UInt256"} +{"name": "toUInt256OrNull", "args": [{"type": "any"}], "returns": "UInt256", "nullable": true} +{"name": "toFloat32", "args": [{"type": "any"}], "returns": "Float32"} +{"name": "toFloat32OrZero", "args": [{"type": "any"}], "returns": "Float32"} +{"name": "toFloat32OrNull", "args": [{"type": "any"}], "returns": "Float32", "nullable": true} +{"name": "toFloat64", "args": [{"type": "any"}], "returns": "Float64"} +{"name": "toFloat64OrZero", "args": [{"type": "any"}], "returns": "Float64"} +{"name": "toFloat64OrNull", "args": [{"type": "any"}], "returns": "Float64", "nullable": true} +{"name": "toDate", "args": [{"type": "any"}], "returns": "Date"} +{"name": "toDateOrZero", "args": [{"type": "any"}], "returns": "Date"} +{"name": "toDateOrNull", "args": [{"type": "any"}], "returns": "Date", "nullable": true} +{"name": "toDate32", "args": [{"type": "any"}], "returns": "Date32"} +{"name": "toDate32OrZero", "args": [{"type": "any"}], "returns": "Date32"} +{"name": "toDate32OrNull", "args": [{"type": "any"}], "returns": "Date32", "nullable": true} +{"name": "toDateTime", "args": [{"type": "any"}], "returns": "DateTime"} +{"name": "toDateTimeOrZero", "args": [{"type": "any"}], "returns": "DateTime"} +{"name": "toDateTimeOrNull", "args": [{"type": "any"}], "returns": "DateTime", "nullable": true} +{"name": "toUUID", "args": [{"type": "any"}], "returns": "UUID"} +{"name": "toUUIDOrZero", "args": [{"type": "any"}], "returns": "UUID"} +{"name": "toUUIDOrNull", "args": [{"type": "any"}], "returns": "UUID", "nullable": true} +{"name": "toIPv4", "args": [{"type": "any"}], "returns": "IPv4"} +{"name": "toIPv4OrZero", "args": [{"type": "any"}], "returns": "IPv4"} +{"name": "toIPv4OrNull", "args": [{"type": "any"}], "returns": "IPv4", "nullable": true} +{"name": "toIPv6", "args": [{"type": "any"}], "returns": "IPv6"} +{"name": "toIPv6OrZero", "args": [{"type": "any"}], "returns": "IPv6"} +{"name": "toIPv6OrNull", "args": [{"type": "any"}], "returns": "IPv6", "nullable": true} +{"name": "toBool", "args": [{"type": "any"}], "returns": "Bool"} +{"name": "toString", "args": [{"type": "any"}], "returns": "String"} +{"name": "toDateTime", "args": [{"type": "any"}, {"type": "any"}], "returns": "DateTime"} +{"name": "toDate", "args": [{"type": "any"}, {"type": "any"}], "returns": "Date"} +{"name": "toDateTime64", "args": [{"type": "any"}, {"type": "any"}], "returns": "DateTime64"} +{"name": "toDateTime64", "args": [{"type": "any"}, {"type": "any"}, {"type": "any"}], "returns": "DateTime64"} +{"name": "toDecimal32", "args": [{"type": "any"}, {"type": "any"}], "returns": "Decimal32"} +{"name": "toDecimal64", "args": [{"type": "any"}, {"type": "any"}], "returns": "Decimal64"} +{"name": "toDecimal128", "args": [{"type": "any"}, {"type": "any"}], "returns": "Decimal128"} +{"name": "toDecimal256", "args": [{"type": "any"}, {"type": "any"}], "returns": "Decimal256"} +{"name": "toFixedString", "args": [{"type": "any"}, {"type": "any"}], "returns": "FixedString"} +{"name": "toStringCutToZero", "args": [{"type": "any"}], "returns": "String"} +{"name": "reinterpretAsString", "args": [{"type": "any"}], "returns": "String"} +{"name": "reinterpretAsUInt64", "args": [{"type": "any"}], "returns": "UInt64"} +{"name": "toUnixTimestamp", "args": [{"type": "any"}], "returns": "UInt32"} +{"name": "toUnixTimestamp", "args": [{"type": "any"}, {"type": "any"}], "returns": "UInt32"} +{"name": "fromUnixTimestamp", "args": [{"type": "any"}], "returns": "DateTime"} +{"name": "toUUIDOrNull", "args": [{"type": "any"}], "returns": "UUID", "nullable": true} +{"name": "toUUIDOrZero", "args": [{"type": "any"}], "returns": "UUID"} +{"name": "generateUUIDv4", "returns": "UUID", "never_null": true} +{"name": "generateUUIDv7", "returns": "UUID", "never_null": true} +{"name": "toIntervalSecond", "args": [{"type": "any"}], "returns": "IntervalSecond"} +{"name": "toIntervalMinute", "args": [{"type": "any"}], "returns": "IntervalMinute"} +{"name": "toIntervalHour", "args": [{"type": "any"}], "returns": "IntervalHour"} +{"name": "toIntervalDay", "args": [{"type": "any"}], "returns": "IntervalDay"} +{"name": "toIntervalWeek", "args": [{"type": "any"}], "returns": "IntervalWeek"} +{"name": "toIntervalMonth", "args": [{"type": "any"}], "returns": "IntervalMonth"} +{"name": "toIntervalYear", "args": [{"type": "any"}], "returns": "IntervalYear"} +{"name": "now", "returns": "DateTime", "never_null": true} +{"name": "now", "args": [{"type": "any"}], "returns": "DateTime", "never_null": true} +{"name": "now64", "returns": "DateTime64", "never_null": true} +{"name": "now64", "args": [{"type": "any"}], "returns": "DateTime64", "never_null": true} +{"name": "today", "returns": "Date", "never_null": true} +{"name": "yesterday", "returns": "Date", "never_null": true} +{"name": "timeZone", "returns": "String", "never_null": true} +{"name": "serverTimezone", "returns": "String", "never_null": true} +{"name": "toYear", "args": [{"type": "any"}], "returns": "UInt16"} +{"name": "toISOYear", "args": [{"type": "any"}], "returns": "UInt16"} +{"name": "toDayOfYear", "args": [{"type": "any"}], "returns": "UInt16"} +{"name": "toRelativeYearNum", "args": [{"type": "any"}], "returns": "UInt16"} +{"name": "toWeek", "args": [{"type": "any"}], "returns": "UInt8"} +{"name": "toISOWeek", "args": [{"type": "any"}], "returns": "UInt8"} +{"name": "toYearWeek", "args": [{"type": "any"}], "returns": "UInt32"} +{"name": "toQuarter", "args": [{"type": "any"}], "returns": "UInt8"} +{"name": "toMonth", "args": [{"type": "any"}], "returns": "UInt8"} +{"name": "toDayOfMonth", "args": [{"type": "any"}], "returns": "UInt8"} +{"name": "toDayOfWeek", "args": [{"type": "any"}], "returns": "UInt8"} +{"name": "toHour", "args": [{"type": "any"}], "returns": "UInt8"} +{"name": "toMinute", "args": [{"type": "any"}], "returns": "UInt8"} +{"name": "toSecond", "args": [{"type": "any"}], "returns": "UInt8"} +{"name": "toMillisecond", "args": [{"type": "any"}], "returns": "UInt8"} +{"name": "toYYYYMM", "args": [{"type": "any"}], "returns": "UInt32"} +{"name": "toYYYYMMDD", "args": [{"type": "any"}], "returns": "UInt32"} +{"name": "toYYYYMMDDhhmmss", "args": [{"type": "any"}], "returns": "UInt64"} +{"name": "toStartOfDay", "args": [{"type": "any"}], "returns": "DateTime"} +{"name": "toStartOfHour", "args": [{"type": "any"}], "returns": "DateTime"} +{"name": "toStartOfMinute", "args": [{"type": "any"}], "returns": "DateTime"} +{"name": "toStartOfFiveMinutes", "args": [{"type": "any"}], "returns": "DateTime"} +{"name": "toStartOfFifteenMinutes", "args": [{"type": "any"}], "returns": "DateTime"} +{"name": "toStartOfTenMinutes", "args": [{"type": "any"}], "returns": "DateTime"} +{"name": "toTime", "args": [{"type": "any"}], "returns": "DateTime"} +{"name": "timeSlot", "args": [{"type": "any"}], "returns": "DateTime"} +{"name": "toLastDayOfMonth", "args": [{"type": "any"}], "returns": "DateTime"} +{"name": "toLastDayOfMonth", "args": [{"type": "any"}], "returns": "Date"} +{"name": "toStartOfMonth", "args": [{"type": "any"}], "returns": "Date"} +{"name": "toStartOfQuarter", "args": [{"type": "any"}], "returns": "Date"} +{"name": "toStartOfYear", "args": [{"type": "any"}], "returns": "Date"} +{"name": "toStartOfWeek", "args": [{"type": "any"}], "returns": "Date"} +{"name": "toMonday", "args": [{"type": "any"}], "returns": "Date"} +{"name": "toStartOfISOYear", "args": [{"type": "any"}], "returns": "Date"} +{"name": "toStartOfWeek", "args": [{"type": "any"}, {"type": "any"}], "returns": "Date"} +{"name": "toStartOfInterval", "args": [{"type": "any"}, {"type": "any"}], "returns": "$1"} +{"name": "addYears", "args": [{"type": "any"}, {"type": "any"}], "returns": "$1"} +{"name": "addQuarters", "args": [{"type": "any"}, {"type": "any"}], "returns": "$1"} +{"name": "addMonths", "args": [{"type": "any"}, {"type": "any"}], "returns": "$1"} +{"name": "addWeeks", "args": [{"type": "any"}, {"type": "any"}], "returns": "$1"} +{"name": "addDays", "args": [{"type": "any"}, {"type": "any"}], "returns": "$1"} +{"name": "addHours", "args": [{"type": "any"}, {"type": "any"}], "returns": "$1"} +{"name": "addMinutes", "args": [{"type": "any"}, {"type": "any"}], "returns": "$1"} +{"name": "addSeconds", "args": [{"type": "any"}, {"type": "any"}], "returns": "$1"} +{"name": "addMilliseconds", "args": [{"type": "any"}, {"type": "any"}], "returns": "$1"} +{"name": "subtractYears", "args": [{"type": "any"}, {"type": "any"}], "returns": "$1"} +{"name": "subtractQuarters", "args": [{"type": "any"}, {"type": "any"}], "returns": "$1"} +{"name": "subtractMonths", "args": [{"type": "any"}, {"type": "any"}], "returns": "$1"} +{"name": "subtractWeeks", "args": [{"type": "any"}, {"type": "any"}], "returns": "$1"} +{"name": "subtractDays", "args": [{"type": "any"}, {"type": "any"}], "returns": "$1"} +{"name": "subtractHours", "args": [{"type": "any"}, {"type": "any"}], "returns": "$1"} +{"name": "subtractMinutes", "args": [{"type": "any"}, {"type": "any"}], "returns": "$1"} +{"name": "subtractSeconds", "args": [{"type": "any"}, {"type": "any"}], "returns": "$1"} +{"name": "addInterval", "args": [{"type": "any"}, {"type": "any"}], "returns": "$1"} +{"name": "subtractInterval", "args": [{"type": "any"}, {"type": "any"}], "returns": "$1"} +{"name": "dateDiff", "args": [{"type": "any"}, {"type": "any"}, {"type": "any"}], "returns": "Int64"} +{"name": "date_diff", "args": [{"type": "any"}, {"type": "any"}, {"type": "any"}], "returns": "Int64"} +{"name": "timestampDiff", "args": [{"type": "any"}, {"type": "any"}, {"type": "any"}], "returns": "Int64"} +{"name": "timestamp_diff", "args": [{"type": "any"}, {"type": "any"}, {"type": "any"}], "returns": "Int64"} +{"name": "age", "args": [{"type": "any"}, {"type": "any"}, {"type": "any"}], "returns": "Int64"} +{"name": "dateAdd", "args": [{"type": "any"}, {"type": "any"}, {"type": "any"}], "returns": "$3"} +{"name": "date_add", "args": [{"type": "any"}, {"type": "any"}, {"type": "any"}], "returns": "$3"} +{"name": "dateSub", "args": [{"type": "any"}, {"type": "any"}, {"type": "any"}], "returns": "$3"} +{"name": "date_sub", "args": [{"type": "any"}, {"type": "any"}, {"type": "any"}], "returns": "$3"} +{"name": "timestampAdd", "args": [{"type": "any"}, {"type": "any"}, {"type": "any"}], "returns": "$3"} +{"name": "timestampSub", "args": [{"type": "any"}, {"type": "any"}, {"type": "any"}], "returns": "$3"} +{"name": "dateTrunc", "args": [{"type": "any"}, {"type": "any"}, {"type": "any"}], "returns": "$3"} +{"name": "date_trunc", "args": [{"type": "any"}, {"type": "any"}, {"type": "any"}], "returns": "$3"} +{"name": "dateTrunc", "args": [{"type": "any"}, {"type": "any"}], "returns": "$2"} +{"name": "date_trunc", "args": [{"type": "any"}, {"type": "any"}], "returns": "$2"} +{"name": "toRelativeDayNum", "args": [{"type": "any"}], "returns": "UInt32"} +{"name": "toRelativeHourNum", "args": [{"type": "any"}], "returns": "UInt32"} +{"name": "toRelativeMinuteNum", "args": [{"type": "any"}], "returns": "UInt32"} +{"name": "toRelativeSecondNum", "args": [{"type": "any"}], "returns": "UInt32"} +{"name": "toRelativeMonthNum", "args": [{"type": "any"}], "returns": "UInt32"} +{"name": "toRelativeWeekNum", "args": [{"type": "any"}], "returns": "UInt32"} +{"name": "toTimeZone", "args": [{"type": "any"}, {"type": "any"}], "returns": "$1"} +{"name": "toTimezone", "args": [{"type": "any"}, {"type": "any"}], "returns": "$1"} +{"name": "parseDateTimeBestEffort", "args": [{"type": "any"}], "returns": "DateTime"} +{"name": "parseDateTimeBestEffortOrNull", "args": [{"type": "any"}], "returns": "DateTime", "nullable": true} +{"name": "parseDateTimeBestEffortOrZero", "args": [{"type": "any"}], "returns": "DateTime"} +{"name": "parseDateTime", "args": [{"type": "any"}, {"type": "any"}], "returns": "DateTime"} +{"name": "parseDateTime64BestEffort", "args": [{"type": "any"}], "returns": "DateTime64"} +{"name": "makeDate", "args": [{"type": "any"}, {"type": "any"}, {"type": "any"}], "returns": "Date"} +{"name": "makeDateTime", "args": [{"type": "any"}, {"type": "any"}, {"type": "any"}, {"type": "any"}, {"type": "any"}, {"type": "any"}], "returns": "DateTime"} +{"name": "arrayConcat", "args": [{"type": "any"}], "returns": "$1"} +{"name": "arrayConcat", "args": [{"type": "any"}, {"type": "any"}], "returns": "$1"} +{"name": "arrayConcat", "args": [{"type": "any"}, {"type": "any"}, {"type": "any"}], "returns": "$1"} +{"name": "arrayDistinct", "args": [{"type": "any"}], "returns": "$1"} +{"name": "arrayDistinct", "args": [{"type": "any"}, {"type": "any"}], "returns": "$1"} +{"name": "arrayDistinct", "args": [{"type": "any"}, {"type": "any"}, {"type": "any"}], "returns": "$1"} +{"name": "arraySort", "args": [{"type": "any"}], "returns": "$1"} +{"name": "arraySort", "args": [{"type": "any"}, {"type": "any"}], "returns": "$1"} +{"name": "arraySort", "args": [{"type": "any"}, {"type": "any"}, {"type": "any"}], "returns": "$1"} +{"name": "arrayReverseSort", "args": [{"type": "any"}], "returns": "$1"} +{"name": "arrayReverseSort", "args": [{"type": "any"}, {"type": "any"}], "returns": "$1"} +{"name": "arrayReverseSort", "args": [{"type": "any"}, {"type": "any"}, {"type": "any"}], "returns": "$1"} +{"name": "arrayReverse", "args": [{"type": "any"}], "returns": "$1"} +{"name": "arrayReverse", "args": [{"type": "any"}, {"type": "any"}], "returns": "$1"} +{"name": "arrayReverse", "args": [{"type": "any"}, {"type": "any"}, {"type": "any"}], "returns": "$1"} +{"name": "arraySlice", "args": [{"type": "any"}], "returns": "$1"} +{"name": "arraySlice", "args": [{"type": "any"}, {"type": "any"}], "returns": "$1"} +{"name": "arraySlice", "args": [{"type": "any"}, {"type": "any"}, {"type": "any"}], "returns": "$1"} +{"name": "arrayResize", "args": [{"type": "any"}], "returns": "$1"} +{"name": "arrayResize", "args": [{"type": "any"}, {"type": "any"}], "returns": "$1"} +{"name": "arrayResize", "args": [{"type": "any"}, {"type": "any"}, {"type": "any"}], "returns": "$1"} +{"name": "arrayPushBack", "args": [{"type": "any"}], "returns": "$1"} +{"name": "arrayPushBack", "args": [{"type": "any"}, {"type": "any"}], "returns": "$1"} +{"name": "arrayPushBack", "args": [{"type": "any"}, {"type": "any"}, {"type": "any"}], "returns": "$1"} +{"name": "arrayPushFront", "args": [{"type": "any"}], "returns": "$1"} +{"name": "arrayPushFront", "args": [{"type": "any"}, {"type": "any"}], "returns": "$1"} +{"name": "arrayPushFront", "args": [{"type": "any"}, {"type": "any"}, {"type": "any"}], "returns": "$1"} +{"name": "arrayPopBack", "args": [{"type": "any"}], "returns": "$1"} +{"name": "arrayPopBack", "args": [{"type": "any"}, {"type": "any"}], "returns": "$1"} +{"name": "arrayPopBack", "args": [{"type": "any"}, {"type": "any"}, {"type": "any"}], "returns": "$1"} +{"name": "arrayPopFront", "args": [{"type": "any"}], "returns": "$1"} +{"name": "arrayPopFront", "args": [{"type": "any"}, {"type": "any"}], "returns": "$1"} +{"name": "arrayPopFront", "args": [{"type": "any"}, {"type": "any"}, {"type": "any"}], "returns": "$1"} +{"name": "arrayFlatten", "args": [{"type": "any"}], "returns": "$1"} +{"name": "arrayFlatten", "args": [{"type": "any"}, {"type": "any"}], "returns": "$1"} +{"name": "arrayFlatten", "args": [{"type": "any"}, {"type": "any"}, {"type": "any"}], "returns": "$1"} +{"name": "arrayCompact", "args": [{"type": "any"}], "returns": "$1"} +{"name": "arrayCompact", "args": [{"type": "any"}, {"type": "any"}], "returns": "$1"} +{"name": "arrayCompact", "args": [{"type": "any"}, {"type": "any"}, {"type": "any"}], "returns": "$1"} +{"name": "arrayIntersect", "args": [{"type": "any"}], "returns": "$1"} +{"name": "arrayIntersect", "args": [{"type": "any"}, {"type": "any"}], "returns": "$1"} +{"name": "arrayIntersect", "args": [{"type": "any"}, {"type": "any"}, {"type": "any"}], "returns": "$1"} +{"name": "arrayUnion", "args": [{"type": "any"}], "returns": "$1"} +{"name": "arrayUnion", "args": [{"type": "any"}, {"type": "any"}], "returns": "$1"} +{"name": "arrayUnion", "args": [{"type": "any"}, {"type": "any"}, {"type": "any"}], "returns": "$1"} +{"name": "arrayShuffle", "args": [{"type": "any"}], "returns": "$1"} +{"name": "arrayShuffle", "args": [{"type": "any"}, {"type": "any"}], "returns": "$1"} +{"name": "arrayShuffle", "args": [{"type": "any"}, {"type": "any"}, {"type": "any"}], "returns": "$1"} +{"name": "arrayCount", "args": [{"type": "any"}], "returns": "UInt32"} +{"name": "arrayCount", "args": [{"type": "any"}, {"type": "any"}], "returns": "UInt32"} +{"name": "arrayUniq", "args": [{"type": "any"}], "returns": "UInt32"} +{"name": "arrayExists", "args": [{"type": "any"}, {"type": "any"}], "returns": "UInt8"} +{"name": "arrayAll", "args": [{"type": "any"}, {"type": "any"}], "returns": "UInt8"} +{"name": "hasSubstr", "args": [{"type": "any"}, {"type": "any"}], "returns": "UInt8"} +{"name": "arrayEnumerate", "args": [{"type": "any"}], "returns": "Array(UInt32)"} +{"name": "arrayEnumerateUniq", "args": [{"type": "any"}], "returns": "Array(UInt32)"} +{"name": "range", "args": [{"type": "any"}], "returns": "Array(UInt64)"} +{"name": "range", "args": [{"type": "any"}, {"type": "any"}], "returns": "Array(UInt64)"} +{"name": "range", "args": [{"type": "any"}, {"type": "any"}, {"type": "any"}], "returns": "Array(UInt64)"} +{"name": "mapKeys", "args": [{"type": "any"}], "returns": "$1"} +{"name": "mapValues", "args": [{"type": "any"}], "returns": "$1"} +{"name": "length", "args": [{"type": "any"}], "returns": "UInt64"} +{"name": "JSONExtractInt", "args": [{"type": "any"}, {"type": "any"}], "returns": "Int64"} +{"name": "JSONExtractInt64", "args": [{"type": "any"}, {"type": "any"}], "returns": "Int64"} +{"name": "JSONExtractUInt", "args": [{"type": "any"}, {"type": "any"}], "returns": "UInt64"} +{"name": "JSONExtractFloat", "args": [{"type": "any"}, {"type": "any"}], "returns": "Float64"} +{"name": "JSONExtractBool", "args": [{"type": "any"}, {"type": "any"}], "returns": "UInt8"} +{"name": "JSONHas", "args": [{"type": "any"}, {"type": "any"}], "returns": "UInt8"} +{"name": "JSONLength", "args": [{"type": "any"}, {"type": "any"}], "returns": "UInt64"} +{"name": "JSONType", "args": [{"type": "any"}, {"type": "any"}], "returns": "Enum8"} +{"name": "isValidJSON", "args": [{"type": "any"}], "returns": "UInt8"} +{"name": "JSONArrayLength", "args": [{"type": "any"}], "returns": "UInt64", "nullable": true} +{"name": "simpleJSONExtractInt", "args": [{"type": "any"}, {"type": "any"}], "returns": "Int64"} +{"name": "simpleJSONExtractUInt", "args": [{"type": "any"}, {"type": "any"}], "returns": "UInt64"} +{"name": "simpleJSONExtractFloat", "args": [{"type": "any"}, {"type": "any"}], "returns": "Float64"} +{"name": "simpleJSONExtractBool", "args": [{"type": "any"}, {"type": "any"}], "returns": "UInt8"} +{"name": "simpleJSONHas", "args": [{"type": "any"}, {"type": "any"}], "returns": "UInt8"} +{"name": "visitParamExtractInt", "args": [{"type": "any"}, {"type": "any"}], "returns": "Int64"} +{"name": "visitParamExtractUInt", "args": [{"type": "any"}, {"type": "any"}], "returns": "UInt64"} +{"name": "visitParamExtractFloat", "args": [{"type": "any"}, {"type": "any"}], "returns": "Float64"} +{"name": "visitParamExtractBool", "args": [{"type": "any"}, {"type": "any"}], "returns": "UInt8"} +{"name": "visitParamHas", "args": [{"type": "any"}, {"type": "any"}], "returns": "UInt8"} +{"name": "IPv4StringToNum", "args": [{"type": "any"}], "returns": "UInt32"} +{"name": "IPv4StringToNumOrNull", "args": [{"type": "any"}], "returns": "UInt32", "nullable": true} +{"name": "IPv6StringToNum", "args": [{"type": "any"}], "returns": "FixedString(16)"} +{"name": "isIPv4String", "args": [{"type": "any"}], "returns": "UInt8"} +{"name": "isIPv6String", "args": [{"type": "any"}], "returns": "UInt8"} +{"name": "sleep", "args": [{"type": "any"}], "returns": "UInt8"} +{"name": "blockSize", "returns": "UInt64", "never_null": true} +{"name": "rowNumberInAllBlocks", "returns": "UInt64", "never_null": true} +{"name": "materialize", "args": [{"type": "any"}], "returns": "$1"} +{"name": "ignore", "args": [{"type": "any"}], "returns": "UInt8", "never_null": true} +{"name": "ifNotFinite", "args": [{"type": "any"}, {"type": "any"}], "returns": "$1"} +{"name": "throwIf", "args": [{"type": "any"}], "returns": "UInt8"} +{"name": "transform", "args": [{"type": "any"}, {"type": "any"}, {"type": "any"}, {"type": "any"}], "returns": "$4"} +{"name": "bar", "args": [{"type": "any"}, {"type": "any"}, {"type": "any"}], "returns": "String"} +{"name": "bar", "args": [{"type": "any"}, {"type": "any"}, {"type": "any"}, {"type": "any"}], "returns": "String"} +{"name": "dictGetString", "args": [{"type": "any"}, {"type": "any"}, {"type": "any"}], "returns": "String"} +{"name": "dictGetUInt64", "args": [{"type": "any"}, {"type": "any"}, {"type": "any"}], "returns": "UInt64"} +{"name": "dictGetInt64", "args": [{"type": "any"}, {"type": "any"}, {"type": "any"}], "returns": "Int64"} +{"name": "dictGetFloat64", "args": [{"type": "any"}, {"type": "any"}, {"type": "any"}], "returns": "Float64"} +{"name": "dictGetDate", "args": [{"type": "any"}, {"type": "any"}, {"type": "any"}], "returns": "Date"} +{"name": "dictGetDateTime", "args": [{"type": "any"}, {"type": "any"}, {"type": "any"}], "returns": "DateTime"} +{"name": "dictHas", "args": [{"type": "any"}, {"type": "any"}], "returns": "UInt8"} +{"name": "tuple", "args": [{"type": "any"}], "returns": "Tuple"} +{"name": "tupleElement", "args": [{"type": "any"}, {"type": "any"}], "returns": "any"} +{"name": "map", "args": [{"type": "any"}, {"type": "any"}], "returns": "Map"} +{"name": "multiIf", "args": [{"type": "any"}, {"type": "any"}, {"type": "any"}], "returns": "$2"} +{"name": "multiIf", "args": [{"type": "any"}, {"type": "any"}, {"type": "any"}, {"type": "any"}, {"type": "any"}], "returns": "$2"} +{"name": "multiIf", "args": [{"type": "any"}, {"type": "any"}, {"type": "any"}, {"type": "any"}, {"type": "any"}, {"type": "any"}, {"type": "any"}], "returns": "$2"} diff --git a/internal/goldeneye/README.md b/internal/goldeneye/README.md index 3bd0a35787..16e4a2c0f7 100644 --- a/internal/goldeneye/README.md +++ b/internal/goldeneye/README.md @@ -60,10 +60,29 @@ the hand-written files alone, and the checks do not look at them. - `dialect/` — the record types the files are made of, mirrored from `internal/core/seed`, and the helpers that write a generated set of files into an engine directory or diff it against what is committed. +- `endtoend/` — finds the analyze cases and compares an engine's answer with + a case's committed output. - `postgresql/`, `duckdb/`, `clickhouse/` — one package per engine, each - exposing `Locate`, `Version` and `Generate`, and a test that runs the check. + exposing `Locate`, `Version` and `Generate`, `Analyze` where the engine has + an analysis check, and tests that run the checks. - `cmd/goldeneye/` — the command. -The analysis checks — verifying the `analyze_*` cases under -`internal/endtoend/testdata` against what each database itself reports — are -meant to live here too, alongside the dialect checks. +## Analysis checks + +`check` also verifies the `analyze_*` cases under `internal/endtoend/testdata` +against what the database itself reports. A case is an +`analyze_/` directory whose `exec.json` runs the analyze +command; `endtoend/` finds them. The engine package loads the case's +`schema.sql` and optional `fixture.sql` into the database, runs `query.sql` +there, prints what the database reports in the JSON shape `sqlc analyze` +prints, and compares it with the committed `output.json` byte for byte. A +difference means sqlc's analysis disagrees with the database. A case that +asks for `--ast` is skipped, since only sqlc can print that. + +- **`clickhouse`** runs each case in an ephemeral `clickhouse local` process. + Column types come from the executed query's result header, provenance from + `EXPLAIN QUERY TREE`, and parameters from sentinel constants substituted for + `?`, `sqlc.arg()` and `sqlc.narg()`, since ClickHouse itself never sees a + placeholder; `INSERT ... VALUES` parameters map onto `DESCRIBE TABLE`. + +The other engines have no analysis check yet. diff --git a/internal/goldeneye/clickhouse/analyze.go b/internal/goldeneye/clickhouse/analyze.go new file mode 100644 index 0000000000..69795aacf8 --- /dev/null +++ b/internal/goldeneye/clickhouse/analyze.go @@ -0,0 +1,290 @@ +package clickhouse + +import ( + "context" + "encoding/json" + "fmt" + "regexp" + "strconv" + "strings" +) + +// The output is the JSON `sqlc analyze` prints, so a case's committed +// output.json can be compared with it byte for byte. + +type analyzedQuery struct { + Name string `json:"name"` + Cmd string `json:"cmd"` + Columns []analyzedColumn `json:"columns"` + Params []analyzedParam `json:"params"` +} + +type analyzedColumn struct { + Name string `json:"name"` + Type *typeExpr `json:"type,omitempty"` + Table string `json:"table,omitempty"` +} + +type analyzedParam struct { + Number int `json:"number"` + Column analyzedColumn `json:"column"` +} + +// analyze runs every query against the schema and fixture and records what +// ClickHouse reports about each. +func analyze(ctx context.Context, l local, schema, fixture string, queries []query) ([]analyzedQuery, error) { + out := make([]analyzedQuery, 0, len(queries)) + for _, q := range queries { + aq, err := analyzeQuery(ctx, l, schema, fixture, q) + if err != nil { + return nil, fmt.Errorf("%s: %w", q.Name, err) + } + out = append(out, aq) + } + return out, nil +} + +func analyzeQuery(ctx context.Context, l local, schema, fixture string, q query) (analyzedQuery, error) { + sql, phs := bindPlaceholders(q.SQL) + explain := returnsRows(sql) + + var script strings.Builder + for _, stmt := range []string{schema, fixture} { + if s := strings.TrimRight(strings.TrimSpace(stmt), ";"); s != "" { + script.WriteString(s) + script.WriteString(";\n") + } + } + if explain { + script.WriteString("EXPLAIN QUERY TREE " + sql + ";\n") + } + script.WriteString(sql + ";\n") + + results, err := l.run(ctx, script.String()) + if err != nil { + return analyzedQuery{}, err + } + + aq := analyzedQuery{ + Name: q.Name, + Cmd: q.Cmd, + Columns: []analyzedColumn{}, + Params: []analyzedParam{}, + } + if !explain { + return analyzeExec(ctx, l, script.String(), sql, phs, aq) + } + if len(results) != 2 { + return analyzedQuery{}, fmt.Errorf("expected the query tree and one result set, got %d results", len(results)) + } + + var lines []string + for _, row := range results[0].Data { + var line string + if err := json.Unmarshal(row["explain"], &line); err != nil { + return analyzedQuery{}, fmt.Errorf("reading query tree: %w", err) + } + lines = append(lines, line) + } + tree, err := parseQueryTree(lines) + if err != nil { + return analyzedQuery{}, err + } + + // Names and types come from the block header of the executed query, the + // same header a driver sees. The tree only adds where each came from. + _, nodes := projection(firstNode(tree.root.children)) + for i, col := range results[1].Meta { + ac := column(col.Name, col.Type) + if i < len(nodes) && len(nodes) == len(results[1].Meta) { + ac.Table = tree.sourceTable(nodes[i]) + } + aq.Columns = append(aq.Columns, ac) + } + + sentinels := tree.sentinels() + for i, ph := range phs { + ac := analyzedColumn{} + if sentinel := sentinels[i+1]; sentinel != nil { + ac = tree.paramColumn(sentinel) + } + if ph.Name != "" { + ac.Name = ph.Name + } + aq.Params = append(aq.Params, analyzedParam{Number: ph.Number, Column: ac}) + } + return aq, nil +} + +func column(name, typ string) analyzedColumn { + if typ == "" { + return analyzedColumn{Name: name} + } + return analyzedColumn{Name: name, Type: parseType(typ)} +} + +// returnsRows reports whether a statement produces a result set and so can +// be explained as a query tree. +func returnsRows(sql string) bool { + head := strings.ToLower(strings.TrimSpace(sql)) + if strings.HasPrefix(head, "(") { + return true + } + for _, kw := range []string{"select", "with", "show", "describe", "desc", "exists"} { + if strings.HasPrefix(head, kw) && (len(head) == len(kw) || !isWordByte(head[len(kw)])) { + return true + } + } + return false +} + +// sentinels finds the constants the placeholders were substituted with, +// keyed by placeholder ordinal. +func (t *queryTree) sentinels() map[int]*treeNode { + found := map[int]*treeNode{} + var walk func(n *treeNode) + walk = func(n *treeNode) { + if n.kind == "CONSTANT" { + if k, ok := sentinelOrdinal(n); ok { + found[k] = n + return + } + } + for _, c := range n.children { + walk(c) + } + } + walk(t.root) + return found +} + +// sentinelOrdinal recognises a constant folded directly from `NULL + k` or +// `4294967295 + k`, the shapes sentinelFor produces, and returns k. A +// constant folded from a larger expression that merely contains a sentinel +// does not match; the sentinel is found nested inside it instead. +func sentinelOrdinal(c *treeNode) (int, bool) { + fn := firstNode(c.section("EXPRESSION").childrenOrNil()) + if fn == nil || fn.kind != "FUNCTION" || fn.attrs["function_name"] != "plus" { + return 0, false + } + args := fn.section("ARGUMENTS").list() + if len(args) != 2 || args[0].kind != "CONSTANT" || args[1].kind != "CONSTANT" { + return 0, false + } + base := args[0].attrs["constant_value"] + if base != "NULL" && base != "UInt64_"+limitBase { + return 0, false + } + k, err := strconv.Atoi(strings.TrimPrefix(args[1].attrs["constant_value"], "UInt64_")) + if err != nil { + return 0, false + } + return k, true +} + +// paramColumn describes what a placeholder is compared with or assigned to: +// the other operand of the function it is an argument of, preferring a +// column over an expression, or the projected column it stands for. +func (t *queryTree) paramColumn(sentinel *treeNode) analyzedColumn { + list := sentinel.parent + if list != nil && list.kind == "LIST" && list.parent != nil { + switch owner := list.parent; { + case owner.kind == "" && owner.text == "ARGUMENTS": + // Prefer a column operand, then an expression, then a constant. + rank := map[string]int{"COLUMN": 0, "FUNCTION": 1, "CONSTANT": 2} + var best *treeNode + for _, sib := range list.children { + if sib == sentinel { + continue + } + if r, ok := rank[sib.kind]; ok && (best == nil || r < rank[best.kind]) { + best = sib + } + } + if best != nil { + return t.describe(best) + } + case owner.kind == "" && owner.text == "PROJECTION": + names, nodes := projection(owner.parent) + for i, n := range nodes { + if n == sentinel && i < len(names) { + ac := column(names[i], sentinel.attrs["constant_value_type"]) + return ac + } + } + } + } + return column("", sentinel.attrs["constant_value_type"]) +} + +// describe turns a tree expression into a column description. +func (t *queryTree) describe(n *treeNode) analyzedColumn { + switch n.kind { + case "COLUMN": + ac := column(n.attrs["column_name"], n.attrs["result_type"]) + ac.Table = t.sourceTable(n) + return ac + case "FUNCTION": + return column(n.attrs["function_name"], n.attrs["result_type"]) + case "CONSTANT": + // A constant folded from a function call, such as toDate(now()), + // still names the function, which is what a placeholder compared + // with it is named after. + name := "" + if fn := firstNode(n.section("EXPRESSION").childrenOrNil()); fn != nil && fn.kind == "FUNCTION" { + name = fn.attrs["function_name"] + } + return column(name, n.attrs["constant_value_type"]) + } + return analyzedColumn{} +} + +var insertValuesRe = regexp.MustCompile(`(?is)^insert\s+into\s+(?:table\s+)?([\w.` + "`" + `"]+)\s*(?:\(([^)]*)\))?\s*(?:format\s+)?values\b`) + +// analyzeExec runs a statement that returns no rows. The only parameters it +// can describe are those of an INSERT ... VALUES, which map positionally +// onto the target columns reported by DESCRIBE TABLE. +func analyzeExec(ctx context.Context, l local, script, sql string, phs []placeholder, aq analyzedQuery) (analyzedQuery, error) { + m := insertValuesRe.FindStringSubmatch(sql) + if m != nil { + script += "DESCRIBE TABLE " + m[1] + ";\n" + } + results, err := l.run(ctx, script) + if err != nil { + return analyzedQuery{}, err + } + + var targets []analyzedColumn + if m != nil && len(results) == 1 { + byName := map[string]analyzedColumn{} + var all []analyzedColumn + table := strings.Trim(m[1][strings.LastIndexByte(m[1], '.')+1:], "`\"") + for _, row := range results[0].Data { + var name, typ string + json.Unmarshal(row["name"], &name) + json.Unmarshal(row["type"], &typ) + ac := column(name, typ) + ac.Table = table + byName[name] = ac + all = append(all, ac) + } + if strings.TrimSpace(m[2]) == "" { + targets = all + } else { + for _, name := range strings.Split(m[2], ",") { + targets = append(targets, byName[strings.Trim(strings.TrimSpace(name), "`\"")]) + } + } + } + for i, ph := range phs { + ac := analyzedColumn{} + if len(targets) > 0 { + ac = targets[i%len(targets)] + } + if ph.Name != "" { + ac.Name = ph.Name + } + aq.Params = append(aq.Params, analyzedParam{Number: ph.Number, Column: ac}) + } + return aq, nil +} diff --git a/internal/goldeneye/clickhouse/check.go b/internal/goldeneye/clickhouse/check.go new file mode 100644 index 0000000000..1ad8d9cb93 --- /dev/null +++ b/internal/goldeneye/clickhouse/check.go @@ -0,0 +1,55 @@ +package clickhouse + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "os" + + "github.com/sqlc-dev/sqlc/internal/goldeneye/endtoend" +) + +// Analyze runs a case's queries through the clickhouse binary and returns +// the analysis in the JSON shape sqlc analyze prints. +func Analyze(ctx context.Context, binary string, c endtoend.Case) ([]byte, error) { + schema, err := os.ReadFile(c.Schema) + if err != nil { + return nil, err + } + var fixture []byte + if c.Fixture != "" { + if fixture, err = os.ReadFile(c.Fixture); err != nil { + return nil, err + } + } + src, err := os.ReadFile(c.Query) + if err != nil { + return nil, err + } + queries, err := parseQueries(string(src)) + if err != nil { + return nil, fmt.Errorf("%s: %w", c.Query, err) + } + out, err := analyze(ctx, local{binary: binary}, string(schema), string(fixture), queries) + if err != nil { + return nil, err + } + var buf bytes.Buffer + enc := json.NewEncoder(&buf) + enc.SetIndent("", " ") + if err := enc.Encode(out); err != nil { + return nil, err + } + return buf.Bytes(), nil +} + +// Check compares what ClickHouse reports for a case with the output the +// case committed, returning a diff when they differ. +func Check(ctx context.Context, binary string, c endtoend.Case) (string, error) { + got, err := Analyze(ctx, binary, c) + if err != nil { + return "", err + } + return c.Compare(got) +} diff --git a/internal/goldeneye/clickhouse/clickhouse.go b/internal/goldeneye/clickhouse/clickhouse.go index 5eb10d282c..d65539eec0 100644 --- a/internal/goldeneye/clickhouse/clickhouse.go +++ b/internal/goldeneye/clickhouse/clickhouse.go @@ -9,6 +9,15 @@ // system.functions carries no signatures — so functions.jsonl is written by // hand and is not this package's business. // +// The package also verifies the ClickHouse analyze cases under +// internal/endtoend/testdata against the same binary: each case's schema and +// fixture are loaded into a `clickhouse local` process and its queries run +// there. Result column types come from the executed query's result header, +// provenance from EXPLAIN QUERY TREE, and parameters from sentinel constants +// substituted for the placeholders, since ClickHouse itself never sees a ?. +// The answer is printed in the JSON shape sqlc analyze prints and compared +// with the case's committed output.json byte for byte. +// // The binary is downloaded once per pinned version by Install, or supplied // through the CLICKHOUSE environment variable. package clickhouse diff --git a/internal/goldeneye/clickhouse/clickhouse_test.go b/internal/goldeneye/clickhouse/clickhouse_test.go index 0e074eaf21..f4d58b8346 100644 --- a/internal/goldeneye/clickhouse/clickhouse_test.go +++ b/internal/goldeneye/clickhouse/clickhouse_test.go @@ -5,6 +5,7 @@ import ( "testing" "github.com/sqlc-dev/sqlc/internal/goldeneye/dialect" + "github.com/sqlc-dev/sqlc/internal/goldeneye/endtoend" ) // TestDialect verifies the committed ClickHouse dialect against what the @@ -35,3 +36,31 @@ func TestDialect(t *testing.T) { t.Errorf("%s does not match what %s reports:\n%s", dir, version, report) } } + +// TestAnalyzeCases verifies every ClickHouse analyze case under +// internal/endtoend/testdata against what ClickHouse reports. It skips +// unless the binary is installed. +func TestAnalyzeCases(t *testing.T) { + binary, err := Locate() + if err != nil { + t.Skip(err) + } + cases, err := endtoend.Cases(Engine) + if err != nil { + t.Fatal(err) + } + if len(cases) == 0 { + t.Fatal("no clickhouse analyze cases found") + } + for _, c := range cases { + t.Run(c.Name, func(t *testing.T) { + diff, err := Check(context.Background(), binary, c) + if err != nil { + t.Fatal(err) + } + if diff != "" { + t.Errorf("%s does not match what ClickHouse reports (-committed +clickhouse):\n%s", c.Output, diff) + } + }) + } +} diff --git a/internal/goldeneye/clickhouse/queries.go b/internal/goldeneye/clickhouse/queries.go new file mode 100644 index 0000000000..69e3b1ac65 --- /dev/null +++ b/internal/goldeneye/clickhouse/queries.go @@ -0,0 +1,174 @@ +package clickhouse + +import ( + "fmt" + "regexp" + "strings" + "unicode" +) + +// query is one entry of a sqlc query file. +type query struct { + Name string + Cmd string + SQL string +} + +var headerRe = regexp.MustCompile(`^\s*--\s*name:\s*(\S+)\s+(:\S+)\s*$`) + +// parseQueries splits a sqlc query file on its `-- name: X :cmd` headers. +func parseQueries(src string) ([]query, error) { + var queries []query + var cur *query + var body []string + flush := func() error { + if cur == nil { + return nil + } + sql := strings.TrimSpace(strings.Join(body, "\n")) + sql = strings.TrimRight(sql, "; \t\r\n") + if sql == "" { + return fmt.Errorf("query %s has no body", cur.Name) + } + cur.SQL = sql + queries = append(queries, *cur) + return nil + } + for _, line := range strings.Split(src, "\n") { + if m := headerRe.FindStringSubmatch(line); m != nil { + if err := flush(); err != nil { + return nil, err + } + cur = &query{Name: m[1], Cmd: m[2]} + body = body[:0] + continue + } + if cur != nil { + body = append(body, line) + } + } + if err := flush(); err != nil { + return nil, err + } + if len(queries) == 0 { + return nil, fmt.Errorf("no queries found: expected `-- name: Name :cmd` headers") + } + return queries, nil +} + +// placeholder is one parameter reference in a query, in order of appearance. +type placeholder struct { + Number int + Name string // sqlc.arg / sqlc.narg name, empty for ? +} + +// Placeholders are substituted with constant expressions that carry their +// ordinal, so they can be told apart from each other and from literal NULLs +// once ClickHouse has folded them: the query tree prints the expression a +// folded constant came from. NULL coerces to any type, so a comparison +// against it analyzes with the other operand's type. LIMIT and OFFSET +// reject NULL and only accept unsigned integers, so those get a value no +// query would plausibly contain. +const limitBase = "4294967295" + +func sentinelFor(lastWord string, ordinal int) string { + switch strings.ToLower(lastWord) { + case "limit", "offset": + return fmt.Sprintf("toUInt64(%s + %d)", limitBase, ordinal) + } + return fmt.Sprintf("(NULL + %d)", ordinal) +} + +var namedArgRe = regexp.MustCompile(`^sqlc\.(n?arg)\(\s*'?([A-Za-z_][A-Za-z0-9_]*)'?\s*\)`) + +// bindPlaceholders rewrites sqlc's parameter syntax (?, sqlc.arg(name), +// sqlc.narg(name)) into constants ClickHouse can analyze, skipping string +// literals, quoted identifiers and comments. ClickHouse binds every ? +// positionally, so each placeholder is its own parameter even when a name +// repeats, which is how sqlc numbers them too. +func bindPlaceholders(sql string) (string, []placeholder) { + var ( + out strings.Builder + phs []placeholder + lastWord string + i = 0 + ) + number := func(string) int { return len(phs) + 1 } + for i < len(sql) { + c := sql[i] + switch { + case c == '\'' || c == '"' || c == '`': + end := skipQuoted(sql, i) + out.WriteString(sql[i:end]) + i = end + case strings.HasPrefix(sql[i:], "--"): + end := strings.IndexByte(sql[i:], '\n') + if end < 0 { + end = len(sql) + } else { + end += i + } + out.WriteString(sql[i:end]) + i = end + case strings.HasPrefix(sql[i:], "/*"): + end := strings.Index(sql[i:], "*/") + if end < 0 { + end = len(sql) + } else { + end += i + 2 + } + out.WriteString(sql[i:end]) + i = end + case c == '?': + out.WriteString(sentinelFor(lastWord, len(phs)+1)) + phs = append(phs, placeholder{Number: number("")}) + lastWord = "" + i++ + case c == 's' && namedArgRe.MatchString(sql[i:]): + m := namedArgRe.FindStringSubmatch(sql[i:]) + out.WriteString(sentinelFor(lastWord, len(phs)+1)) + phs = append(phs, placeholder{Number: number(m[2]), Name: m[2]}) + lastWord = "" + i += len(m[0]) + case isWordByte(c): + end := i + for end < len(sql) && isWordByte(sql[end]) { + end++ + } + lastWord = sql[i:end] + out.WriteString(lastWord) + i = end + default: + if !unicode.IsSpace(rune(c)) && c != ',' && c != '(' { + lastWord = "" + } + out.WriteByte(c) + i++ + } + } + return out.String(), phs +} + +func isWordByte(c byte) bool { + return c == '_' || c == '.' || c >= '0' && c <= '9' || c >= 'a' && c <= 'z' || c >= 'A' && c <= 'Z' +} + +// skipQuoted returns the index just past the quoted token starting at i, +// honouring backslash escapes and doubled quotes. +func skipQuoted(s string, i int) int { + q := s[i] + j := i + 1 + for j < len(s) { + switch { + case s[j] == '\\' && j+1 < len(s): + j += 2 + case s[j] == q && j+1 < len(s) && s[j+1] == q: + j += 2 + case s[j] == q: + return j + 1 + default: + j++ + } + } + return len(s) +} diff --git a/internal/goldeneye/clickhouse/tree.go b/internal/goldeneye/clickhouse/tree.go new file mode 100644 index 0000000000..d0a8a38924 --- /dev/null +++ b/internal/goldeneye/clickhouse/tree.go @@ -0,0 +1,236 @@ +package clickhouse + +import ( + "fmt" + "regexp" + "strconv" + "strings" +) + +// treeNode is one line of EXPLAIN QUERY TREE output. Lines of the form +// `KIND id: N, key: value, ...` are nodes with a kind and attributes; every +// other line (section headers such as PROJECTION or ARGUMENTS, and the +// `name Type` lines under PROJECTION COLUMNS) keeps only its text. +type treeNode struct { + kind string + id int + attrs map[string]string + text string + parent *treeNode + children []*treeNode +} + +// queryTree is a parsed EXPLAIN QUERY TREE dump. +type queryTree struct { + root *treeNode + byID map[int]*treeNode +} + +var nodeLineRe = regexp.MustCompile(`^([A-Z_]+) id: (\d+)(?:, (.*))?$`) + +// parseQueryTree builds the tree from the dump's lines, using the two-space +// indentation to recover nesting. +func parseQueryTree(lines []string) (*queryTree, error) { + root := &treeNode{id: -1, text: ""} + t := &queryTree{root: root, byID: map[int]*treeNode{}} + stack := []*treeNode{root} + for _, line := range lines { + if strings.TrimSpace(line) == "" { + continue + } + indent := len(line) - len(strings.TrimLeft(line, " ")) + if indent%2 != 0 { + return nil, fmt.Errorf("query tree: unexpected indentation in %q", line) + } + depth := indent/2 + 1 + if depth > len(stack) { + return nil, fmt.Errorf("query tree: line %q is nested too deeply", line) + } + stack = stack[:depth] + parent := stack[len(stack)-1] + + n := &treeNode{id: -1, text: line[indent:], parent: parent} + if m := nodeLineRe.FindStringSubmatch(n.text); m != nil { + n.kind = m[1] + n.id, _ = strconv.Atoi(m[2]) + n.attrs = parseAttrs(m[3]) + t.byID[n.id] = n + } + parent.children = append(parent.children, n) + stack = append(stack, n) + } + return t, nil +} + +var attrKeyRe = regexp.MustCompile(`^, ([a-z_]+): `) + +// parseAttrs splits `key: value, key: value` where values may themselves +// contain commas inside parentheses or quotes, as types and constants do. +func parseAttrs(s string) map[string]string { + attrs := map[string]string{} + if s == "" { + return attrs + } + // Positions at which a new `, key: ` begins at top level. + var cuts []int + depth := 0 + inQuote := false + for i := 0; i < len(s); i++ { + switch c := s[i]; { + case c == '\'': + inQuote = !inQuote + case inQuote: + case c == '(': + depth++ + case c == ')': + depth-- + case c == ',' && depth == 0 && attrKeyRe.MatchString(s[i:]): + cuts = append(cuts, i) + } + } + cuts = append(cuts, len(s)) + start := 0 + for _, cut := range cuts { + pair := s[start:cut] + if key, value, ok := strings.Cut(pair, ": "); ok { + attrs[key] = value + } + start = cut + 2 + } + return attrs +} + +// section returns the child section header of a node, such as PROJECTION +// or JOIN TREE, or nil. +func (n *treeNode) section(name string) *treeNode { + for _, c := range n.children { + if c.kind == "" && c.text == name { + return c + } + } + return nil +} + +// list returns the nodes of the LIST under a section header. +func (n *treeNode) list() []*treeNode { + if n == nil { + return nil + } + for _, c := range n.children { + if c.kind == "LIST" { + return c.children + } + } + return nil +} + +// firstQuery returns the QUERY node describing a result set: the node itself +// or, for a UNION, its first branch, whose projection names the union's +// columns. +func firstQuery(n *treeNode) *treeNode { + for depth := 0; n != nil && depth < 32; depth++ { + switch n.kind { + case "QUERY": + return n + case "UNION": + n = firstNode(n.section("QUERIES").list()) + default: + return nil + } + } + return nil +} + +func firstNode(nodes []*treeNode) *treeNode { + if len(nodes) == 0 { + return nil + } + return nodes[0] +} + +// projection returns the output column names of a query and the expression +// node producing each, in order. +func projection(q *treeNode) (names []string, nodes []*treeNode) { + q = firstQuery(q) + if q == nil { + return nil, nil + } + if cols := q.section("PROJECTION COLUMNS"); cols != nil { + for _, c := range cols.children { + names = append(names, projectedName(c.text)) + } + } + return names, q.section("PROJECTION").list() +} + +// projectedName strips the trailing type from a `name Type` line. The type +// is a single token that may carry a parenthesised argument list; the name +// may contain spaces of its own, as `plus(id, 1)` does. +func projectedName(line string) string { + end := len(line) + if strings.HasSuffix(line, ")") { + depth := 0 + for end > 0 { + end-- + if line[end] == ')' { + depth++ + } else if line[end] == '(' { + depth-- + if depth == 0 { + break + } + } + } + } + for end > 0 && isWordByte(line[end-1]) { + end-- + } + return strings.TrimSpace(line[:end]) +} + +// sourceTable resolves a COLUMN node to the table it reads from, following +// column references through subqueries and CTEs. Columns computed by an +// expression have no source and yield "". +func (t *queryTree) sourceTable(col *treeNode) string { + for depth := 0; col != nil && col.kind == "COLUMN" && depth < 32; depth++ { + id, err := strconv.Atoi(col.attrs["source_id"]) + if err != nil { + return "" + } + src := t.byID[id] + if src == nil { + return "" + } + switch src.kind { + case "TABLE": + name := src.attrs["table_name"] + if i := strings.LastIndexByte(name, '.'); i >= 0 { + name = name[i+1:] + } + return name + case "QUERY", "UNION": + // The column is one of the subquery's output columns; keep + // following whatever expression produces it there. + want := col.attrs["column_name"] + names, nodes := projection(src) + col = nil + for i, name := range names { + if name == want && i < len(nodes) { + col = nodes[i] + break + } + } + default: + return "" + } + } + return "" +} + +// childrenOrNil returns a node's children, tolerating a nil node. +func (n *treeNode) childrenOrNil() []*treeNode { + if n == nil { + return nil + } + return n.children +} diff --git a/internal/goldeneye/clickhouse/types.go b/internal/goldeneye/clickhouse/types.go new file mode 100644 index 0000000000..77ed8d8c19 --- /dev/null +++ b/internal/goldeneye/clickhouse/types.go @@ -0,0 +1,158 @@ +package clickhouse + +import ( + "strconv" + "strings" +) + +// A type is a call expression, the way ClickHouse itself models one: a +// lowercased name applied to an ordered argument list. Each argument is +// another type, an integer, a boolean or a quoted string, optionally +// labelled, so Array, Map and LowCardinality are ordinary names and nothing +// about a nested type is lost. Nullability is an attribute of a type rather +// than a wrapper, since every engine has it and only ClickHouse spells it as +// a type: Nullable(T) becomes T with nullable set, at whatever depth it +// appears. The shape maps one to one onto a protobuf message with a oneof +// for the argument value: +// +// Map(String, Nullable(UInt32)) +// {"name": "map", "args": [ +// {"type": {"name": "string"}}, +// {"type": {"name": "uint32", "nullable": true}}]} +// +// Tuple(lat Float64, lon Float64) +// {"name": "tuple", "args": [ +// {"label": "lat", "type": {"name": "float64"}}, +// {"label": "lon", "type": {"name": "float64"}}]} +// +// Enum8('a' = 1, 'b' = 2) +// {"name": "enum8", "args": [{"label": "a", "int": 1}, {"label": "b", "int": 2}]} +// +// DateTime64(3, 'UTC') +// {"name": "datetime64", "args": [{"int": 3}, {"string": "UTC"}]} +// +// An identifier argument such as the function in AggregateFunction(uniq, +// String) is a type with no arguments. Resolving names against the catalog +// is the reader's job; the output only records what was said. + +type typeExpr struct { + Name string `json:"name"` + Nullable bool `json:"nullable,omitempty"` + Args []typeArg `json:"args,omitempty"` +} + +type typeArg struct { + Label string `json:"label,omitempty"` + Type *typeExpr `json:"type,omitempty"` + Int *int64 `json:"int,omitempty"` + Bool *bool `json:"bool,omitempty"` + String *string `json:"string,omitempty"` +} + +// parseType turns a ClickHouse type string into its expression. +func parseType(t string) *typeExpr { + name, args := splitType(t) + name = strings.ToLower(strings.TrimSpace(name)) + if name == "nullable" && len(args) == 1 { + expr := parseType(args[0]) + expr.Nullable = true + return expr + } + if name == "" { + name = "nothing" + } + expr := &typeExpr{Name: name} + for _, a := range args { + expr.Args = append(expr.Args, parseArg(a)) + } + return expr +} + +// parseArg parses one argument: a quoted string, an integer, a boolean, a +// labelled argument (`lat Float64` in a Tuple, `'a' = 1` in an Enum), or a +// type. +func parseArg(a string) typeArg { + a = strings.TrimSpace(a) + if strings.HasPrefix(a, "'") { + end := skipQuoted(a, 0) + lit := unquote(a[1 : end-1]) + if rest := strings.TrimSpace(a[end:]); strings.HasPrefix(rest, "=") { + arg := parseArg(rest[1:]) + arg.Label = lit + return arg + } + return typeArg{String: &lit} + } + if n, err := strconv.ParseInt(a, 10, 64); err == nil { + return typeArg{Int: &n} + } + switch strings.ToLower(a) { + case "true", "false": + b := strings.EqualFold(a, "true") + return typeArg{Bool: &b} + } + if i := labelEnd(a); i > 0 { + arg := parseArg(a[i+1:]) + arg.Label = a[:i] + return arg + } + return typeArg{Type: parseType(a)} +} + +// labelEnd returns the index of the space separating a label from the type +// it labels, or -1 when the argument has no label: a space that comes before +// any parenthesis, as in `lat Float64` or `tags Array(String)`. +func labelEnd(a string) int { + head := a + if p := strings.IndexByte(a, '('); p >= 0 { + head = a[:p] + } + return strings.IndexByte(head, ' ') +} + +// unquote undoes the escaping inside a single-quoted ClickHouse literal. +func unquote(s string) string { + s = strings.ReplaceAll(s, `\'`, `'`) + s = strings.ReplaceAll(s, `''`, `'`) + return strings.ReplaceAll(s, `\\`, `\`) +} + +// splitType splits `Base(arg, arg)` into its base name and top-level +// arguments, leaving nested parentheses and quoted strings intact. +func splitType(t string) (string, []string) { + t = strings.TrimSpace(t) + open := strings.IndexByte(t, '(') + if open < 0 || !strings.HasSuffix(t, ")") { + return t, nil + } + base := strings.TrimSpace(t[:open]) + inner := t[open+1 : len(t)-1] + var ( + args []string + depth int + quote byte + start int + ) + for i := 0; i < len(inner); i++ { + c := inner[i] + switch { + case quote != 0: + if c == '\\' { + i++ + } else if c == quote { + quote = 0 + } + case c == '\'' || c == '"' || c == '`': + quote = c + case c == '(': + depth++ + case c == ')': + depth-- + case c == ',' && depth == 0: + args = append(args, strings.TrimSpace(inner[start:i])) + start = i + 1 + } + } + args = append(args, strings.TrimSpace(inner[start:])) + return base, args +} diff --git a/internal/goldeneye/cmd/goldeneye/main.go b/internal/goldeneye/cmd/goldeneye/main.go index c3af78df38..bda6e14b76 100644 --- a/internal/goldeneye/cmd/goldeneye/main.go +++ b/internal/goldeneye/cmd/goldeneye/main.go @@ -1,12 +1,13 @@ // Command goldeneye generates the dialect seeds under -// internal/engine//dialect from a live database, and checks the -// committed ones against it. +// internal/engine//dialect from a live database, checks the +// committed ones against it, and checks the analyze cases under +// internal/endtoend/testdata against what the database itself reports. // // Usage, from internal/goldeneye: // // go run ./cmd/goldeneye install clickhouse # download the pinned clickhouse binary // go run ./cmd/goldeneye generate [engine] # rewrite the generated files from the database -// go run ./cmd/goldeneye check [engine] # compare the committed files with the database +// go run ./cmd/goldeneye check [engine] # compare the committed files and analyze cases with the database // // Without an engine, generate and check cover every engine whose database // is available and say which ones they skipped. `go test ./...` runs the @@ -21,10 +22,12 @@ import ( "io" "os" "runtime" + "strings" "github.com/sqlc-dev/sqlc/internal/goldeneye/clickhouse" "github.com/sqlc-dev/sqlc/internal/goldeneye/dialect" "github.com/sqlc-dev/sqlc/internal/goldeneye/duckdb" + "github.com/sqlc-dev/sqlc/internal/goldeneye/endtoend" "github.com/sqlc-dev/sqlc/internal/goldeneye/postgresql" ) @@ -41,7 +44,7 @@ const usage = `usage: goldeneye generate [engine] rewrite the generated dialect files from the database, for every available engine or one goldeneye check [engine] - compare the committed dialect files with the database, for every available engine or one + compare the committed dialect files and analyze cases with the database, for every available engine or one engines: clickhouse, duckdb, postgresql` @@ -55,12 +58,15 @@ type engine struct { version func(context.Context, string) (string, error) // generate reads the dialect from the database. generate func(context.Context, string) (dialect.Files, error) + // analyze asks the database what it reports for an analyze case, in + // the shape sqlc analyze prints. Nil for an engine without one yet. + analyze func(context.Context, string, endtoend.Case) ([]byte, error) } var engines = []engine{ - {clickhouse.Engine, clickhouse.Locate, clickhouse.Version, clickhouse.Generate}, - {duckdb.Engine, duckdb.Locate, duckdb.Version, duckdb.Generate}, - {postgresql.Engine, postgresql.Locate, postgresql.Version, postgresql.Generate}, + {clickhouse.Engine, clickhouse.Locate, clickhouse.Version, clickhouse.Generate, clickhouse.Analyze}, + {duckdb.Engine, duckdb.Locate, duckdb.Version, duckdb.Generate, nil}, + {postgresql.Engine, postgresql.Locate, postgresql.Version, postgresql.Generate, nil}, } func run(ctx context.Context, args []string, stdout, stderr io.Writer) error { @@ -184,5 +190,37 @@ func check(ctx context.Context, e engine, handle string, stderr io.Writer) error return fmt.Errorf("%s does not match the database\n%s", dir, report) } fmt.Fprintf(stderr, "%s: ok, %d file(s) match\n", e.name, len(files)) + return checkAnalyzeCases(ctx, e, handle, stderr) +} + +// checkAnalyzeCases compares what the database reports for each of the +// engine's analyze cases with the case's committed output. +func checkAnalyzeCases(ctx context.Context, e engine, handle string, stderr io.Writer) error { + if e.analyze == nil { + return nil + } + cases, err := endtoend.Cases(e.name) + if err != nil { + return err + } + var report strings.Builder + for _, c := range cases { + got, err := e.analyze(ctx, handle, c) + if err != nil { + fmt.Fprintf(&report, "%s: %v\n", c.Name, err) + continue + } + diff, err := c.Compare(got) + if err != nil { + return err + } + if diff != "" { + fmt.Fprintf(&report, "%s (-committed +database)\n%s", c.Name, diff) + } + } + if report.Len() > 0 { + return fmt.Errorf("analyze cases do not match the database\n%s", report.String()) + } + fmt.Fprintf(stderr, "%s: ok, %d analyze case(s) match\n", e.name, len(cases)) return nil } diff --git a/internal/goldeneye/endtoend/endtoend.go b/internal/goldeneye/endtoend/endtoend.go new file mode 100644 index 0000000000..e6506f0cc0 --- /dev/null +++ b/internal/goldeneye/endtoend/endtoend.go @@ -0,0 +1,140 @@ +// Package endtoend finds the analyze cases under internal/endtoend/testdata +// and compares an engine's own answer with the output a case committed, +// the way the dialect package compares a generated dialect with the +// committed one. +package endtoend + +import ( + "bytes" + "encoding/json" + "errors" + "fmt" + "os" + "path/filepath" + "runtime" + "strings" + + "github.com/sqlc-dev/sqlc/internal/goldeneye/dialect" +) + +// Case is one analyze case: the files sqlc analyze ran with, the fixture +// loaded before the queries run against a real database, and the output +// sqlc committed. +type Case struct { + Name string // analyze_params/clickhouse + Dir string + Schema string + Query string + Fixture string // empty when the case has no fixture.sql + Output string +} + +// Testdata returns the end-to-end testdata directory, found relative to this +// source file so the working directory does not matter. +func Testdata() (string, error) { + _, file, _, ok := runtime.Caller(0) + if !ok { + return "", errors.New("cannot locate the testcheck source directory") + } + dir := filepath.Join(filepath.Dir(file), "..", "..", "endtoend", "testdata") + if _, err := os.Stat(dir); err != nil { + return "", err + } + return filepath.Clean(dir), nil +} + +// Cases lists the analyze cases for an engine: every analyze_*/ +// directory whose exec.json runs the analyze command. A case that asks for +// the AST is skipped, since only sqlc can print that. +func Cases(engine string) ([]Case, error) { + root, err := Testdata() + if err != nil { + return nil, err + } + dirs, err := filepath.Glob(filepath.Join(root, "analyze_*", engine)) + if err != nil { + return nil, err + } + var cases []Case + for _, dir := range dirs { + c, ok, err := load(dir) + if err != nil { + return nil, fmt.Errorf("%s: %w", dir, err) + } + if ok { + cases = append(cases, c) + } + } + return cases, nil +} + +func load(dir string) (Case, bool, error) { + blob, err := os.ReadFile(filepath.Join(dir, "exec.json")) + if errors.Is(err, os.ErrNotExist) { + return Case{}, false, nil + } + if err != nil { + return Case{}, false, err + } + var exec struct { + Command string `json:"command"` + Args []string `json:"args"` + } + if err := json.Unmarshal(blob, &exec); err != nil { + return Case{}, false, fmt.Errorf("exec.json: %w", err) + } + if exec.Command != "analyze" { + return Case{}, false, nil + } + var schema, query string + for i := 0; i < len(exec.Args); i++ { + switch arg := exec.Args[i]; arg { + case "--schema", "-s", "--dialect", "-d": + i++ + if arg == "--schema" || arg == "-s" { + if i < len(exec.Args) { + schema = exec.Args[i] + } + } + case "--ast": + return Case{}, false, nil + default: + if !strings.HasPrefix(arg, "-") { + query = arg + } + } + } + if schema == "" || query == "" { + return Case{}, false, errors.New("exec.json: analyze needs --schema and a query file") + } + c := Case{ + Name: filepath.Join(filepath.Base(filepath.Dir(dir)), filepath.Base(dir)), + Dir: dir, + Schema: filepath.Join(dir, schema), + Query: filepath.Join(dir, query), + Output: filepath.Join(dir, "output.json"), + } + if fixture := filepath.Join(dir, "fixture.sql"); fileExists(fixture) { + c.Fixture = fixture + } + return c, true, nil +} + +func fileExists(path string) bool { + _, err := os.Stat(path) + return err == nil +} + +// Compare checks an engine's answer against the case's committed output +// byte for byte and returns a line diff when they differ, or "" when they +// match. +func (c Case) Compare(got []byte) (string, error) { + want, err := os.ReadFile(c.Output) + if err != nil { + return "", err + } + if bytes.Equal(want, got) { + return "", nil + } + return dialect.Diff(string(want), string(got)), nil +} diff --git a/internal/sql/preprocess/dialect.go b/internal/sql/preprocess/dialect.go index 29340bb1f4..4d8d4ae85b 100644 --- a/internal/sql/preprocess/dialect.go +++ b/internal/sql/preprocess/dialect.go @@ -88,11 +88,19 @@ var dialects = map[config.Engine]Dialect{ Backtick: true, Backslash: true, }, + // ClickHouse binds with an unnumbered ?, like MySQL, and its identifiers + // keep their case. + config.EngineClickHouse: { + Style: StyleQuestion, + Question: true, + Backtick: true, + Backslash: true, + }, } // DialectFor returns the lexical rules for an engine, and whether the engine is -// preprocessed at all. GoogleSQL and ClickHouse are not: they handle their own -// parameter syntax, so their queries reach the parser unchanged. +// preprocessed at all. GoogleSQL is not: it handles its own parameter syntax, +// so its queries reach the parser unchanged. func DialectFor(engine config.Engine) (Dialect, bool) { d, ok := dialects[engine] return d, ok diff --git a/internal/sql/preprocess/preprocess.go b/internal/sql/preprocess/preprocess.go index 76501cb60a..ad25bc9b2f 100644 --- a/internal/sql/preprocess/preprocess.go +++ b/internal/sql/preprocess/preprocess.go @@ -166,10 +166,10 @@ type occurrence struct { // File rewrites every sqlc construct in src to native SQL for the given engine. // -// Engines that are not preprocessed — GoogleSQL and ClickHouse, which handle -// their own parameter syntax — get the source back unchanged, with an empty -// side table. sqlc.arg() and friends are not rewritten for them, so they reach -// the parser as the function calls they look like. +// An engine that is not preprocessed — GoogleSQL, which handles its own +// parameter syntax — gets the source back unchanged, with an empty side +// table. sqlc.arg() and friends are not rewritten for it, so they reach the +// parser as the function calls they look like. func File(engine config.Engine, src string) *Result { d, ok := DialectFor(engine) if !ok {