From 114cd257e21feb3a404454463c1f6b830575e19a Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 2 Sep 2026 17:45:38 +0000 Subject: [PATCH 01/10] Add ClickHouse testgen module for recording ground-truth query analysis Add a nested, dependency-free Go module at internal/engine/clickhouse/testgen that records what ClickHouse itself reports about a schema, fixture and sqlc query file, in the same JSON shape as `sqlc analyze`, so the two can be diffed. `testgen install` downloads the pinned clickhouse release for the running platform into the user cache directory. `testgen analyze` runs each query in its own `clickhouse local` process: result column types and nullability come from the executed query's result header, provenance from EXPLAIN QUERY TREE (followed through subqueries, CTEs and unions), and parameters from ordinal-carrying sentinel constants substituted for ?, sqlc.arg() and sqlc.narg(), or from DESCRIBE TABLE for INSERT ... VALUES. Golden tests under testdata/ cover the type lowering, expressions, subqueries and exec statements. The analyze_params case reproduces the existing sqlc analyze golden byte for byte. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01Y6XkyWnx7iJFEb8q3AnYps --- CLAUDE.md | 3 + internal/engine/clickhouse/testgen/README.md | 57 +++ internal/engine/clickhouse/testgen/analyze.go | 283 +++++++++++++++ internal/engine/clickhouse/testgen/go.mod | 3 + internal/engine/clickhouse/testgen/install.go | 172 +++++++++ internal/engine/clickhouse/testgen/local.go | 82 +++++ internal/engine/clickhouse/testgen/main.go | 132 +++++++ internal/engine/clickhouse/testgen/queries.go | 187 ++++++++++ .../testdata/analyze_params/fixture.sql | 2 + .../testgen/testdata/analyze_params/query.sql | 5 + .../testdata/analyze_params/schema.sql | 7 + .../testdata/analyze_params/stdout.txt | 76 ++++ .../testgen/testdata/exec/fixture.sql | 2 + .../testgen/testdata/exec/query.sql | 11 + .../testgen/testdata/exec/schema.sql | 11 + .../testgen/testdata/exec/stdout.txt | 129 +++++++ .../testgen/testdata/expressions/fixture.sql | 2 + .../testgen/testdata/expressions/query.sql | 29 ++ .../testgen/testdata/expressions/schema.sql | 11 + .../testgen/testdata/expressions/stdout.txt | 335 ++++++++++++++++++ .../testgen/testdata/subqueries/fixture.sql | 2 + .../testgen/testdata/subqueries/query.sql | 26 ++ .../testgen/testdata/subqueries/schema.sql | 11 + .../testgen/testdata/subqueries/stdout.txt | 156 ++++++++ .../testgen/testdata/types/fixture.sql | 1 + .../testgen/testdata/types/query.sql | 5 + .../testgen/testdata/types/schema.sql | 20 ++ .../testgen/testdata/types/stdout.txt | 268 ++++++++++++++ .../engine/clickhouse/testgen/testgen_test.go | 55 +++ internal/engine/clickhouse/testgen/tree.go | 236 ++++++++++++ internal/engine/clickhouse/testgen/types.go | 80 +++++ 31 files changed, 2399 insertions(+) create mode 100644 internal/engine/clickhouse/testgen/README.md create mode 100644 internal/engine/clickhouse/testgen/analyze.go create mode 100644 internal/engine/clickhouse/testgen/go.mod create mode 100644 internal/engine/clickhouse/testgen/install.go create mode 100644 internal/engine/clickhouse/testgen/local.go create mode 100644 internal/engine/clickhouse/testgen/main.go create mode 100644 internal/engine/clickhouse/testgen/queries.go create mode 100644 internal/engine/clickhouse/testgen/testdata/analyze_params/fixture.sql create mode 100644 internal/engine/clickhouse/testgen/testdata/analyze_params/query.sql create mode 100644 internal/engine/clickhouse/testgen/testdata/analyze_params/schema.sql create mode 100644 internal/engine/clickhouse/testgen/testdata/analyze_params/stdout.txt create mode 100644 internal/engine/clickhouse/testgen/testdata/exec/fixture.sql create mode 100644 internal/engine/clickhouse/testgen/testdata/exec/query.sql create mode 100644 internal/engine/clickhouse/testgen/testdata/exec/schema.sql create mode 100644 internal/engine/clickhouse/testgen/testdata/exec/stdout.txt create mode 100644 internal/engine/clickhouse/testgen/testdata/expressions/fixture.sql create mode 100644 internal/engine/clickhouse/testgen/testdata/expressions/query.sql create mode 100644 internal/engine/clickhouse/testgen/testdata/expressions/schema.sql create mode 100644 internal/engine/clickhouse/testgen/testdata/expressions/stdout.txt create mode 100644 internal/engine/clickhouse/testgen/testdata/subqueries/fixture.sql create mode 100644 internal/engine/clickhouse/testgen/testdata/subqueries/query.sql create mode 100644 internal/engine/clickhouse/testgen/testdata/subqueries/schema.sql create mode 100644 internal/engine/clickhouse/testgen/testdata/subqueries/stdout.txt create mode 100644 internal/engine/clickhouse/testgen/testdata/types/fixture.sql create mode 100644 internal/engine/clickhouse/testgen/testdata/types/query.sql create mode 100644 internal/engine/clickhouse/testgen/testdata/types/schema.sql create mode 100644 internal/engine/clickhouse/testgen/testdata/types/stdout.txt create mode 100644 internal/engine/clickhouse/testgen/testgen_test.go create mode 100644 internal/engine/clickhouse/testgen/tree.go create mode 100644 internal/engine/clickhouse/testgen/types.go diff --git a/CLAUDE.md b/CLAUDE.md index dfce6cfb08..3eecfb0f04 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -233,6 +233,9 @@ MYSQL_SERVER_URI="root:mysecretpassword@tcp(127.0.0.1:3306)/mysql?multiStatement - `/postgresql/` - PostgreSQL parser and converter - `/dolphin/` - MySQL parser (uses TiDB parser) - `/sqlite/` - SQLite parser + - `/clickhouse/testgen/` - Nested module that records what a real + ClickHouse reports about a schema, fixture and queries, in the `sqlc + analyze` output format; see its README - `/duckdb/` - DuckDB 2.0 parser (uses darkwing, the pure Go port of DuckDB's PEG parser) - `/dialect/` - The engine's type system and standard library, as diff --git a/internal/engine/clickhouse/testgen/README.md b/internal/engine/clickhouse/testgen/README.md new file mode 100644 index 0000000000..6f98a4c734 --- /dev/null +++ b/internal/engine/clickhouse/testgen/README.md @@ -0,0 +1,57 @@ +# ClickHouse testgen + +`testgen` records what ClickHouse itself reports about a set of sqlc queries, +so the answer can be committed as a golden file and compared with what sqlc's +own analysis produces for the same schema and queries. + +It is a nested Go module with no dependencies beyond the standard library. +Run it from this directory: + +```bash +# Download the pinned clickhouse binary into the user cache directory. +go run . install + +# Analyze every query in query.sql against schema.sql with fixture.sql loaded. +go run . analyze --schema schema.sql --fixture fixture.sql query.sql +``` + +The binary is looked up in the `CLICKHOUSE` environment variable first, then +in the cache populated by `install`. The version is pinned in `install.go`; +bumping it can change the query tree format and type inference, so regenerate +and review the goldens afterwards. + +## How it works + +Each query runs in its own `clickhouse local` process, which needs no server, +no network and no configuration: the schema and fixture are loaded fresh, the +query is explained and then executed, and the process exits. + +- **Types and nullability** come from the executed query's result header, + exactly as a driver would see them. +- **Provenance** comes from `EXPLAIN QUERY TREE`, whose resolved columns point + at the table expression they read from. References are followed through + subqueries, CTEs and unions to the base table. +- **Parameters** are invisible to ClickHouse, which substitutes `?` on the + client. Each `?`, `sqlc.arg()` and `sqlc.narg()` is replaced by a constant + expression that carries its ordinal, `(NULL + k)`, or `toUInt64(4294967295 + k)` + after `LIMIT` and `OFFSET`. The query tree prints the expression each folded + constant came from, so the placeholder is found again and described by the + operand it is compared with. Parameters of `INSERT ... VALUES` map onto the + target columns reported by `DESCRIBE TABLE`. + +The output has the same shape as `sqlc analyze`, with ClickHouse types lowered +the way sqlc's ClickHouse engine lowers them: the lowercased base name with +parameters dropped, `Nullable` and `Array` folded into `not_null` and +`is_array`, and `LowCardinality` discarded. + +## Tests + +`testdata//` holds a `schema.sql`, `query.sql`, an optional +`fixture.sql` and the expected `stdout.txt`. The test skips unless a binary is +available. + +```bash +go run . install +go test . +go test . -update # rewrite every stdout.txt +``` diff --git a/internal/engine/clickhouse/testgen/analyze.go b/internal/engine/clickhouse/testgen/analyze.go new file mode 100644 index 0000000000..4b2b7cd85e --- /dev/null +++ b/internal/engine/clickhouse/testgen/analyze.go @@ -0,0 +1,283 @@ +package main + +import ( + "context" + "encoding/json" + "fmt" + "regexp" + "strconv" + "strings" +) + +// The output mirrors the JSON `sqlc analyze` prints so a golden generated +// here can be diffed against sqlc's own analysis of the same files. + +type analyzedQuery struct { + Name string `json:"name"` + Cmd string `json:"cmd"` + Columns []analyzedColumn `json:"columns"` + Params []analyzedParam `json:"params"` +} + +type analyzedColumn struct { + Name string `json:"name"` + DataType string `json:"data_type"` + NotNull bool `json:"not_null"` + IsArray bool `json:"is_array"` + Table string `json:"table,omitempty"` +} + +type analyzedParam struct { + Number int `json:"number"` + Column analyzedColumn `json:"column"` +} + +// analyze runs every query against the schema and fixture and records what +// ClickHouse reports about each. +func analyze(ctx context.Context, l local, schema, fixture string, queries []query) ([]analyzedQuery, error) { + out := make([]analyzedQuery, 0, len(queries)) + for _, q := range queries { + aq, err := analyzeQuery(ctx, l, schema, fixture, q) + if err != nil { + return nil, fmt.Errorf("%s: %w", q.Name, err) + } + out = append(out, aq) + } + return out, nil +} + +func analyzeQuery(ctx context.Context, l local, schema, fixture string, q query) (analyzedQuery, error) { + sql, phs := bindPlaceholders(q.SQL) + explain := returnsRows(sql) + + var script strings.Builder + for _, stmt := range []string{schema, fixture} { + if s := strings.TrimRight(strings.TrimSpace(stmt), ";"); s != "" { + script.WriteString(s) + script.WriteString(";\n") + } + } + if explain { + script.WriteString("EXPLAIN QUERY TREE " + sql + ";\n") + } + script.WriteString(sql + ";\n") + + results, err := l.run(ctx, script.String()) + if err != nil { + return analyzedQuery{}, err + } + + aq := analyzedQuery{ + Name: q.Name, + Cmd: q.Cmd, + Columns: []analyzedColumn{}, + Params: []analyzedParam{}, + } + if !explain { + return analyzeExec(ctx, l, script.String(), sql, phs, aq) + } + if len(results) != 2 { + return analyzedQuery{}, fmt.Errorf("expected the query tree and one result set, got %d results", len(results)) + } + + var lines []string + for _, row := range results[0].Data { + var line string + if err := json.Unmarshal(row["explain"], &line); err != nil { + return analyzedQuery{}, fmt.Errorf("reading query tree: %w", err) + } + lines = append(lines, line) + } + tree, err := parseQueryTree(lines) + if err != nil { + return analyzedQuery{}, err + } + + // Names and types come from the block header of the executed query, the + // same header a driver sees. The tree only adds where each came from. + _, nodes := projection(firstNode(tree.root.children)) + for i, col := range results[1].Meta { + ac := column(col.Name, col.Type) + if i < len(nodes) && len(nodes) == len(results[1].Meta) { + ac.Table = tree.sourceTable(nodes[i]) + } + aq.Columns = append(aq.Columns, ac) + } + + sentinels := tree.sentinels() + for i, ph := range phs { + ac := analyzedColumn{} + if sentinel := sentinels[i+1]; sentinel != nil { + ac = tree.paramColumn(sentinel) + } + if ph.Name != "" { + ac.Name = ph.Name + } + aq.Params = append(aq.Params, analyzedParam{Number: ph.Number, Column: ac}) + } + return aq, nil +} + +func column(name, typ string) analyzedColumn { + dataType, isArray, notNull := sqlcType(typ) + return analyzedColumn{Name: name, DataType: dataType, NotNull: notNull, IsArray: isArray} +} + +// returnsRows reports whether a statement produces a result set and so can +// be explained as a query tree. +func returnsRows(sql string) bool { + head := strings.ToLower(strings.TrimSpace(sql)) + if strings.HasPrefix(head, "(") { + return true + } + for _, kw := range []string{"select", "with", "show", "describe", "desc", "exists"} { + if strings.HasPrefix(head, kw) && (len(head) == len(kw) || !isWordByte(head[len(kw)])) { + return true + } + } + return false +} + +// sentinels finds the constants the placeholders were substituted with, +// keyed by placeholder ordinal. +func (t *queryTree) sentinels() map[int]*treeNode { + found := map[int]*treeNode{} + var walk func(n *treeNode) + walk = func(n *treeNode) { + if n.kind == "CONSTANT" { + if k, ok := sentinelOrdinal(n); ok { + found[k] = n + return + } + } + for _, c := range n.children { + walk(c) + } + } + walk(t.root) + return found +} + +// sentinelOrdinal recognises a constant folded directly from `NULL + k` or +// `4294967295 + k`, the shapes sentinelFor produces, and returns k. A +// constant folded from a larger expression that merely contains a sentinel +// does not match; the sentinel is found nested inside it instead. +func sentinelOrdinal(c *treeNode) (int, bool) { + fn := firstNode(c.section("EXPRESSION").childrenOrNil()) + if fn == nil || fn.kind != "FUNCTION" || fn.attrs["function_name"] != "plus" { + return 0, false + } + args := fn.section("ARGUMENTS").list() + if len(args) != 2 || args[0].kind != "CONSTANT" || args[1].kind != "CONSTANT" { + return 0, false + } + base := args[0].attrs["constant_value"] + if base != "NULL" && base != "UInt64_"+limitBase { + return 0, false + } + k, err := strconv.Atoi(strings.TrimPrefix(args[1].attrs["constant_value"], "UInt64_")) + if err != nil { + return 0, false + } + return k, true +} + +// paramColumn describes what a placeholder is compared with or assigned to: +// the other operand of the function it is an argument of, preferring a +// column over an expression, or the projected column it stands for. +func (t *queryTree) paramColumn(sentinel *treeNode) analyzedColumn { + list := sentinel.parent + if list != nil && list.kind == "LIST" && list.parent != nil { + switch owner := list.parent; { + case owner.kind == "" && owner.text == "ARGUMENTS": + // Prefer a column operand, then an expression, then a constant. + rank := map[string]int{"COLUMN": 0, "FUNCTION": 1, "CONSTANT": 2} + var best *treeNode + for _, sib := range list.children { + if sib == sentinel { + continue + } + if r, ok := rank[sib.kind]; ok && (best == nil || r < rank[best.kind]) { + best = sib + } + } + if best != nil { + return t.describe(best) + } + case owner.kind == "" && owner.text == "PROJECTION": + names, nodes := projection(owner.parent) + for i, n := range nodes { + if n == sentinel && i < len(names) { + ac := column(names[i], sentinel.attrs["constant_value_type"]) + return ac + } + } + } + } + return column("", sentinel.attrs["constant_value_type"]) +} + +// describe turns a tree expression into a column description. +func (t *queryTree) describe(n *treeNode) analyzedColumn { + switch n.kind { + case "COLUMN": + ac := column(n.attrs["column_name"], n.attrs["result_type"]) + ac.Table = t.sourceTable(n) + return ac + case "FUNCTION": + return column(n.attrs["function_name"], n.attrs["result_type"]) + case "CONSTANT": + return column("", n.attrs["constant_value_type"]) + } + return analyzedColumn{} +} + +var insertValuesRe = regexp.MustCompile(`(?is)^insert\s+into\s+(?:table\s+)?([\w.` + "`" + `"]+)\s*(?:\(([^)]*)\))?\s*(?:format\s+)?values\b`) + +// analyzeExec runs a statement that returns no rows. The only parameters it +// can describe are those of an INSERT ... VALUES, which map positionally +// onto the target columns reported by DESCRIBE TABLE. +func analyzeExec(ctx context.Context, l local, script, sql string, phs []placeholder, aq analyzedQuery) (analyzedQuery, error) { + m := insertValuesRe.FindStringSubmatch(sql) + if m != nil { + script += "DESCRIBE TABLE " + m[1] + ";\n" + } + results, err := l.run(ctx, script) + if err != nil { + return analyzedQuery{}, err + } + + var targets []analyzedColumn + if m != nil && len(results) == 1 { + byName := map[string]analyzedColumn{} + var all []analyzedColumn + table := strings.Trim(m[1][strings.LastIndexByte(m[1], '.')+1:], "`\"") + for _, row := range results[0].Data { + var name, typ string + json.Unmarshal(row["name"], &name) + json.Unmarshal(row["type"], &typ) + ac := column(name, typ) + ac.Table = table + byName[name] = ac + all = append(all, ac) + } + if strings.TrimSpace(m[2]) == "" { + targets = all + } else { + for _, name := range strings.Split(m[2], ",") { + targets = append(targets, byName[strings.Trim(strings.TrimSpace(name), "`\"")]) + } + } + } + for i, ph := range phs { + ac := analyzedColumn{} + if len(targets) > 0 { + ac = targets[i%len(targets)] + } + if ph.Name != "" { + ac.Name = ph.Name + } + aq.Params = append(aq.Params, analyzedParam{Number: ph.Number, Column: ac}) + } + return aq, nil +} diff --git a/internal/engine/clickhouse/testgen/go.mod b/internal/engine/clickhouse/testgen/go.mod new file mode 100644 index 0000000000..fdace64397 --- /dev/null +++ b/internal/engine/clickhouse/testgen/go.mod @@ -0,0 +1,3 @@ +module github.com/sqlc-dev/sqlc/internal/engine/clickhouse/testgen + +go 1.24.0 diff --git a/internal/engine/clickhouse/testgen/install.go b/internal/engine/clickhouse/testgen/install.go new file mode 100644 index 0000000000..6eb7dde49e --- /dev/null +++ b/internal/engine/clickhouse/testgen/install.go @@ -0,0 +1,172 @@ +package main + +import ( + "archive/tar" + "compress/gzip" + "context" + "errors" + "fmt" + "io" + "net/http" + "os" + "path/filepath" + "strconv" + "strings" +) + +// DefaultVersion is the ClickHouse release the goldens are generated with. +// Bumping it is a deliberate change: the query tree format and type +// inference can shift between releases, so regenerate and review the goldens +// after changing it. +const DefaultVersion = "25.8.2.29" + +// releaseTag returns the GitHub release tag for a version. ClickHouse tags +// its March and August releases as LTS and everything else as stable. +func releaseTag(version string) (string, error) { + parts := strings.Split(version, ".") + if len(parts) != 4 { + return "", fmt.Errorf("invalid ClickHouse version %q: want MAJOR.MINOR.PATCH.BUILD", version) + } + minor, err := strconv.Atoi(parts[1]) + if err != nil { + return "", fmt.Errorf("invalid ClickHouse version %q: %w", version, err) + } + suffix := "-stable" + if minor == 3 || minor == 8 { + suffix = "-lts" + } + return "v" + version + suffix, nil +} + +// releaseAsset returns the download URL for a platform and whether it is a +// tarball holding the binary at usr/bin/clickhouse rather than the bare +// binary. Linux builds are only published as tarballs; macOS builds only as +// bare binaries. +func releaseAsset(version, goos, goarch string) (url string, tarball bool, err error) { + tag, err := releaseTag(version) + if err != nil { + return "", false, err + } + base := "https://github.com/ClickHouse/ClickHouse/releases/download/" + tag + "/" + switch goos + "/" + goarch { + case "linux/amd64": + return base + "clickhouse-common-static-" + version + "-amd64.tgz", true, nil + case "linux/arm64": + return base + "clickhouse-common-static-" + version + "-arm64.tgz", true, nil + case "darwin/amd64": + return base + "clickhouse-macos", false, nil + case "darwin/arm64": + return base + "clickhouse-macos-aarch64", false, nil + } + return "", false, fmt.Errorf("no ClickHouse build is published for %s/%s", goos, goarch) +} + +// cachedBinary is where Install puts the binary for a version. +func cachedBinary(version string) (string, error) { + dir, err := os.UserCacheDir() + if err != nil { + return "", err + } + return filepath.Join(dir, "sqlc-clickhouse", version, "clickhouse"), nil +} + +// Locate finds a clickhouse binary: the CLICKHOUSE environment variable wins, +// then the cached copy of DefaultVersion. +func Locate() (string, error) { + if path := os.Getenv("CLICKHOUSE"); path != "" { + return path, nil + } + path, err := cachedBinary(DefaultVersion) + if err != nil { + return "", err + } + if _, err := os.Stat(path); err != nil { + return "", fmt.Errorf("clickhouse %s is not installed: run `testgen install` or set CLICKHOUSE to a clickhouse binary", DefaultVersion) + } + return path, nil +} + +// Install downloads the clickhouse binary for a version into the cache and +// returns its path. It is a no-op when the version is already cached. +func Install(ctx context.Context, version, goos, goarch string, progress io.Writer) (string, error) { + dest, err := cachedBinary(version) + if err != nil { + return "", err + } + if _, err := os.Stat(dest); err == nil { + return dest, nil + } + url, tarball, err := releaseAsset(version, goos, goarch) + if err != nil { + return "", err + } + if err := os.MkdirAll(filepath.Dir(dest), 0o755); err != nil { + return "", err + } + + fmt.Fprintf(progress, "downloading %s\n", url) + req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) + if err != nil { + return "", err + } + resp, err := http.DefaultClient.Do(req) + if err != nil { + return "", err + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + return "", fmt.Errorf("downloading %s: %s", url, resp.Status) + } + + // Write next to the destination and rename so a partial download never + // masquerades as an installed binary. + tmp, err := os.CreateTemp(filepath.Dir(dest), "clickhouse-*.partial") + if err != nil { + return "", err + } + defer os.Remove(tmp.Name()) + + var src io.Reader = resp.Body + if tarball { + src, err = binaryInTarball(resp.Body) + if err != nil { + return "", fmt.Errorf("downloading %s: %w", url, err) + } + } + if _, err := io.Copy(tmp, src); err != nil { + tmp.Close() + return "", err + } + if err := tmp.Close(); err != nil { + return "", err + } + if err := os.Chmod(tmp.Name(), 0o755); err != nil { + return "", err + } + if err := os.Rename(tmp.Name(), dest); err != nil { + return "", err + } + return dest, nil +} + +// binaryInTarball positions a reader at the clickhouse binary inside a +// clickhouse-common-static tarball. +func binaryInTarball(r io.Reader) (io.Reader, error) { + gz, err := gzip.NewReader(r) + if err != nil { + return nil, err + } + tr := tar.NewReader(gz) + for { + hdr, err := tr.Next() + if errors.Is(err, io.EOF) { + return nil, errors.New("tarball does not contain usr/bin/clickhouse") + } + if err != nil { + return nil, err + } + if hdr.Typeflag == tar.TypeReg && strings.HasSuffix(hdr.Name, "/usr/bin/clickhouse") { + return tr, nil + } + } +} diff --git a/internal/engine/clickhouse/testgen/local.go b/internal/engine/clickhouse/testgen/local.go new file mode 100644 index 0000000000..36f899b510 --- /dev/null +++ b/internal/engine/clickhouse/testgen/local.go @@ -0,0 +1,82 @@ +package main + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "io" + "os" + "os/exec" + "path/filepath" + "strings" +) + +// local runs SQL through an ephemeral `clickhouse local` process. +type local struct { + binary string +} + +// resultSet is one JSON-format result printed by clickhouse local. Only +// statements that return rows print one; DDL and INSERT print nothing. +type resultSet struct { + Meta []resultColumn `json:"meta"` + Data []map[string]json.RawMessage `json:"data"` + Rows int `json:"rows"` +} + +type resultColumn struct { + Name string `json:"name"` + Type string `json:"type"` +} + +// run executes a multi-statement script in a fresh process with fresh +// storage and returns the result sets in statement order. Any statement +// failing fails the whole run with ClickHouse's own error message. +func (l local) run(ctx context.Context, script string) ([]resultSet, error) { + dir, err := os.MkdirTemp("", "sqlc-clickhouse-testgen-*") + if err != nil { + return nil, err + } + defer os.RemoveAll(dir) + + queries := filepath.Join(dir, "queries.sql") + if err := os.WriteFile(queries, []byte(script), 0o600); err != nil { + return nil, err + } + + // stdin must not be inherited: clickhouse local reads it as table data + // and blocks until it is closed. + cmd := exec.CommandContext(ctx, l.binary, "local", + "--multiquery", + "--queries-file", queries, + "--output-format", "JSON", + "--path", filepath.Join(dir, "data"), + ) + cmd.Stdin = nil + var stdout, stderr bytes.Buffer + cmd.Stdout = &stdout + cmd.Stderr = &stderr + if err := cmd.Run(); err != nil { + msg := strings.TrimSpace(stderr.String()) + if msg == "" { + msg = err.Error() + } + return nil, errors.New(msg) + } + + var results []resultSet + dec := json.NewDecoder(&stdout) + for { + var rs resultSet + err := dec.Decode(&rs) + if errors.Is(err, io.EOF) { + return results, nil + } + if err != nil { + return nil, fmt.Errorf("decoding clickhouse local output: %w", err) + } + results = append(results, rs) + } +} diff --git a/internal/engine/clickhouse/testgen/main.go b/internal/engine/clickhouse/testgen/main.go new file mode 100644 index 0000000000..57d8cb82f3 --- /dev/null +++ b/internal/engine/clickhouse/testgen/main.go @@ -0,0 +1,132 @@ +// Command testgen records what ClickHouse itself says about a set of sqlc +// queries, so the answer can be committed as a golden file and compared with +// what sqlc's own analysis produces. +// +// It loads a schema and a fixture into an ephemeral `clickhouse local` +// process, runs each query found in a sqlc query file against that data, and +// prints the result column types, nullability and source tables along with +// the parameters each query binds, in the same JSON shape `sqlc analyze` +// prints. +// +// The clickhouse binary is downloaded once per pinned version with +// `testgen install`, or supplied through the CLICKHOUSE environment variable. +// +// Usage: +// +// go run . install +// go run . analyze --schema schema.sql --fixture fixture.sql query.sql +package main + +import ( + "context" + "encoding/json" + "errors" + "flag" + "fmt" + "io" + "os" + "runtime" +) + +func main() { + if err := run(context.Background(), os.Args[1:], os.Stdout, os.Stderr); err != nil { + fmt.Fprintln(os.Stderr, "testgen:", err) + os.Exit(1) + } +} + +const usage = `usage: + testgen install [-version V] + download the pinned clickhouse binary into the user cache directory + testgen analyze [-clickhouse PATH] --schema FILE [--fixture FILE] QUERY_FILE + analyze every query in QUERY_FILE and print the result as JSON + +The binary is looked up in the CLICKHOUSE environment variable first, then in +the cache directory populated by "testgen install".` + +func run(ctx context.Context, args []string, stdout, stderr io.Writer) error { + if len(args) == 0 { + fmt.Fprintln(stderr, usage) + return errors.New("a command is required") + } + switch args[0] { + case "install": + return runInstall(ctx, args[1:], stdout, stderr) + case "analyze": + return runAnalyze(ctx, args[1:], stdout, stderr) + case "help", "-h", "--help": + fmt.Fprintln(stdout, usage) + return nil + default: + fmt.Fprintln(stderr, usage) + return fmt.Errorf("unknown command %q", args[0]) + } +} + +func runInstall(ctx context.Context, args []string, stdout, stderr io.Writer) error { + fs := flag.NewFlagSet("install", flag.ContinueOnError) + fs.SetOutput(stderr) + version := fs.String("version", DefaultVersion, "ClickHouse release to install") + if err := fs.Parse(args); err != nil { + return err + } + path, err := Install(ctx, *version, runtime.GOOS, runtime.GOARCH, stderr) + if err != nil { + return err + } + fmt.Fprintln(stdout, path) + return nil +} + +func runAnalyze(ctx context.Context, args []string, stdout, stderr io.Writer) error { + fs := flag.NewFlagSet("analyze", flag.ContinueOnError) + fs.SetOutput(stderr) + binary := fs.String("clickhouse", "", "path to the clickhouse binary (defaults to $CLICKHOUSE, then the cache)") + schemaPath := fs.String("schema", "", "path to the schema file") + fixturePath := fs.String("fixture", "", "path to the fixture file loaded after the schema") + if err := fs.Parse(args); err != nil { + return err + } + if fs.NArg() != 1 { + fmt.Fprintln(stderr, usage) + return errors.New("analyze takes exactly one query file") + } + if *schemaPath == "" { + return errors.New("--schema is required") + } + if *binary == "" { + path, err := Locate() + if err != nil { + return err + } + *binary = path + } + + schema, err := os.ReadFile(*schemaPath) + if err != nil { + return err + } + var fixture []byte + if *fixturePath != "" { + fixture, err = os.ReadFile(*fixturePath) + if err != nil { + return err + } + } + querySrc, err := os.ReadFile(fs.Arg(0)) + if err != nil { + return err + } + queries, err := parseQueries(string(querySrc)) + if err != nil { + return fmt.Errorf("%s: %w", fs.Arg(0), err) + } + + out, err := analyze(ctx, local{binary: *binary}, string(schema), string(fixture), queries) + if err != nil { + return err + } + enc := json.NewEncoder(stdout) + enc.SetIndent("", " ") + return enc.Encode(out) +} diff --git a/internal/engine/clickhouse/testgen/queries.go b/internal/engine/clickhouse/testgen/queries.go new file mode 100644 index 0000000000..c4880c284a --- /dev/null +++ b/internal/engine/clickhouse/testgen/queries.go @@ -0,0 +1,187 @@ +package main + +import ( + "fmt" + "regexp" + "strings" + "unicode" +) + +// query is one entry of a sqlc query file. +type query struct { + Name string + Cmd string + SQL string +} + +var headerRe = regexp.MustCompile(`^\s*--\s*name:\s*(\S+)\s+(:\S+)\s*$`) + +// parseQueries splits a sqlc query file on its `-- name: X :cmd` headers. +func parseQueries(src string) ([]query, error) { + var queries []query + var cur *query + var body []string + flush := func() error { + if cur == nil { + return nil + } + sql := strings.TrimSpace(strings.Join(body, "\n")) + sql = strings.TrimRight(sql, "; \t\r\n") + if sql == "" { + return fmt.Errorf("query %s has no body", cur.Name) + } + cur.SQL = sql + queries = append(queries, *cur) + return nil + } + for _, line := range strings.Split(src, "\n") { + if m := headerRe.FindStringSubmatch(line); m != nil { + if err := flush(); err != nil { + return nil, err + } + cur = &query{Name: m[1], Cmd: m[2]} + body = body[:0] + continue + } + if cur != nil { + body = append(body, line) + } + } + if err := flush(); err != nil { + return nil, err + } + if len(queries) == 0 { + return nil, fmt.Errorf("no queries found: expected `-- name: Name :cmd` headers") + } + return queries, nil +} + +// placeholder is one parameter reference in a query, in order of appearance. +type placeholder struct { + Number int + Name string // sqlc.arg / sqlc.narg name, empty for ? +} + +// Placeholders are substituted with constant expressions that carry their +// ordinal, so they can be told apart from each other and from literal NULLs +// once ClickHouse has folded them: the query tree prints the expression a +// folded constant came from. NULL coerces to any type, so a comparison +// against it analyzes with the other operand's type. LIMIT and OFFSET +// reject NULL and only accept unsigned integers, so those get a value no +// query would plausibly contain. +const limitBase = "4294967295" + +func sentinelFor(lastWord string, ordinal int) string { + switch strings.ToLower(lastWord) { + case "limit", "offset": + return fmt.Sprintf("toUInt64(%s + %d)", limitBase, ordinal) + } + return fmt.Sprintf("(NULL + %d)", ordinal) +} + +var namedArgRe = regexp.MustCompile(`^sqlc\.(n?arg)\(\s*'?([A-Za-z_][A-Za-z0-9_]*)'?\s*\)`) + +// bindPlaceholders rewrites sqlc's parameter syntax (?, sqlc.arg(name), +// sqlc.narg(name)) into constants ClickHouse can analyze, skipping string +// literals, quoted identifiers and comments. Placeholders are numbered the +// way sqlc numbers them: positionally, with a repeated name sharing a number. +func bindPlaceholders(sql string) (string, []placeholder) { + var ( + out strings.Builder + phs []placeholder + numbers = map[string]int{} + lastWord string + i = 0 + ) + number := func(name string) int { + if name != "" { + if n, ok := numbers[name]; ok { + return n + } + } + n := len(numbers) + 1 + if name == "" { + // Positional placeholders never repeat; give them a private key. + name = fmt.Sprintf("?%d", n) + } + numbers[name] = n + return n + } + for i < len(sql) { + c := sql[i] + switch { + case c == '\'' || c == '"' || c == '`': + end := skipQuoted(sql, i) + out.WriteString(sql[i:end]) + i = end + case strings.HasPrefix(sql[i:], "--"): + end := strings.IndexByte(sql[i:], '\n') + if end < 0 { + end = len(sql) + } else { + end += i + } + out.WriteString(sql[i:end]) + i = end + case strings.HasPrefix(sql[i:], "/*"): + end := strings.Index(sql[i:], "*/") + if end < 0 { + end = len(sql) + } else { + end += i + 2 + } + out.WriteString(sql[i:end]) + i = end + case c == '?': + out.WriteString(sentinelFor(lastWord, len(phs)+1)) + phs = append(phs, placeholder{Number: number("")}) + lastWord = "" + i++ + case c == 's' && namedArgRe.MatchString(sql[i:]): + m := namedArgRe.FindStringSubmatch(sql[i:]) + out.WriteString(sentinelFor(lastWord, len(phs)+1)) + phs = append(phs, placeholder{Number: number(m[2]), Name: m[2]}) + lastWord = "" + i += len(m[0]) + case isWordByte(c): + end := i + for end < len(sql) && isWordByte(sql[end]) { + end++ + } + lastWord = sql[i:end] + out.WriteString(lastWord) + i = end + default: + if !unicode.IsSpace(rune(c)) && c != ',' && c != '(' { + lastWord = "" + } + out.WriteByte(c) + i++ + } + } + return out.String(), phs +} + +func isWordByte(c byte) bool { + return c == '_' || c == '.' || c >= '0' && c <= '9' || c >= 'a' && c <= 'z' || c >= 'A' && c <= 'Z' +} + +// skipQuoted returns the index just past the quoted token starting at i, +// honouring backslash escapes and doubled quotes. +func skipQuoted(s string, i int) int { + q := s[i] + j := i + 1 + for j < len(s) { + switch { + case s[j] == '\\' && j+1 < len(s): + j += 2 + case s[j] == q && j+1 < len(s) && s[j+1] == q: + j += 2 + case s[j] == q: + return j + 1 + default: + j++ + } + } + return len(s) +} diff --git a/internal/engine/clickhouse/testgen/testdata/analyze_params/fixture.sql b/internal/engine/clickhouse/testgen/testdata/analyze_params/fixture.sql new file mode 100644 index 0000000000..f13ad59a8d --- /dev/null +++ b/internal/engine/clickhouse/testgen/testdata/analyze_params/fixture.sql @@ -0,0 +1,2 @@ +INSERT INTO events (id, name, tag, amount, created) VALUES (1, 'signup', NULL, 9.5, '2024-01-01 00:00:00'); +INSERT INTO events (id, name, tag, amount, created) VALUES (2, 'login', 'web', 0.5, '2024-01-02 00:00:00'); diff --git a/internal/engine/clickhouse/testgen/testdata/analyze_params/query.sql b/internal/engine/clickhouse/testgen/testdata/analyze_params/query.sql new file mode 100644 index 0000000000..5405a5baa8 --- /dev/null +++ b/internal/engine/clickhouse/testgen/testdata/analyze_params/query.sql @@ -0,0 +1,5 @@ +-- name: GetEvent :one +SELECT id, name FROM events WHERE id = ?; + +-- name: FilterEvents :many +SELECT id, name FROM events WHERE name = ? AND amount > ?; diff --git a/internal/engine/clickhouse/testgen/testdata/analyze_params/schema.sql b/internal/engine/clickhouse/testgen/testdata/analyze_params/schema.sql new file mode 100644 index 0000000000..29960ee63d --- /dev/null +++ b/internal/engine/clickhouse/testgen/testdata/analyze_params/schema.sql @@ -0,0 +1,7 @@ +CREATE TABLE events ( + id UInt64, + name String, + tag Nullable(String), + amount Float64, + created DateTime +) ENGINE = MergeTree ORDER BY id; diff --git a/internal/engine/clickhouse/testgen/testdata/analyze_params/stdout.txt b/internal/engine/clickhouse/testgen/testdata/analyze_params/stdout.txt new file mode 100644 index 0000000000..6fb0cc654b --- /dev/null +++ b/internal/engine/clickhouse/testgen/testdata/analyze_params/stdout.txt @@ -0,0 +1,76 @@ +[ + { + "name": "GetEvent", + "cmd": ":one", + "columns": [ + { + "name": "id", + "data_type": "uint64", + "not_null": true, + "is_array": false, + "table": "events" + }, + { + "name": "name", + "data_type": "string", + "not_null": true, + "is_array": false, + "table": "events" + } + ], + "params": [ + { + "number": 1, + "column": { + "name": "id", + "data_type": "uint64", + "not_null": true, + "is_array": false, + "table": "events" + } + } + ] + }, + { + "name": "FilterEvents", + "cmd": ":many", + "columns": [ + { + "name": "id", + "data_type": "uint64", + "not_null": true, + "is_array": false, + "table": "events" + }, + { + "name": "name", + "data_type": "string", + "not_null": true, + "is_array": false, + "table": "events" + } + ], + "params": [ + { + "number": 1, + "column": { + "name": "name", + "data_type": "string", + "not_null": true, + "is_array": false, + "table": "events" + } + }, + { + "number": 2, + "column": { + "name": "amount", + "data_type": "float64", + "not_null": true, + "is_array": false, + "table": "events" + } + } + ] + } +] diff --git a/internal/engine/clickhouse/testgen/testdata/exec/fixture.sql b/internal/engine/clickhouse/testgen/testdata/exec/fixture.sql new file mode 100644 index 0000000000..9adc5fa938 --- /dev/null +++ b/internal/engine/clickhouse/testgen/testdata/exec/fixture.sql @@ -0,0 +1,2 @@ +INSERT INTO events VALUES (1, 'signup', NULL, 9.5), (2, 'login', 'web', 0.5); +INSERT INTO users VALUES (1, 'a@example.com'); diff --git a/internal/engine/clickhouse/testgen/testdata/exec/query.sql b/internal/engine/clickhouse/testgen/testdata/exec/query.sql new file mode 100644 index 0000000000..a27579ef0f --- /dev/null +++ b/internal/engine/clickhouse/testgen/testdata/exec/query.sql @@ -0,0 +1,11 @@ +-- name: CreateUser :exec +INSERT INTO users (id, email) VALUES (?, ?); + +-- name: CreateEvent :exec +INSERT INTO events VALUES (?, ?, ?, ?); + +-- name: CreateEvents :exec +INSERT INTO events (id, name) VALUES (?, ?), (?, ?); + +-- name: DropUsers :exec +TRUNCATE TABLE users; diff --git a/internal/engine/clickhouse/testgen/testdata/exec/schema.sql b/internal/engine/clickhouse/testgen/testdata/exec/schema.sql new file mode 100644 index 0000000000..9c04e10685 --- /dev/null +++ b/internal/engine/clickhouse/testgen/testdata/exec/schema.sql @@ -0,0 +1,11 @@ +CREATE TABLE events ( + id UInt64, + name String, + tag Nullable(String), + amount Float64 +) ENGINE = MergeTree ORDER BY id; + +CREATE TABLE users ( + id UInt64, + email String +) ENGINE = MergeTree ORDER BY id; diff --git a/internal/engine/clickhouse/testgen/testdata/exec/stdout.txt b/internal/engine/clickhouse/testgen/testdata/exec/stdout.txt new file mode 100644 index 0000000000..bc9ab7f8d6 --- /dev/null +++ b/internal/engine/clickhouse/testgen/testdata/exec/stdout.txt @@ -0,0 +1,129 @@ +[ + { + "name": "CreateUser", + "cmd": ":exec", + "columns": [], + "params": [ + { + "number": 1, + "column": { + "name": "id", + "data_type": "uint64", + "not_null": true, + "is_array": false, + "table": "users" + } + }, + { + "number": 2, + "column": { + "name": "email", + "data_type": "string", + "not_null": true, + "is_array": false, + "table": "users" + } + } + ] + }, + { + "name": "CreateEvent", + "cmd": ":exec", + "columns": [], + "params": [ + { + "number": 1, + "column": { + "name": "id", + "data_type": "uint64", + "not_null": true, + "is_array": false, + "table": "events" + } + }, + { + "number": 2, + "column": { + "name": "name", + "data_type": "string", + "not_null": true, + "is_array": false, + "table": "events" + } + }, + { + "number": 3, + "column": { + "name": "tag", + "data_type": "string", + "not_null": false, + "is_array": false, + "table": "events" + } + }, + { + "number": 4, + "column": { + "name": "amount", + "data_type": "float64", + "not_null": true, + "is_array": false, + "table": "events" + } + } + ] + }, + { + "name": "CreateEvents", + "cmd": ":exec", + "columns": [], + "params": [ + { + "number": 1, + "column": { + "name": "id", + "data_type": "uint64", + "not_null": true, + "is_array": false, + "table": "events" + } + }, + { + "number": 2, + "column": { + "name": "name", + "data_type": "string", + "not_null": true, + "is_array": false, + "table": "events" + } + }, + { + "number": 3, + "column": { + "name": "id", + "data_type": "uint64", + "not_null": true, + "is_array": false, + "table": "events" + } + }, + { + "number": 4, + "column": { + "name": "name", + "data_type": "string", + "not_null": true, + "is_array": false, + "table": "events" + } + } + ] + }, + { + "name": "DropUsers", + "cmd": ":exec", + "columns": [], + "params": [] + } +] diff --git a/internal/engine/clickhouse/testgen/testdata/expressions/fixture.sql b/internal/engine/clickhouse/testgen/testdata/expressions/fixture.sql new file mode 100644 index 0000000000..9adc5fa938 --- /dev/null +++ b/internal/engine/clickhouse/testgen/testdata/expressions/fixture.sql @@ -0,0 +1,2 @@ +INSERT INTO events VALUES (1, 'signup', NULL, 9.5), (2, 'login', 'web', 0.5); +INSERT INTO users VALUES (1, 'a@example.com'); diff --git a/internal/engine/clickhouse/testgen/testdata/expressions/query.sql b/internal/engine/clickhouse/testgen/testdata/expressions/query.sql new file mode 100644 index 0000000000..dd5604e452 --- /dev/null +++ b/internal/engine/clickhouse/testgen/testdata/expressions/query.sql @@ -0,0 +1,29 @@ +-- name: Aggregates :one +SELECT count() AS n, max(amount) AS top, sum(amount), min(tag) AS first_tag, uniq(name) +FROM events; + +-- name: Expressions :many +SELECT id + 1 AS next_id, nullIf(name, '') AS maybe, coalesce(tag, 'none') AS tag_or_none, + lower(name), toDate(now()) AS today, id > 1 AS big, if(id = 1, 'one', NULL) AS word +FROM events; + +-- name: LeftJoinDefaults :many +SELECT e.id, e.name, u.email +FROM events e +LEFT JOIN users u ON u.id = e.id; + +-- name: Positional :many +SELECT id, name FROM events WHERE name = ? AND amount > ? AND tag = ?; + +-- name: Named :many +SELECT id, name FROM events +WHERE name = sqlc.arg(name) AND tag = sqlc.narg(tag) AND (amount > sqlc.arg(amount) OR amount < sqlc.arg(amount)); + +-- name: Functions :many +SELECT id FROM events WHERE lower(name) = ? AND id IN (?) AND toDate(now()) > ?; + +-- name: Paging :many +SELECT id FROM events ORDER BY id LIMIT ? OFFSET ?; + +-- name: Projected :one +SELECT ? AS echo, 'literal' AS lit; diff --git a/internal/engine/clickhouse/testgen/testdata/expressions/schema.sql b/internal/engine/clickhouse/testgen/testdata/expressions/schema.sql new file mode 100644 index 0000000000..9c04e10685 --- /dev/null +++ b/internal/engine/clickhouse/testgen/testdata/expressions/schema.sql @@ -0,0 +1,11 @@ +CREATE TABLE events ( + id UInt64, + name String, + tag Nullable(String), + amount Float64 +) ENGINE = MergeTree ORDER BY id; + +CREATE TABLE users ( + id UInt64, + email String +) ENGINE = MergeTree ORDER BY id; diff --git a/internal/engine/clickhouse/testgen/testdata/expressions/stdout.txt b/internal/engine/clickhouse/testgen/testdata/expressions/stdout.txt new file mode 100644 index 0000000000..a77fe1af43 --- /dev/null +++ b/internal/engine/clickhouse/testgen/testdata/expressions/stdout.txt @@ -0,0 +1,335 @@ +[ + { + "name": "Aggregates", + "cmd": ":one", + "columns": [ + { + "name": "n", + "data_type": "uint64", + "not_null": true, + "is_array": false + }, + { + "name": "top", + "data_type": "float64", + "not_null": true, + "is_array": false + }, + { + "name": "sum(amount)", + "data_type": "float64", + "not_null": true, + "is_array": false + }, + { + "name": "first_tag", + "data_type": "string", + "not_null": false, + "is_array": false + }, + { + "name": "uniq(name)", + "data_type": "uint64", + "not_null": true, + "is_array": false + } + ], + "params": [] + }, + { + "name": "Expressions", + "cmd": ":many", + "columns": [ + { + "name": "next_id", + "data_type": "uint64", + "not_null": true, + "is_array": false + }, + { + "name": "maybe", + "data_type": "string", + "not_null": false, + "is_array": false + }, + { + "name": "tag_or_none", + "data_type": "string", + "not_null": true, + "is_array": false + }, + { + "name": "lower(name)", + "data_type": "string", + "not_null": true, + "is_array": false + }, + { + "name": "today", + "data_type": "date", + "not_null": true, + "is_array": false + }, + { + "name": "big", + "data_type": "uint8", + "not_null": true, + "is_array": false + }, + { + "name": "word", + "data_type": "string", + "not_null": false, + "is_array": false + } + ], + "params": [] + }, + { + "name": "LeftJoinDefaults", + "cmd": ":many", + "columns": [ + { + "name": "id", + "data_type": "uint64", + "not_null": true, + "is_array": false, + "table": "events" + }, + { + "name": "name", + "data_type": "string", + "not_null": true, + "is_array": false, + "table": "events" + }, + { + "name": "email", + "data_type": "string", + "not_null": true, + "is_array": false, + "table": "users" + } + ], + "params": [] + }, + { + "name": "Positional", + "cmd": ":many", + "columns": [ + { + "name": "id", + "data_type": "uint64", + "not_null": true, + "is_array": false, + "table": "events" + }, + { + "name": "name", + "data_type": "string", + "not_null": true, + "is_array": false, + "table": "events" + } + ], + "params": [ + { + "number": 1, + "column": { + "name": "name", + "data_type": "string", + "not_null": true, + "is_array": false, + "table": "events" + } + }, + { + "number": 2, + "column": { + "name": "amount", + "data_type": "float64", + "not_null": true, + "is_array": false, + "table": "events" + } + }, + { + "number": 3, + "column": { + "name": "tag", + "data_type": "string", + "not_null": false, + "is_array": false, + "table": "events" + } + } + ] + }, + { + "name": "Named", + "cmd": ":many", + "columns": [ + { + "name": "id", + "data_type": "uint64", + "not_null": true, + "is_array": false, + "table": "events" + }, + { + "name": "name", + "data_type": "string", + "not_null": true, + "is_array": false, + "table": "events" + } + ], + "params": [ + { + "number": 1, + "column": { + "name": "name", + "data_type": "string", + "not_null": true, + "is_array": false, + "table": "events" + } + }, + { + "number": 2, + "column": { + "name": "tag", + "data_type": "string", + "not_null": false, + "is_array": false, + "table": "events" + } + }, + { + "number": 3, + "column": { + "name": "amount", + "data_type": "float64", + "not_null": true, + "is_array": false, + "table": "events" + } + }, + { + "number": 3, + "column": { + "name": "amount", + "data_type": "float64", + "not_null": true, + "is_array": false, + "table": "events" + } + } + ] + }, + { + "name": "Functions", + "cmd": ":many", + "columns": [ + { + "name": "id", + "data_type": "uint64", + "not_null": true, + "is_array": false, + "table": "events" + } + ], + "params": [ + { + "number": 1, + "column": { + "name": "lower", + "data_type": "string", + "not_null": true, + "is_array": false + } + }, + { + "number": 2, + "column": { + "name": "id", + "data_type": "uint64", + "not_null": true, + "is_array": false, + "table": "events" + } + }, + { + "number": 3, + "column": { + "name": "", + "data_type": "date", + "not_null": true, + "is_array": false + } + } + ] + }, + { + "name": "Paging", + "cmd": ":many", + "columns": [ + { + "name": "id", + "data_type": "uint64", + "not_null": true, + "is_array": false, + "table": "events" + } + ], + "params": [ + { + "number": 1, + "column": { + "name": "", + "data_type": "uint64", + "not_null": true, + "is_array": false + } + }, + { + "number": 2, + "column": { + "name": "", + "data_type": "uint64", + "not_null": true, + "is_array": false + } + } + ] + }, + { + "name": "Projected", + "cmd": ":one", + "columns": [ + { + "name": "echo", + "data_type": "nothing", + "not_null": false, + "is_array": false + }, + { + "name": "lit", + "data_type": "string", + "not_null": true, + "is_array": false + } + ], + "params": [ + { + "number": 1, + "column": { + "name": "echo", + "data_type": "nothing", + "not_null": false, + "is_array": false + } + } + ] + } +] diff --git a/internal/engine/clickhouse/testgen/testdata/subqueries/fixture.sql b/internal/engine/clickhouse/testgen/testdata/subqueries/fixture.sql new file mode 100644 index 0000000000..9adc5fa938 --- /dev/null +++ b/internal/engine/clickhouse/testgen/testdata/subqueries/fixture.sql @@ -0,0 +1,2 @@ +INSERT INTO events VALUES (1, 'signup', NULL, 9.5), (2, 'login', 'web', 0.5); +INSERT INTO users VALUES (1, 'a@example.com'); diff --git a/internal/engine/clickhouse/testgen/testdata/subqueries/query.sql b/internal/engine/clickhouse/testgen/testdata/subqueries/query.sql new file mode 100644 index 0000000000..297e082951 --- /dev/null +++ b/internal/engine/clickhouse/testgen/testdata/subqueries/query.sql @@ -0,0 +1,26 @@ +-- name: Cte :many +WITH t AS (SELECT id, tag FROM events) +SELECT t.id, t.tag, s.cnt +FROM t +JOIN (SELECT id, count() AS cnt FROM events GROUP BY id) s ON s.id = t.id; + +-- name: Aliased :many +SELECT s.x, s.total, s.email +FROM ( + SELECT e.id AS x, sum(e.amount) AS total, any(u.email) AS email + FROM events e JOIN users u ON u.id = e.id + GROUP BY e.id +) s; + +-- name: Star :many +SELECT * FROM users u JOIN events e ON e.id = u.id; + +-- name: Union :many +SELECT id, name FROM events +UNION ALL +SELECT id, email FROM users; + +-- name: ScalarSubquery :many +SELECT id, (SELECT count() FROM users) AS user_count +FROM events +WHERE id IN (SELECT id FROM users WHERE email = ?); diff --git a/internal/engine/clickhouse/testgen/testdata/subqueries/schema.sql b/internal/engine/clickhouse/testgen/testdata/subqueries/schema.sql new file mode 100644 index 0000000000..9c04e10685 --- /dev/null +++ b/internal/engine/clickhouse/testgen/testdata/subqueries/schema.sql @@ -0,0 +1,11 @@ +CREATE TABLE events ( + id UInt64, + name String, + tag Nullable(String), + amount Float64 +) ENGINE = MergeTree ORDER BY id; + +CREATE TABLE users ( + id UInt64, + email String +) ENGINE = MergeTree ORDER BY id; diff --git a/internal/engine/clickhouse/testgen/testdata/subqueries/stdout.txt b/internal/engine/clickhouse/testgen/testdata/subqueries/stdout.txt new file mode 100644 index 0000000000..df8dc0a2ad --- /dev/null +++ b/internal/engine/clickhouse/testgen/testdata/subqueries/stdout.txt @@ -0,0 +1,156 @@ +[ + { + "name": "Cte", + "cmd": ":many", + "columns": [ + { + "name": "id", + "data_type": "uint64", + "not_null": true, + "is_array": false, + "table": "events" + }, + { + "name": "tag", + "data_type": "string", + "not_null": false, + "is_array": false, + "table": "events" + }, + { + "name": "cnt", + "data_type": "uint64", + "not_null": true, + "is_array": false + } + ], + "params": [] + }, + { + "name": "Aliased", + "cmd": ":many", + "columns": [ + { + "name": "x", + "data_type": "uint64", + "not_null": true, + "is_array": false, + "table": "events" + }, + { + "name": "total", + "data_type": "float64", + "not_null": true, + "is_array": false + }, + { + "name": "email", + "data_type": "string", + "not_null": true, + "is_array": false + } + ], + "params": [] + }, + { + "name": "Star", + "cmd": ":many", + "columns": [ + { + "name": "id", + "data_type": "uint64", + "not_null": true, + "is_array": false, + "table": "users" + }, + { + "name": "email", + "data_type": "string", + "not_null": true, + "is_array": false, + "table": "users" + }, + { + "name": "e.id", + "data_type": "uint64", + "not_null": true, + "is_array": false, + "table": "events" + }, + { + "name": "name", + "data_type": "string", + "not_null": true, + "is_array": false, + "table": "events" + }, + { + "name": "tag", + "data_type": "string", + "not_null": false, + "is_array": false, + "table": "events" + }, + { + "name": "amount", + "data_type": "float64", + "not_null": true, + "is_array": false, + "table": "events" + } + ], + "params": [] + }, + { + "name": "Union", + "cmd": ":many", + "columns": [ + { + "name": "id", + "data_type": "uint64", + "not_null": true, + "is_array": false, + "table": "events" + }, + { + "name": "name", + "data_type": "string", + "not_null": true, + "is_array": false, + "table": "events" + } + ], + "params": [] + }, + { + "name": "ScalarSubquery", + "cmd": ":many", + "columns": [ + { + "name": "id", + "data_type": "uint64", + "not_null": true, + "is_array": false, + "table": "events" + }, + { + "name": "user_count", + "data_type": "uint64", + "not_null": false, + "is_array": false + } + ], + "params": [ + { + "number": 1, + "column": { + "name": "email", + "data_type": "string", + "not_null": true, + "is_array": false, + "table": "users" + } + } + ] + } +] diff --git a/internal/engine/clickhouse/testgen/testdata/types/fixture.sql b/internal/engine/clickhouse/testgen/testdata/types/fixture.sql new file mode 100644 index 0000000000..bb776593a6 --- /dev/null +++ b/internal/engine/clickhouse/testgen/testdata/types/fixture.sql @@ -0,0 +1 @@ +INSERT INTO things VALUES (1, 'a', NULL, 1.5, ['x'], [NULL, 'y'], [[1, 2]], 'k', '2024-01-01 00:00:00', '2024-01-01 00:00:00.123', 1.25, 'active', {'k': 1}, (1.0, 2.0), '127.0.0.1', '00000000-0000-0000-0000-000000000000', 'abcd', true); diff --git a/internal/engine/clickhouse/testgen/testdata/types/query.sql b/internal/engine/clickhouse/testgen/testdata/types/query.sql new file mode 100644 index 0000000000..229a09b285 --- /dev/null +++ b/internal/engine/clickhouse/testgen/testdata/types/query.sql @@ -0,0 +1,5 @@ +-- name: AllTypes :many +SELECT * FROM things; + +-- name: StarColumns :many +SELECT id, name, tag, amount, tags, labels, matrix, kind, created, updated, price, status, attrs, pos, ip, uid, fixed, flag FROM things; diff --git a/internal/engine/clickhouse/testgen/testdata/types/schema.sql b/internal/engine/clickhouse/testgen/testdata/types/schema.sql new file mode 100644 index 0000000000..3747769d2e --- /dev/null +++ b/internal/engine/clickhouse/testgen/testdata/types/schema.sql @@ -0,0 +1,20 @@ +CREATE TABLE things ( + id UInt64, + name String, + tag Nullable(String), + amount Float64, + tags Array(String), + labels Array(Nullable(String)), + matrix Array(Array(UInt8)), + kind LowCardinality(Nullable(String)), + created DateTime, + updated DateTime64(3, 'UTC'), + price Decimal(10, 2), + status Enum8('active' = 1, 'deleted' = 2), + attrs Map(String, UInt32), + pos Tuple(Float64, Float64), + ip IPv4, + uid UUID, + fixed FixedString(4), + flag Bool +) ENGINE = MergeTree ORDER BY id; diff --git a/internal/engine/clickhouse/testgen/testdata/types/stdout.txt b/internal/engine/clickhouse/testgen/testdata/types/stdout.txt new file mode 100644 index 0000000000..041b05363b --- /dev/null +++ b/internal/engine/clickhouse/testgen/testdata/types/stdout.txt @@ -0,0 +1,268 @@ +[ + { + "name": "AllTypes", + "cmd": ":many", + "columns": [ + { + "name": "id", + "data_type": "uint64", + "not_null": true, + "is_array": false, + "table": "things" + }, + { + "name": "name", + "data_type": "string", + "not_null": true, + "is_array": false, + "table": "things" + }, + { + "name": "tag", + "data_type": "string", + "not_null": false, + "is_array": false, + "table": "things" + }, + { + "name": "amount", + "data_type": "float64", + "not_null": true, + "is_array": false, + "table": "things" + }, + { + "name": "tags", + "data_type": "string", + "not_null": true, + "is_array": true, + "table": "things" + }, + { + "name": "labels", + "data_type": "string", + "not_null": false, + "is_array": true, + "table": "things" + }, + { + "name": "matrix", + "data_type": "uint8", + "not_null": true, + "is_array": true, + "table": "things" + }, + { + "name": "kind", + "data_type": "string", + "not_null": false, + "is_array": false, + "table": "things" + }, + { + "name": "created", + "data_type": "datetime", + "not_null": true, + "is_array": false, + "table": "things" + }, + { + "name": "updated", + "data_type": "datetime64", + "not_null": true, + "is_array": false, + "table": "things" + }, + { + "name": "price", + "data_type": "decimal", + "not_null": true, + "is_array": false, + "table": "things" + }, + { + "name": "status", + "data_type": "enum8", + "not_null": true, + "is_array": false, + "table": "things" + }, + { + "name": "attrs", + "data_type": "map", + "not_null": true, + "is_array": false, + "table": "things" + }, + { + "name": "pos", + "data_type": "tuple", + "not_null": true, + "is_array": false, + "table": "things" + }, + { + "name": "ip", + "data_type": "ipv4", + "not_null": true, + "is_array": false, + "table": "things" + }, + { + "name": "uid", + "data_type": "uuid", + "not_null": true, + "is_array": false, + "table": "things" + }, + { + "name": "fixed", + "data_type": "fixedstring", + "not_null": true, + "is_array": false, + "table": "things" + }, + { + "name": "flag", + "data_type": "bool", + "not_null": true, + "is_array": false, + "table": "things" + } + ], + "params": [] + }, + { + "name": "StarColumns", + "cmd": ":many", + "columns": [ + { + "name": "id", + "data_type": "uint64", + "not_null": true, + "is_array": false, + "table": "things" + }, + { + "name": "name", + "data_type": "string", + "not_null": true, + "is_array": false, + "table": "things" + }, + { + "name": "tag", + "data_type": "string", + "not_null": false, + "is_array": false, + "table": "things" + }, + { + "name": "amount", + "data_type": "float64", + "not_null": true, + "is_array": false, + "table": "things" + }, + { + "name": "tags", + "data_type": "string", + "not_null": true, + "is_array": true, + "table": "things" + }, + { + "name": "labels", + "data_type": "string", + "not_null": false, + "is_array": true, + "table": "things" + }, + { + "name": "matrix", + "data_type": "uint8", + "not_null": true, + "is_array": true, + "table": "things" + }, + { + "name": "kind", + "data_type": "string", + "not_null": false, + "is_array": false, + "table": "things" + }, + { + "name": "created", + "data_type": "datetime", + "not_null": true, + "is_array": false, + "table": "things" + }, + { + "name": "updated", + "data_type": "datetime64", + "not_null": true, + "is_array": false, + "table": "things" + }, + { + "name": "price", + "data_type": "decimal", + "not_null": true, + "is_array": false, + "table": "things" + }, + { + "name": "status", + "data_type": "enum8", + "not_null": true, + "is_array": false, + "table": "things" + }, + { + "name": "attrs", + "data_type": "map", + "not_null": true, + "is_array": false, + "table": "things" + }, + { + "name": "pos", + "data_type": "tuple", + "not_null": true, + "is_array": false, + "table": "things" + }, + { + "name": "ip", + "data_type": "ipv4", + "not_null": true, + "is_array": false, + "table": "things" + }, + { + "name": "uid", + "data_type": "uuid", + "not_null": true, + "is_array": false, + "table": "things" + }, + { + "name": "fixed", + "data_type": "fixedstring", + "not_null": true, + "is_array": false, + "table": "things" + }, + { + "name": "flag", + "data_type": "bool", + "not_null": true, + "is_array": false, + "table": "things" + } + ], + "params": [] + } +] diff --git a/internal/engine/clickhouse/testgen/testgen_test.go b/internal/engine/clickhouse/testgen/testgen_test.go new file mode 100644 index 0000000000..0cca493058 --- /dev/null +++ b/internal/engine/clickhouse/testgen/testgen_test.go @@ -0,0 +1,55 @@ +package main + +import ( + "bytes" + "context" + "flag" + "os" + "path/filepath" + "testing" +) + +var update = flag.Bool("update", false, "rewrite the expected stdout.txt of every case") + +// TestAnalyze runs the CLI over each directory under testdata, which holds +// the same files a sqlc analyze case does plus a fixture, and compares the +// output with the committed stdout.txt. It needs the clickhouse binary and +// skips when none is installed. +func TestAnalyze(t *testing.T) { + if _, err := Locate(); err != nil { + t.Skip(err) + } + dirs, err := filepath.Glob("testdata/*") + if err != nil { + t.Fatal(err) + } + for _, dir := range dirs { + t.Run(filepath.Base(dir), func(t *testing.T) { + args := []string{"analyze", "--schema", filepath.Join(dir, "schema.sql")} + if _, err := os.Stat(filepath.Join(dir, "fixture.sql")); err == nil { + args = append(args, "--fixture", filepath.Join(dir, "fixture.sql")) + } + args = append(args, filepath.Join(dir, "query.sql")) + + var stdout, stderr bytes.Buffer + if err := run(context.Background(), args, &stdout, &stderr); err != nil { + t.Fatalf("%v\n%s", err, stderr.String()) + } + + golden := filepath.Join(dir, "stdout.txt") + if *update { + if err := os.WriteFile(golden, stdout.Bytes(), 0o644); err != nil { + t.Fatal(err) + } + return + } + want, err := os.ReadFile(golden) + if err != nil { + t.Fatal(err) + } + if !bytes.Equal(want, stdout.Bytes()) { + t.Errorf("output differs from %s (run with -update to rewrite)\n--- want\n%s\n--- got\n%s", golden, want, stdout.Bytes()) + } + }) + } +} diff --git a/internal/engine/clickhouse/testgen/tree.go b/internal/engine/clickhouse/testgen/tree.go new file mode 100644 index 0000000000..9ff0d329fc --- /dev/null +++ b/internal/engine/clickhouse/testgen/tree.go @@ -0,0 +1,236 @@ +package main + +import ( + "fmt" + "regexp" + "strconv" + "strings" +) + +// treeNode is one line of EXPLAIN QUERY TREE output. Lines of the form +// `KIND id: N, key: value, ...` are nodes with a kind and attributes; every +// other line (section headers such as PROJECTION or ARGUMENTS, and the +// `name Type` lines under PROJECTION COLUMNS) keeps only its text. +type treeNode struct { + kind string + id int + attrs map[string]string + text string + parent *treeNode + children []*treeNode +} + +// queryTree is a parsed EXPLAIN QUERY TREE dump. +type queryTree struct { + root *treeNode + byID map[int]*treeNode +} + +var nodeLineRe = regexp.MustCompile(`^([A-Z_]+) id: (\d+)(?:, (.*))?$`) + +// parseQueryTree builds the tree from the dump's lines, using the two-space +// indentation to recover nesting. +func parseQueryTree(lines []string) (*queryTree, error) { + root := &treeNode{id: -1, text: ""} + t := &queryTree{root: root, byID: map[int]*treeNode{}} + stack := []*treeNode{root} + for _, line := range lines { + if strings.TrimSpace(line) == "" { + continue + } + indent := len(line) - len(strings.TrimLeft(line, " ")) + if indent%2 != 0 { + return nil, fmt.Errorf("query tree: unexpected indentation in %q", line) + } + depth := indent/2 + 1 + if depth > len(stack) { + return nil, fmt.Errorf("query tree: line %q is nested too deeply", line) + } + stack = stack[:depth] + parent := stack[len(stack)-1] + + n := &treeNode{id: -1, text: line[indent:], parent: parent} + if m := nodeLineRe.FindStringSubmatch(n.text); m != nil { + n.kind = m[1] + n.id, _ = strconv.Atoi(m[2]) + n.attrs = parseAttrs(m[3]) + t.byID[n.id] = n + } + parent.children = append(parent.children, n) + stack = append(stack, n) + } + return t, nil +} + +var attrKeyRe = regexp.MustCompile(`^, ([a-z_]+): `) + +// parseAttrs splits `key: value, key: value` where values may themselves +// contain commas inside parentheses or quotes, as types and constants do. +func parseAttrs(s string) map[string]string { + attrs := map[string]string{} + if s == "" { + return attrs + } + // Positions at which a new `, key: ` begins at top level. + var cuts []int + depth := 0 + inQuote := false + for i := 0; i < len(s); i++ { + switch c := s[i]; { + case c == '\'': + inQuote = !inQuote + case inQuote: + case c == '(': + depth++ + case c == ')': + depth-- + case c == ',' && depth == 0 && attrKeyRe.MatchString(s[i:]): + cuts = append(cuts, i) + } + } + cuts = append(cuts, len(s)) + start := 0 + for _, cut := range cuts { + pair := s[start:cut] + if key, value, ok := strings.Cut(pair, ": "); ok { + attrs[key] = value + } + start = cut + 2 + } + return attrs +} + +// section returns the child section header of a node, such as PROJECTION +// or JOIN TREE, or nil. +func (n *treeNode) section(name string) *treeNode { + for _, c := range n.children { + if c.kind == "" && c.text == name { + return c + } + } + return nil +} + +// list returns the nodes of the LIST under a section header. +func (n *treeNode) list() []*treeNode { + if n == nil { + return nil + } + for _, c := range n.children { + if c.kind == "LIST" { + return c.children + } + } + return nil +} + +// firstQuery returns the QUERY node describing a result set: the node itself +// or, for a UNION, its first branch, whose projection names the union's +// columns. +func firstQuery(n *treeNode) *treeNode { + for depth := 0; n != nil && depth < 32; depth++ { + switch n.kind { + case "QUERY": + return n + case "UNION": + n = firstNode(n.section("QUERIES").list()) + default: + return nil + } + } + return nil +} + +func firstNode(nodes []*treeNode) *treeNode { + if len(nodes) == 0 { + return nil + } + return nodes[0] +} + +// projection returns the output column names of a query and the expression +// node producing each, in order. +func projection(q *treeNode) (names []string, nodes []*treeNode) { + q = firstQuery(q) + if q == nil { + return nil, nil + } + if cols := q.section("PROJECTION COLUMNS"); cols != nil { + for _, c := range cols.children { + names = append(names, projectedName(c.text)) + } + } + return names, q.section("PROJECTION").list() +} + +// projectedName strips the trailing type from a `name Type` line. The type +// is a single token that may carry a parenthesised argument list; the name +// may contain spaces of its own, as `plus(id, 1)` does. +func projectedName(line string) string { + end := len(line) + if strings.HasSuffix(line, ")") { + depth := 0 + for end > 0 { + end-- + if line[end] == ')' { + depth++ + } else if line[end] == '(' { + depth-- + if depth == 0 { + break + } + } + } + } + for end > 0 && isWordByte(line[end-1]) { + end-- + } + return strings.TrimSpace(line[:end]) +} + +// sourceTable resolves a COLUMN node to the table it reads from, following +// column references through subqueries and CTEs. Columns computed by an +// expression have no source and yield "". +func (t *queryTree) sourceTable(col *treeNode) string { + for depth := 0; col != nil && col.kind == "COLUMN" && depth < 32; depth++ { + id, err := strconv.Atoi(col.attrs["source_id"]) + if err != nil { + return "" + } + src := t.byID[id] + if src == nil { + return "" + } + switch src.kind { + case "TABLE": + name := src.attrs["table_name"] + if i := strings.LastIndexByte(name, '.'); i >= 0 { + name = name[i+1:] + } + return name + case "QUERY", "UNION": + // The column is one of the subquery's output columns; keep + // following whatever expression produces it there. + want := col.attrs["column_name"] + names, nodes := projection(src) + col = nil + for i, name := range names { + if name == want && i < len(nodes) { + col = nodes[i] + break + } + } + default: + return "" + } + } + return "" +} + +// childrenOrNil returns a node's children, tolerating a nil node. +func (n *treeNode) childrenOrNil() []*treeNode { + if n == nil { + return nil + } + return n.children +} diff --git a/internal/engine/clickhouse/testgen/types.go b/internal/engine/clickhouse/testgen/types.go new file mode 100644 index 0000000000..5a9c2907fc --- /dev/null +++ b/internal/engine/clickhouse/testgen/types.go @@ -0,0 +1,80 @@ +package main + +import "strings" + +// sqlcType lowers a ClickHouse type name to the shape sqlc's ClickHouse +// engine reports: the lowercased base name with parameters dropped, +// Nullable and Array wrappers folded into flags and LowCardinality +// discarded. It mirrors unwrapTypeString in the engine so goldens generated +// here diff cleanly against `sqlc analyze`. +func sqlcType(t string) (dataType string, isArray, notNull bool) { + name, arr, nullable := unwrapType(t) + return name, arr, !nullable +} + +func unwrapType(t string) (name string, isArray, nullable bool) { + base, args := splitType(t) + switch strings.ToLower(base) { + case "nullable": + if len(args) == 1 { + inner, arr, _ := unwrapType(args[0]) + return inner, arr, true + } + return "nullable", false, true + case "lowcardinality": + if len(args) == 1 { + return unwrapType(args[0]) + } + return "lowcardinality", false, false + case "array": + if len(args) == 1 { + inner, _, nul := unwrapType(args[0]) + return inner, true, nul + } + return "array", true, false + case "": + return "nothing", false, false + default: + return strings.ToLower(base), false, false + } +} + +// splitType splits `Base(arg, arg)` into its base name and top-level +// arguments, leaving nested parentheses and quoted strings intact. +func splitType(t string) (string, []string) { + t = strings.TrimSpace(t) + open := strings.IndexByte(t, '(') + if open < 0 || !strings.HasSuffix(t, ")") { + return t, nil + } + base := strings.TrimSpace(t[:open]) + inner := t[open+1 : len(t)-1] + var ( + args []string + depth int + quote byte + start int + ) + for i := 0; i < len(inner); i++ { + c := inner[i] + switch { + case quote != 0: + if c == '\\' { + i++ + } else if c == quote { + quote = 0 + } + case c == '\'' || c == '"' || c == '`': + quote = c + case c == '(': + depth++ + case c == ')': + depth-- + case c == ',' && depth == 0: + args = append(args, strings.TrimSpace(inner[start:i])) + start = i + 1 + } + } + args = append(args, strings.TrimSpace(inner[start:])) + return base, args +} From 94078a023be06cb616586d3f4baeae3e7abea7a0 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 2 Sep 2026 19:11:30 +0000 Subject: [PATCH 02/10] Write testgen types as call expressions Replace the data_type and is_array fields with one type expression per column: a lowercased name applied to arguments that are numbers, quoted strings, identifiers, other calls, or any of those with a label. Nullable, Array and LowCardinality are ordinary names in that grammar, so nested types such as Array(Nullable(String)), Map(String, Nullable(UInt8)) and Tuple(lat Float64, lon Float64) survive intact. An outer Nullable is lifted into the column's not_null flag; deeper ones stay in the expression. Resolving the names is left to the reader of the output. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01Y6XkyWnx7iJFEb8q3AnYps --- CLAUDE.md | 4 +- internal/engine/clickhouse/testgen/README.md | 25 ++- internal/engine/clickhouse/testgen/analyze.go | 17 +- internal/engine/clickhouse/testgen/main.go | 4 +- .../testdata/analyze_params/stdout.txt | 21 +-- .../testgen/testdata/exec/stdout.txt | 30 ++-- .../testgen/testdata/expressions/stdout.txt | 146 +++++++----------- .../testgen/testdata/subqueries/stdout.txt | 59 +++---- .../testgen/testdata/types/fixture.sql | 2 +- .../testgen/testdata/types/query.sql | 2 +- .../testgen/testdata/types/schema.sql | 2 + .../testgen/testdata/types/stdout.txt | 136 ++++++++-------- internal/engine/clickhouse/testgen/types.go | 115 ++++++++++---- 13 files changed, 280 insertions(+), 283 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index 3eecfb0f04..6ac48badce 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -234,8 +234,8 @@ MYSQL_SERVER_URI="root:mysecretpassword@tcp(127.0.0.1:3306)/mysql?multiStatement - `/dolphin/` - MySQL parser (uses TiDB parser) - `/sqlite/` - SQLite parser - `/clickhouse/testgen/` - Nested module that records what a real - ClickHouse reports about a schema, fixture and queries, in the `sqlc - analyze` output format; see its README + ClickHouse reports about a schema, fixture and queries, in the shape of + `sqlc analyze` output with types as call expressions; see its README - `/duckdb/` - DuckDB 2.0 parser (uses darkwing, the pure Go port of DuckDB's PEG parser) - `/dialect/` - The engine's type system and standard library, as diff --git a/internal/engine/clickhouse/testgen/README.md b/internal/engine/clickhouse/testgen/README.md index 6f98a4c734..0314236765 100644 --- a/internal/engine/clickhouse/testgen/README.md +++ b/internal/engine/clickhouse/testgen/README.md @@ -1,7 +1,7 @@ # ClickHouse testgen `testgen` records what ClickHouse itself reports about a set of sqlc queries, -so the answer can be committed as a golden file and compared with what sqlc's +so the answer can be committed as a golden file and held against what sqlc's own analysis produces for the same schema and queries. It is a nested Go module with no dependencies beyond the standard library. @@ -39,10 +39,25 @@ query is explained and then executed, and the process exits. operand it is compared with. Parameters of `INSERT ... VALUES` map onto the target columns reported by `DESCRIBE TABLE`. -The output has the same shape as `sqlc analyze`, with ClickHouse types lowered -the way sqlc's ClickHouse engine lowers them: the lowercased base name with -parameters dropped, `Nullable` and `Array` folded into `not_null` and -`is_array`, and `LowCardinality` discarded. +The output has the shape of `sqlc analyze`, except that each column's type is +one expression rather than a name and flags. A type is written as a call: a +lowercased name applied to arguments that are numbers, quoted strings, +identifiers, other calls, or any of those with a label. `Nullable`, `Array` +and `LowCardinality` are ordinary names in that grammar, so nothing about +nesting is lost: + +| ClickHouse | testgen | +|-----------------------------------|------------------------------------| +| `Array(Nullable(String))` | `array(nullable(string))` | +| `Map(String, UInt32)` | `map(string, uint32)` | +| `Tuple(lat Float64, lon Float64)` | `tuple(lat: float64, lon: float64)`| +| `Enum8('a' = 1, 'b' = 2)` | `enum8('a': 1, 'b': 2)` | +| `DateTime64(3, 'UTC')` | `datetime64(3, 'UTC')` | +| `LowCardinality(Nullable(String))`| `lowcardinality(string)`, `not_null: false` | + +An outer `Nullable` is the column's nullability rather than part of its type, +so it is lifted into `not_null`; a `Nullable` at any greater depth stays in +the expression. Resolving the names is left to whoever reads the output. ## Tests diff --git a/internal/engine/clickhouse/testgen/analyze.go b/internal/engine/clickhouse/testgen/analyze.go index 4b2b7cd85e..b2b446702e 100644 --- a/internal/engine/clickhouse/testgen/analyze.go +++ b/internal/engine/clickhouse/testgen/analyze.go @@ -9,8 +9,8 @@ import ( "strings" ) -// The output mirrors the JSON `sqlc analyze` prints so a golden generated -// here can be diffed against sqlc's own analysis of the same files. +// The output follows the JSON `sqlc analyze` prints, with each type written +// as a call expression (see types.go) instead of a name and flags. type analyzedQuery struct { Name string `json:"name"` @@ -20,11 +20,10 @@ type analyzedQuery struct { } type analyzedColumn struct { - Name string `json:"name"` - DataType string `json:"data_type"` - NotNull bool `json:"not_null"` - IsArray bool `json:"is_array"` - Table string `json:"table,omitempty"` + Name string `json:"name"` + Type string `json:"type"` + NotNull bool `json:"not_null"` + Table string `json:"table,omitempty"` } type analyzedParam struct { @@ -119,8 +118,8 @@ func analyzeQuery(ctx context.Context, l local, schema, fixture string, q query) } func column(name, typ string) analyzedColumn { - dataType, isArray, notNull := sqlcType(typ) - return analyzedColumn{Name: name, DataType: dataType, NotNull: notNull, IsArray: isArray} + expr, notNull := typeExpr(typ) + return analyzedColumn{Name: name, Type: expr, NotNull: notNull} } // returnsRows reports whether a statement produces a result set and so can diff --git a/internal/engine/clickhouse/testgen/main.go b/internal/engine/clickhouse/testgen/main.go index 57d8cb82f3..d798a65cc9 100644 --- a/internal/engine/clickhouse/testgen/main.go +++ b/internal/engine/clickhouse/testgen/main.go @@ -5,8 +5,8 @@ // It loads a schema and a fixture into an ephemeral `clickhouse local` // process, runs each query found in a sqlc query file against that data, and // prints the result column types, nullability and source tables along with -// the parameters each query binds, in the same JSON shape `sqlc analyze` -// prints. +// the parameters each query binds, in the JSON shape `sqlc analyze` prints +// with each type written as a call expression. // // The clickhouse binary is downloaded once per pinned version with // `testgen install`, or supplied through the CLICKHOUSE environment variable. diff --git a/internal/engine/clickhouse/testgen/testdata/analyze_params/stdout.txt b/internal/engine/clickhouse/testgen/testdata/analyze_params/stdout.txt index 6fb0cc654b..3c1408ba71 100644 --- a/internal/engine/clickhouse/testgen/testdata/analyze_params/stdout.txt +++ b/internal/engine/clickhouse/testgen/testdata/analyze_params/stdout.txt @@ -5,16 +5,14 @@ "columns": [ { "name": "id", - "data_type": "uint64", + "type": "uint64", "not_null": true, - "is_array": false, "table": "events" }, { "name": "name", - "data_type": "string", + "type": "string", "not_null": true, - "is_array": false, "table": "events" } ], @@ -23,9 +21,8 @@ "number": 1, "column": { "name": "id", - "data_type": "uint64", + "type": "uint64", "not_null": true, - "is_array": false, "table": "events" } } @@ -37,16 +34,14 @@ "columns": [ { "name": "id", - "data_type": "uint64", + "type": "uint64", "not_null": true, - "is_array": false, "table": "events" }, { "name": "name", - "data_type": "string", + "type": "string", "not_null": true, - "is_array": false, "table": "events" } ], @@ -55,9 +50,8 @@ "number": 1, "column": { "name": "name", - "data_type": "string", + "type": "string", "not_null": true, - "is_array": false, "table": "events" } }, @@ -65,9 +59,8 @@ "number": 2, "column": { "name": "amount", - "data_type": "float64", + "type": "float64", "not_null": true, - "is_array": false, "table": "events" } } diff --git a/internal/engine/clickhouse/testgen/testdata/exec/stdout.txt b/internal/engine/clickhouse/testgen/testdata/exec/stdout.txt index bc9ab7f8d6..aa0b3e3f7b 100644 --- a/internal/engine/clickhouse/testgen/testdata/exec/stdout.txt +++ b/internal/engine/clickhouse/testgen/testdata/exec/stdout.txt @@ -8,9 +8,8 @@ "number": 1, "column": { "name": "id", - "data_type": "uint64", + "type": "uint64", "not_null": true, - "is_array": false, "table": "users" } }, @@ -18,9 +17,8 @@ "number": 2, "column": { "name": "email", - "data_type": "string", + "type": "string", "not_null": true, - "is_array": false, "table": "users" } } @@ -35,9 +33,8 @@ "number": 1, "column": { "name": "id", - "data_type": "uint64", + "type": "uint64", "not_null": true, - "is_array": false, "table": "events" } }, @@ -45,9 +42,8 @@ "number": 2, "column": { "name": "name", - "data_type": "string", + "type": "string", "not_null": true, - "is_array": false, "table": "events" } }, @@ -55,9 +51,8 @@ "number": 3, "column": { "name": "tag", - "data_type": "string", + "type": "string", "not_null": false, - "is_array": false, "table": "events" } }, @@ -65,9 +60,8 @@ "number": 4, "column": { "name": "amount", - "data_type": "float64", + "type": "float64", "not_null": true, - "is_array": false, "table": "events" } } @@ -82,9 +76,8 @@ "number": 1, "column": { "name": "id", - "data_type": "uint64", + "type": "uint64", "not_null": true, - "is_array": false, "table": "events" } }, @@ -92,9 +85,8 @@ "number": 2, "column": { "name": "name", - "data_type": "string", + "type": "string", "not_null": true, - "is_array": false, "table": "events" } }, @@ -102,9 +94,8 @@ "number": 3, "column": { "name": "id", - "data_type": "uint64", + "type": "uint64", "not_null": true, - "is_array": false, "table": "events" } }, @@ -112,9 +103,8 @@ "number": 4, "column": { "name": "name", - "data_type": "string", + "type": "string", "not_null": true, - "is_array": false, "table": "events" } } diff --git a/internal/engine/clickhouse/testgen/testdata/expressions/stdout.txt b/internal/engine/clickhouse/testgen/testdata/expressions/stdout.txt index a77fe1af43..b3cd5f4690 100644 --- a/internal/engine/clickhouse/testgen/testdata/expressions/stdout.txt +++ b/internal/engine/clickhouse/testgen/testdata/expressions/stdout.txt @@ -5,33 +5,28 @@ "columns": [ { "name": "n", - "data_type": "uint64", - "not_null": true, - "is_array": false + "type": "uint64", + "not_null": true }, { "name": "top", - "data_type": "float64", - "not_null": true, - "is_array": false + "type": "float64", + "not_null": true }, { "name": "sum(amount)", - "data_type": "float64", - "not_null": true, - "is_array": false + "type": "float64", + "not_null": true }, { "name": "first_tag", - "data_type": "string", - "not_null": false, - "is_array": false + "type": "string", + "not_null": false }, { "name": "uniq(name)", - "data_type": "uint64", - "not_null": true, - "is_array": false + "type": "uint64", + "not_null": true } ], "params": [] @@ -42,45 +37,38 @@ "columns": [ { "name": "next_id", - "data_type": "uint64", - "not_null": true, - "is_array": false + "type": "uint64", + "not_null": true }, { "name": "maybe", - "data_type": "string", - "not_null": false, - "is_array": false + "type": "string", + "not_null": false }, { "name": "tag_or_none", - "data_type": "string", - "not_null": true, - "is_array": false + "type": "string", + "not_null": true }, { "name": "lower(name)", - "data_type": "string", - "not_null": true, - "is_array": false + "type": "string", + "not_null": true }, { "name": "today", - "data_type": "date", - "not_null": true, - "is_array": false + "type": "date", + "not_null": true }, { "name": "big", - "data_type": "uint8", - "not_null": true, - "is_array": false + "type": "uint8", + "not_null": true }, { "name": "word", - "data_type": "string", - "not_null": false, - "is_array": false + "type": "string", + "not_null": false } ], "params": [] @@ -91,23 +79,20 @@ "columns": [ { "name": "id", - "data_type": "uint64", + "type": "uint64", "not_null": true, - "is_array": false, "table": "events" }, { "name": "name", - "data_type": "string", + "type": "string", "not_null": true, - "is_array": false, "table": "events" }, { "name": "email", - "data_type": "string", + "type": "string", "not_null": true, - "is_array": false, "table": "users" } ], @@ -119,16 +104,14 @@ "columns": [ { "name": "id", - "data_type": "uint64", + "type": "uint64", "not_null": true, - "is_array": false, "table": "events" }, { "name": "name", - "data_type": "string", + "type": "string", "not_null": true, - "is_array": false, "table": "events" } ], @@ -137,9 +120,8 @@ "number": 1, "column": { "name": "name", - "data_type": "string", + "type": "string", "not_null": true, - "is_array": false, "table": "events" } }, @@ -147,9 +129,8 @@ "number": 2, "column": { "name": "amount", - "data_type": "float64", + "type": "float64", "not_null": true, - "is_array": false, "table": "events" } }, @@ -157,9 +138,8 @@ "number": 3, "column": { "name": "tag", - "data_type": "string", + "type": "string", "not_null": false, - "is_array": false, "table": "events" } } @@ -171,16 +151,14 @@ "columns": [ { "name": "id", - "data_type": "uint64", + "type": "uint64", "not_null": true, - "is_array": false, "table": "events" }, { "name": "name", - "data_type": "string", + "type": "string", "not_null": true, - "is_array": false, "table": "events" } ], @@ -189,9 +167,8 @@ "number": 1, "column": { "name": "name", - "data_type": "string", + "type": "string", "not_null": true, - "is_array": false, "table": "events" } }, @@ -199,9 +176,8 @@ "number": 2, "column": { "name": "tag", - "data_type": "string", + "type": "string", "not_null": false, - "is_array": false, "table": "events" } }, @@ -209,9 +185,8 @@ "number": 3, "column": { "name": "amount", - "data_type": "float64", + "type": "float64", "not_null": true, - "is_array": false, "table": "events" } }, @@ -219,9 +194,8 @@ "number": 3, "column": { "name": "amount", - "data_type": "float64", + "type": "float64", "not_null": true, - "is_array": false, "table": "events" } } @@ -233,9 +207,8 @@ "columns": [ { "name": "id", - "data_type": "uint64", + "type": "uint64", "not_null": true, - "is_array": false, "table": "events" } ], @@ -244,18 +217,16 @@ "number": 1, "column": { "name": "lower", - "data_type": "string", - "not_null": true, - "is_array": false + "type": "string", + "not_null": true } }, { "number": 2, "column": { "name": "id", - "data_type": "uint64", + "type": "uint64", "not_null": true, - "is_array": false, "table": "events" } }, @@ -263,9 +234,8 @@ "number": 3, "column": { "name": "", - "data_type": "date", - "not_null": true, - "is_array": false + "type": "date", + "not_null": true } } ] @@ -276,9 +246,8 @@ "columns": [ { "name": "id", - "data_type": "uint64", + "type": "uint64", "not_null": true, - "is_array": false, "table": "events" } ], @@ -287,18 +256,16 @@ "number": 1, "column": { "name": "", - "data_type": "uint64", - "not_null": true, - "is_array": false + "type": "uint64", + "not_null": true } }, { "number": 2, "column": { "name": "", - "data_type": "uint64", - "not_null": true, - "is_array": false + "type": "uint64", + "not_null": true } } ] @@ -309,15 +276,13 @@ "columns": [ { "name": "echo", - "data_type": "nothing", - "not_null": false, - "is_array": false + "type": "nothing", + "not_null": false }, { "name": "lit", - "data_type": "string", - "not_null": true, - "is_array": false + "type": "string", + "not_null": true } ], "params": [ @@ -325,9 +290,8 @@ "number": 1, "column": { "name": "echo", - "data_type": "nothing", - "not_null": false, - "is_array": false + "type": "nothing", + "not_null": false } } ] diff --git a/internal/engine/clickhouse/testgen/testdata/subqueries/stdout.txt b/internal/engine/clickhouse/testgen/testdata/subqueries/stdout.txt index df8dc0a2ad..577db1337c 100644 --- a/internal/engine/clickhouse/testgen/testdata/subqueries/stdout.txt +++ b/internal/engine/clickhouse/testgen/testdata/subqueries/stdout.txt @@ -5,23 +5,20 @@ "columns": [ { "name": "id", - "data_type": "uint64", + "type": "uint64", "not_null": true, - "is_array": false, "table": "events" }, { "name": "tag", - "data_type": "string", + "type": "string", "not_null": false, - "is_array": false, "table": "events" }, { "name": "cnt", - "data_type": "uint64", - "not_null": true, - "is_array": false + "type": "uint64", + "not_null": true } ], "params": [] @@ -32,22 +29,19 @@ "columns": [ { "name": "x", - "data_type": "uint64", + "type": "uint64", "not_null": true, - "is_array": false, "table": "events" }, { "name": "total", - "data_type": "float64", - "not_null": true, - "is_array": false + "type": "float64", + "not_null": true }, { "name": "email", - "data_type": "string", - "not_null": true, - "is_array": false + "type": "string", + "not_null": true } ], "params": [] @@ -58,44 +52,38 @@ "columns": [ { "name": "id", - "data_type": "uint64", + "type": "uint64", "not_null": true, - "is_array": false, "table": "users" }, { "name": "email", - "data_type": "string", + "type": "string", "not_null": true, - "is_array": false, "table": "users" }, { "name": "e.id", - "data_type": "uint64", + "type": "uint64", "not_null": true, - "is_array": false, "table": "events" }, { "name": "name", - "data_type": "string", + "type": "string", "not_null": true, - "is_array": false, "table": "events" }, { "name": "tag", - "data_type": "string", + "type": "string", "not_null": false, - "is_array": false, "table": "events" }, { "name": "amount", - "data_type": "float64", + "type": "float64", "not_null": true, - "is_array": false, "table": "events" } ], @@ -107,16 +95,14 @@ "columns": [ { "name": "id", - "data_type": "uint64", + "type": "uint64", "not_null": true, - "is_array": false, "table": "events" }, { "name": "name", - "data_type": "string", + "type": "string", "not_null": true, - "is_array": false, "table": "events" } ], @@ -128,16 +114,14 @@ "columns": [ { "name": "id", - "data_type": "uint64", + "type": "uint64", "not_null": true, - "is_array": false, "table": "events" }, { "name": "user_count", - "data_type": "uint64", - "not_null": false, - "is_array": false + "type": "uint64", + "not_null": false } ], "params": [ @@ -145,9 +129,8 @@ "number": 1, "column": { "name": "email", - "data_type": "string", + "type": "string", "not_null": true, - "is_array": false, "table": "users" } } diff --git a/internal/engine/clickhouse/testgen/testdata/types/fixture.sql b/internal/engine/clickhouse/testgen/testdata/types/fixture.sql index bb776593a6..bb958af8a2 100644 --- a/internal/engine/clickhouse/testgen/testdata/types/fixture.sql +++ b/internal/engine/clickhouse/testgen/testdata/types/fixture.sql @@ -1 +1 @@ -INSERT INTO things VALUES (1, 'a', NULL, 1.5, ['x'], [NULL, 'y'], [[1, 2]], 'k', '2024-01-01 00:00:00', '2024-01-01 00:00:00.123', 1.25, 'active', {'k': 1}, (1.0, 2.0), '127.0.0.1', '00000000-0000-0000-0000-000000000000', 'abcd', true); +INSERT INTO things VALUES (1, 'a', NULL, 1.5, ['x'], [NULL, 'y'], [[1, 2]], 'k', '2024-01-01 00:00:00', '2024-01-01 00:00:00.123', 1.25, 'active', {'k': 1}, (1.0, 2.0), (51.5, -0.1), {'a': NULL}, '127.0.0.1', '00000000-0000-0000-0000-000000000000', 'abcd', true); diff --git a/internal/engine/clickhouse/testgen/testdata/types/query.sql b/internal/engine/clickhouse/testgen/testdata/types/query.sql index 229a09b285..c55355e50d 100644 --- a/internal/engine/clickhouse/testgen/testdata/types/query.sql +++ b/internal/engine/clickhouse/testgen/testdata/types/query.sql @@ -2,4 +2,4 @@ SELECT * FROM things; -- name: StarColumns :many -SELECT id, name, tag, amount, tags, labels, matrix, kind, created, updated, price, status, attrs, pos, ip, uid, fixed, flag FROM things; +SELECT id, name, tag, amount, tags, labels, matrix, kind, created, updated, price, status, attrs, pos, geo, scores, ip, uid, fixed, flag FROM things; diff --git a/internal/engine/clickhouse/testgen/testdata/types/schema.sql b/internal/engine/clickhouse/testgen/testdata/types/schema.sql index 3747769d2e..5f78064628 100644 --- a/internal/engine/clickhouse/testgen/testdata/types/schema.sql +++ b/internal/engine/clickhouse/testgen/testdata/types/schema.sql @@ -13,6 +13,8 @@ CREATE TABLE things ( status Enum8('active' = 1, 'deleted' = 2), attrs Map(String, UInt32), pos Tuple(Float64, Float64), + geo Tuple(lat Float64, lon Float64), + scores Map(String, Nullable(UInt8)), ip IPv4, uid UUID, fixed FixedString(4), diff --git a/internal/engine/clickhouse/testgen/testdata/types/stdout.txt b/internal/engine/clickhouse/testgen/testdata/types/stdout.txt index 041b05363b..350a8bbec6 100644 --- a/internal/engine/clickhouse/testgen/testdata/types/stdout.txt +++ b/internal/engine/clickhouse/testgen/testdata/types/stdout.txt @@ -5,128 +5,122 @@ "columns": [ { "name": "id", - "data_type": "uint64", + "type": "uint64", "not_null": true, - "is_array": false, "table": "things" }, { "name": "name", - "data_type": "string", + "type": "string", "not_null": true, - "is_array": false, "table": "things" }, { "name": "tag", - "data_type": "string", + "type": "string", "not_null": false, - "is_array": false, "table": "things" }, { "name": "amount", - "data_type": "float64", + "type": "float64", "not_null": true, - "is_array": false, "table": "things" }, { "name": "tags", - "data_type": "string", + "type": "array(string)", "not_null": true, - "is_array": true, "table": "things" }, { "name": "labels", - "data_type": "string", - "not_null": false, - "is_array": true, + "type": "array(nullable(string))", + "not_null": true, "table": "things" }, { "name": "matrix", - "data_type": "uint8", + "type": "array(array(uint8))", "not_null": true, - "is_array": true, "table": "things" }, { "name": "kind", - "data_type": "string", + "type": "lowcardinality(string)", "not_null": false, - "is_array": false, "table": "things" }, { "name": "created", - "data_type": "datetime", + "type": "datetime", "not_null": true, - "is_array": false, "table": "things" }, { "name": "updated", - "data_type": "datetime64", + "type": "datetime64(3, 'UTC')", "not_null": true, - "is_array": false, "table": "things" }, { "name": "price", - "data_type": "decimal", + "type": "decimal(10, 2)", "not_null": true, - "is_array": false, "table": "things" }, { "name": "status", - "data_type": "enum8", + "type": "enum8('active': 1, 'deleted': 2)", "not_null": true, - "is_array": false, "table": "things" }, { "name": "attrs", - "data_type": "map", + "type": "map(string, uint32)", "not_null": true, - "is_array": false, "table": "things" }, { "name": "pos", - "data_type": "tuple", + "type": "tuple(float64, float64)", + "not_null": true, + "table": "things" + }, + { + "name": "geo", + "type": "tuple(lat: float64, lon: float64)", + "not_null": true, + "table": "things" + }, + { + "name": "scores", + "type": "map(string, nullable(uint8))", "not_null": true, - "is_array": false, "table": "things" }, { "name": "ip", - "data_type": "ipv4", + "type": "ipv4", "not_null": true, - "is_array": false, "table": "things" }, { "name": "uid", - "data_type": "uuid", + "type": "uuid", "not_null": true, - "is_array": false, "table": "things" }, { "name": "fixed", - "data_type": "fixedstring", + "type": "fixedstring(4)", "not_null": true, - "is_array": false, "table": "things" }, { "name": "flag", - "data_type": "bool", + "type": "bool", "not_null": true, - "is_array": false, "table": "things" } ], @@ -138,128 +132,122 @@ "columns": [ { "name": "id", - "data_type": "uint64", + "type": "uint64", "not_null": true, - "is_array": false, "table": "things" }, { "name": "name", - "data_type": "string", + "type": "string", "not_null": true, - "is_array": false, "table": "things" }, { "name": "tag", - "data_type": "string", + "type": "string", "not_null": false, - "is_array": false, "table": "things" }, { "name": "amount", - "data_type": "float64", + "type": "float64", "not_null": true, - "is_array": false, "table": "things" }, { "name": "tags", - "data_type": "string", + "type": "array(string)", "not_null": true, - "is_array": true, "table": "things" }, { "name": "labels", - "data_type": "string", - "not_null": false, - "is_array": true, + "type": "array(nullable(string))", + "not_null": true, "table": "things" }, { "name": "matrix", - "data_type": "uint8", + "type": "array(array(uint8))", "not_null": true, - "is_array": true, "table": "things" }, { "name": "kind", - "data_type": "string", + "type": "lowcardinality(string)", "not_null": false, - "is_array": false, "table": "things" }, { "name": "created", - "data_type": "datetime", + "type": "datetime", "not_null": true, - "is_array": false, "table": "things" }, { "name": "updated", - "data_type": "datetime64", + "type": "datetime64(3, 'UTC')", "not_null": true, - "is_array": false, "table": "things" }, { "name": "price", - "data_type": "decimal", + "type": "decimal(10, 2)", "not_null": true, - "is_array": false, "table": "things" }, { "name": "status", - "data_type": "enum8", + "type": "enum8('active': 1, 'deleted': 2)", "not_null": true, - "is_array": false, "table": "things" }, { "name": "attrs", - "data_type": "map", + "type": "map(string, uint32)", "not_null": true, - "is_array": false, "table": "things" }, { "name": "pos", - "data_type": "tuple", + "type": "tuple(float64, float64)", + "not_null": true, + "table": "things" + }, + { + "name": "geo", + "type": "tuple(lat: float64, lon: float64)", + "not_null": true, + "table": "things" + }, + { + "name": "scores", + "type": "map(string, nullable(uint8))", "not_null": true, - "is_array": false, "table": "things" }, { "name": "ip", - "data_type": "ipv4", + "type": "ipv4", "not_null": true, - "is_array": false, "table": "things" }, { "name": "uid", - "data_type": "uuid", + "type": "uuid", "not_null": true, - "is_array": false, "table": "things" }, { "name": "fixed", - "data_type": "fixedstring", + "type": "fixedstring(4)", "not_null": true, - "is_array": false, "table": "things" }, { "name": "flag", - "data_type": "bool", + "type": "bool", "not_null": true, - "is_array": false, "table": "things" } ], diff --git a/internal/engine/clickhouse/testgen/types.go b/internal/engine/clickhouse/testgen/types.go index 5a9c2907fc..efbeb935d1 100644 --- a/internal/engine/clickhouse/testgen/types.go +++ b/internal/engine/clickhouse/testgen/types.go @@ -2,41 +2,104 @@ package main import "strings" -// sqlcType lowers a ClickHouse type name to the shape sqlc's ClickHouse -// engine reports: the lowercased base name with parameters dropped, -// Nullable and Array wrappers folded into flags and LowCardinality -// discarded. It mirrors unwrapTypeString in the engine so goldens generated -// here diff cleanly against `sqlc analyze`. -func sqlcType(t string) (dataType string, isArray, notNull bool) { - name, arr, nullable := unwrapType(t) - return name, arr, !nullable -} +// A type is written as a call expression, the way ClickHouse itself models +// one: a lowercased name applied to an ordered argument list in which each +// argument is a number, a quoted string, an identifier, another call, or +// one of those with a label. Nothing is special-cased, so Nullable, Array +// and LowCardinality are ordinary names, and the text form is the same for +// every engine that spells the same structure differently: +// +// Map(String, Nullable(UInt32)) map(string, nullable(uint32)) +// Tuple(lat Float64, lon Float64) tuple(lat: float64, lon: float64) +// Enum8('a' = 1, 'b' = 2) enum8('a': 1, 'b': 2) +// DateTime64(3, 'UTC') datetime64(3, 'UTC') +// AggregateFunction(uniq, String) aggregatefunction(uniq, string) +// +// The catalog resolves the names afterwards; the output only records what +// was said. -func unwrapType(t string) (name string, isArray, nullable bool) { - base, args := splitType(t) - switch strings.ToLower(base) { +// typeExpr renders a column's type and reports whether it is NOT NULL. An +// outer Nullable is the column's nullability rather than part of its type, +// so it is lifted into the flag, through LowCardinality when needed; a +// Nullable anywhere deeper stays in the expression. +func typeExpr(t string) (expr string, notNull bool) { + name, args := splitType(t) + switch strings.ToLower(name) { case "nullable": if len(args) == 1 { - inner, arr, _ := unwrapType(args[0]) - return inner, arr, true + expr, _ = typeExpr(args[0]) + return expr, false } - return "nullable", false, true case "lowcardinality": if len(args) == 1 { - return unwrapType(args[0]) + inner, notNull := typeExpr(args[0]) + return "lowcardinality(" + inner + ")", notNull } - return "lowcardinality", false, false - case "array": - if len(args) == 1 { - inner, _, nul := unwrapType(args[0]) - return inner, true, nul + } + return renderCall(t), true +} + +// renderCall renders a type as a call expression. +func renderCall(t string) string { + name, args := splitType(t) + name = strings.ToLower(strings.TrimSpace(name)) + if name == "" { + return "nothing" + } + if args == nil { + return name + } + parts := make([]string, len(args)) + for i, a := range args { + parts[i] = renderArg(a) + } + return name + "(" + strings.Join(parts, ", ") + ")" +} + +// renderArg renders one argument: a quoted string, a number, a labelled +// argument (`lat Float64` in a Tuple, `'a' = 1` in an Enum), or a call. +func renderArg(a string) string { + a = strings.TrimSpace(a) + if strings.HasPrefix(a, "'") { + end := skipQuoted(a, 0) + if rest := strings.TrimSpace(a[end:]); strings.HasPrefix(rest, "=") { + return a[:end] + ": " + renderArg(rest[1:]) + } + return a[:end] + } + if isNumber(a) { + return a + } + if i := labelEnd(a); i > 0 { + return a[:i] + ": " + renderArg(a[i+1:]) + } + return renderCall(a) +} + +// labelEnd returns the index of the space separating a label from the type +// it labels, or -1 when the argument has no label: a space that comes before +// any parenthesis, as in `lat Float64` or `tags Array(String)`. +func labelEnd(a string) int { + head := a + if p := strings.IndexByte(a, '('); p >= 0 { + head = a[:p] + } + return strings.IndexByte(head, ' ') +} + +func isNumber(s string) bool { + if s == "" { + return false + } + for i, c := range s { + if c == '-' && i == 0 && len(s) > 1 { + continue + } + if c < '0' || c > '9' { + return false } - return "array", true, false - case "": - return "nothing", false, false - default: - return strings.ToLower(base), false, false } + return true } // splitType splits `Base(arg, arg)` into its base name and top-level From 8431c985ff7521ddcbd6365669eac40fea407bcd Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 2 Sep 2026 19:15:12 +0000 Subject: [PATCH 03/10] Emit testgen types as JSON call trees and drop not_null A column's type is now a JSON expression: a lowercased name applied to arguments, each carrying an optional label and exactly one of type, int or string. The shape maps one to one onto a protobuf message with a oneof for the argument value. There is no separate nullability flag any more: a nullable column is one whose type is nullable(...), which keeps Nullable at every depth where ClickHouse put it. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01Y6XkyWnx7iJFEb8q3AnYps --- internal/engine/clickhouse/testgen/README.md | 42 +- internal/engine/clickhouse/testgen/analyze.go | 13 +- .../testdata/analyze_params/stdout.txt | 35 +- .../testgen/testdata/exec/stdout.txt | 57 +- .../testgen/testdata/expressions/stdout.txt | 229 +++++--- .../testgen/testdata/subqueries/stdout.txt | 106 ++-- .../testgen/testdata/types/stdout.txt | 488 +++++++++++++++--- internal/engine/clickhouse/testgen/types.go | 129 +++-- 8 files changed, 790 insertions(+), 309 deletions(-) diff --git a/internal/engine/clickhouse/testgen/README.md b/internal/engine/clickhouse/testgen/README.md index 0314236765..5984a5e670 100644 --- a/internal/engine/clickhouse/testgen/README.md +++ b/internal/engine/clickhouse/testgen/README.md @@ -40,24 +40,30 @@ query is explained and then executed, and the process exits. target columns reported by `DESCRIBE TABLE`. The output has the shape of `sqlc analyze`, except that each column's type is -one expression rather than a name and flags. A type is written as a call: a -lowercased name applied to arguments that are numbers, quoted strings, -identifiers, other calls, or any of those with a label. `Nullable`, `Array` -and `LowCardinality` are ordinary names in that grammar, so nothing about -nesting is lost: - -| ClickHouse | testgen | -|-----------------------------------|------------------------------------| -| `Array(Nullable(String))` | `array(nullable(string))` | -| `Map(String, UInt32)` | `map(string, uint32)` | -| `Tuple(lat Float64, lon Float64)` | `tuple(lat: float64, lon: float64)`| -| `Enum8('a' = 1, 'b' = 2)` | `enum8('a': 1, 'b': 2)` | -| `DateTime64(3, 'UTC')` | `datetime64(3, 'UTC')` | -| `LowCardinality(Nullable(String))`| `lowcardinality(string)`, `not_null: false` | - -An outer `Nullable` is the column's nullability rather than part of its type, -so it is lifted into `not_null`; a `Nullable` at any greater depth stays in -the expression. Resolving the names is left to whoever reads the output. +one expression rather than a name and flags. A type is a call: a lowercased +`name` applied to `args`, each of which carries an optional `label` and +exactly one of `type`, `int` or `string`. `Nullable`, `Array` and +`LowCardinality` are ordinary names in that grammar, so nothing about nesting +is lost, and there is no separate nullability flag: a nullable column is one +whose type is `nullable(...)`. + +```json +{"name": "map", "args": [ + {"type": {"name": "string"}}, + {"type": {"name": "nullable", "args": [{"type": {"name": "uint8"}}]}}]} + +{"name": "tuple", "args": [ + {"label": "lat", "type": {"name": "float64"}}, + {"label": "lon", "type": {"name": "float64"}}]} + +{"name": "enum8", "args": [{"label": "active", "int": 1}, {"label": "deleted", "int": 2}]} + +{"name": "datetime64", "args": [{"int": 3}, {"string": "UTC"}]} +``` + +An identifier argument, such as the function in `AggregateFunction(uniq, +String)`, is a type with no arguments. Resolving the names is left to +whoever reads the output. ## Tests diff --git a/internal/engine/clickhouse/testgen/analyze.go b/internal/engine/clickhouse/testgen/analyze.go index b2b446702e..b8d6cc5263 100644 --- a/internal/engine/clickhouse/testgen/analyze.go +++ b/internal/engine/clickhouse/testgen/analyze.go @@ -20,10 +20,9 @@ type analyzedQuery struct { } type analyzedColumn struct { - Name string `json:"name"` - Type string `json:"type"` - NotNull bool `json:"not_null"` - Table string `json:"table,omitempty"` + Name string `json:"name"` + Type *typeExpr `json:"type,omitempty"` + Table string `json:"table,omitempty"` } type analyzedParam struct { @@ -118,8 +117,10 @@ func analyzeQuery(ctx context.Context, l local, schema, fixture string, q query) } func column(name, typ string) analyzedColumn { - expr, notNull := typeExpr(typ) - return analyzedColumn{Name: name, Type: expr, NotNull: notNull} + if typ == "" { + return analyzedColumn{Name: name} + } + return analyzedColumn{Name: name, Type: parseType(typ)} } // returnsRows reports whether a statement produces a result set and so can diff --git a/internal/engine/clickhouse/testgen/testdata/analyze_params/stdout.txt b/internal/engine/clickhouse/testgen/testdata/analyze_params/stdout.txt index 3c1408ba71..548bb8b7b1 100644 --- a/internal/engine/clickhouse/testgen/testdata/analyze_params/stdout.txt +++ b/internal/engine/clickhouse/testgen/testdata/analyze_params/stdout.txt @@ -5,14 +5,16 @@ "columns": [ { "name": "id", - "type": "uint64", - "not_null": true, + "type": { + "name": "uint64" + }, "table": "events" }, { "name": "name", - "type": "string", - "not_null": true, + "type": { + "name": "string" + }, "table": "events" } ], @@ -21,8 +23,9 @@ "number": 1, "column": { "name": "id", - "type": "uint64", - "not_null": true, + "type": { + "name": "uint64" + }, "table": "events" } } @@ -34,14 +37,16 @@ "columns": [ { "name": "id", - "type": "uint64", - "not_null": true, + "type": { + "name": "uint64" + }, "table": "events" }, { "name": "name", - "type": "string", - "not_null": true, + "type": { + "name": "string" + }, "table": "events" } ], @@ -50,8 +55,9 @@ "number": 1, "column": { "name": "name", - "type": "string", - "not_null": true, + "type": { + "name": "string" + }, "table": "events" } }, @@ -59,8 +65,9 @@ "number": 2, "column": { "name": "amount", - "type": "float64", - "not_null": true, + "type": { + "name": "float64" + }, "table": "events" } } diff --git a/internal/engine/clickhouse/testgen/testdata/exec/stdout.txt b/internal/engine/clickhouse/testgen/testdata/exec/stdout.txt index aa0b3e3f7b..6d5aa0c14f 100644 --- a/internal/engine/clickhouse/testgen/testdata/exec/stdout.txt +++ b/internal/engine/clickhouse/testgen/testdata/exec/stdout.txt @@ -8,8 +8,9 @@ "number": 1, "column": { "name": "id", - "type": "uint64", - "not_null": true, + "type": { + "name": "uint64" + }, "table": "users" } }, @@ -17,8 +18,9 @@ "number": 2, "column": { "name": "email", - "type": "string", - "not_null": true, + "type": { + "name": "string" + }, "table": "users" } } @@ -33,8 +35,9 @@ "number": 1, "column": { "name": "id", - "type": "uint64", - "not_null": true, + "type": { + "name": "uint64" + }, "table": "events" } }, @@ -42,8 +45,9 @@ "number": 2, "column": { "name": "name", - "type": "string", - "not_null": true, + "type": { + "name": "string" + }, "table": "events" } }, @@ -51,8 +55,16 @@ "number": 3, "column": { "name": "tag", - "type": "string", - "not_null": false, + "type": { + "name": "nullable", + "args": [ + { + "type": { + "name": "string" + } + } + ] + }, "table": "events" } }, @@ -60,8 +72,9 @@ "number": 4, "column": { "name": "amount", - "type": "float64", - "not_null": true, + "type": { + "name": "float64" + }, "table": "events" } } @@ -76,8 +89,9 @@ "number": 1, "column": { "name": "id", - "type": "uint64", - "not_null": true, + "type": { + "name": "uint64" + }, "table": "events" } }, @@ -85,8 +99,9 @@ "number": 2, "column": { "name": "name", - "type": "string", - "not_null": true, + "type": { + "name": "string" + }, "table": "events" } }, @@ -94,8 +109,9 @@ "number": 3, "column": { "name": "id", - "type": "uint64", - "not_null": true, + "type": { + "name": "uint64" + }, "table": "events" } }, @@ -103,8 +119,9 @@ "number": 4, "column": { "name": "name", - "type": "string", - "not_null": true, + "type": { + "name": "string" + }, "table": "events" } } diff --git a/internal/engine/clickhouse/testgen/testdata/expressions/stdout.txt b/internal/engine/clickhouse/testgen/testdata/expressions/stdout.txt index b3cd5f4690..80dbed9194 100644 --- a/internal/engine/clickhouse/testgen/testdata/expressions/stdout.txt +++ b/internal/engine/clickhouse/testgen/testdata/expressions/stdout.txt @@ -5,28 +5,40 @@ "columns": [ { "name": "n", - "type": "uint64", - "not_null": true + "type": { + "name": "uint64" + } }, { "name": "top", - "type": "float64", - "not_null": true + "type": { + "name": "float64" + } }, { "name": "sum(amount)", - "type": "float64", - "not_null": true + "type": { + "name": "float64" + } }, { "name": "first_tag", - "type": "string", - "not_null": false + "type": { + "name": "nullable", + "args": [ + { + "type": { + "name": "string" + } + } + ] + } }, { "name": "uniq(name)", - "type": "uint64", - "not_null": true + "type": { + "name": "uint64" + } } ], "params": [] @@ -37,38 +49,59 @@ "columns": [ { "name": "next_id", - "type": "uint64", - "not_null": true + "type": { + "name": "uint64" + } }, { "name": "maybe", - "type": "string", - "not_null": false + "type": { + "name": "nullable", + "args": [ + { + "type": { + "name": "string" + } + } + ] + } }, { "name": "tag_or_none", - "type": "string", - "not_null": true + "type": { + "name": "string" + } }, { "name": "lower(name)", - "type": "string", - "not_null": true + "type": { + "name": "string" + } }, { "name": "today", - "type": "date", - "not_null": true + "type": { + "name": "date" + } }, { "name": "big", - "type": "uint8", - "not_null": true + "type": { + "name": "uint8" + } }, { "name": "word", - "type": "string", - "not_null": false + "type": { + "name": "nullable", + "args": [ + { + "type": { + "name": "string" + } + } + ] + } } ], "params": [] @@ -79,20 +112,23 @@ "columns": [ { "name": "id", - "type": "uint64", - "not_null": true, + "type": { + "name": "uint64" + }, "table": "events" }, { "name": "name", - "type": "string", - "not_null": true, + "type": { + "name": "string" + }, "table": "events" }, { "name": "email", - "type": "string", - "not_null": true, + "type": { + "name": "string" + }, "table": "users" } ], @@ -104,14 +140,16 @@ "columns": [ { "name": "id", - "type": "uint64", - "not_null": true, + "type": { + "name": "uint64" + }, "table": "events" }, { "name": "name", - "type": "string", - "not_null": true, + "type": { + "name": "string" + }, "table": "events" } ], @@ -120,8 +158,9 @@ "number": 1, "column": { "name": "name", - "type": "string", - "not_null": true, + "type": { + "name": "string" + }, "table": "events" } }, @@ -129,8 +168,9 @@ "number": 2, "column": { "name": "amount", - "type": "float64", - "not_null": true, + "type": { + "name": "float64" + }, "table": "events" } }, @@ -138,8 +178,16 @@ "number": 3, "column": { "name": "tag", - "type": "string", - "not_null": false, + "type": { + "name": "nullable", + "args": [ + { + "type": { + "name": "string" + } + } + ] + }, "table": "events" } } @@ -151,14 +199,16 @@ "columns": [ { "name": "id", - "type": "uint64", - "not_null": true, + "type": { + "name": "uint64" + }, "table": "events" }, { "name": "name", - "type": "string", - "not_null": true, + "type": { + "name": "string" + }, "table": "events" } ], @@ -167,8 +217,9 @@ "number": 1, "column": { "name": "name", - "type": "string", - "not_null": true, + "type": { + "name": "string" + }, "table": "events" } }, @@ -176,8 +227,16 @@ "number": 2, "column": { "name": "tag", - "type": "string", - "not_null": false, + "type": { + "name": "nullable", + "args": [ + { + "type": { + "name": "string" + } + } + ] + }, "table": "events" } }, @@ -185,8 +244,9 @@ "number": 3, "column": { "name": "amount", - "type": "float64", - "not_null": true, + "type": { + "name": "float64" + }, "table": "events" } }, @@ -194,8 +254,9 @@ "number": 3, "column": { "name": "amount", - "type": "float64", - "not_null": true, + "type": { + "name": "float64" + }, "table": "events" } } @@ -207,8 +268,9 @@ "columns": [ { "name": "id", - "type": "uint64", - "not_null": true, + "type": { + "name": "uint64" + }, "table": "events" } ], @@ -217,16 +279,18 @@ "number": 1, "column": { "name": "lower", - "type": "string", - "not_null": true + "type": { + "name": "string" + } } }, { "number": 2, "column": { "name": "id", - "type": "uint64", - "not_null": true, + "type": { + "name": "uint64" + }, "table": "events" } }, @@ -234,8 +298,9 @@ "number": 3, "column": { "name": "", - "type": "date", - "not_null": true + "type": { + "name": "date" + } } } ] @@ -246,8 +311,9 @@ "columns": [ { "name": "id", - "type": "uint64", - "not_null": true, + "type": { + "name": "uint64" + }, "table": "events" } ], @@ -256,16 +322,18 @@ "number": 1, "column": { "name": "", - "type": "uint64", - "not_null": true + "type": { + "name": "uint64" + } } }, { "number": 2, "column": { "name": "", - "type": "uint64", - "not_null": true + "type": { + "name": "uint64" + } } } ] @@ -276,13 +344,22 @@ "columns": [ { "name": "echo", - "type": "nothing", - "not_null": false + "type": { + "name": "nullable", + "args": [ + { + "type": { + "name": "nothing" + } + } + ] + } }, { "name": "lit", - "type": "string", - "not_null": true + "type": { + "name": "string" + } } ], "params": [ @@ -290,8 +367,16 @@ "number": 1, "column": { "name": "echo", - "type": "nothing", - "not_null": false + "type": { + "name": "nullable", + "args": [ + { + "type": { + "name": "nothing" + } + } + ] + } } } ] diff --git a/internal/engine/clickhouse/testgen/testdata/subqueries/stdout.txt b/internal/engine/clickhouse/testgen/testdata/subqueries/stdout.txt index 577db1337c..70bd8c33f1 100644 --- a/internal/engine/clickhouse/testgen/testdata/subqueries/stdout.txt +++ b/internal/engine/clickhouse/testgen/testdata/subqueries/stdout.txt @@ -5,20 +5,30 @@ "columns": [ { "name": "id", - "type": "uint64", - "not_null": true, + "type": { + "name": "uint64" + }, "table": "events" }, { "name": "tag", - "type": "string", - "not_null": false, + "type": { + "name": "nullable", + "args": [ + { + "type": { + "name": "string" + } + } + ] + }, "table": "events" }, { "name": "cnt", - "type": "uint64", - "not_null": true + "type": { + "name": "uint64" + } } ], "params": [] @@ -29,19 +39,22 @@ "columns": [ { "name": "x", - "type": "uint64", - "not_null": true, + "type": { + "name": "uint64" + }, "table": "events" }, { "name": "total", - "type": "float64", - "not_null": true + "type": { + "name": "float64" + } }, { "name": "email", - "type": "string", - "not_null": true + "type": { + "name": "string" + } } ], "params": [] @@ -52,38 +65,51 @@ "columns": [ { "name": "id", - "type": "uint64", - "not_null": true, + "type": { + "name": "uint64" + }, "table": "users" }, { "name": "email", - "type": "string", - "not_null": true, + "type": { + "name": "string" + }, "table": "users" }, { "name": "e.id", - "type": "uint64", - "not_null": true, + "type": { + "name": "uint64" + }, "table": "events" }, { "name": "name", - "type": "string", - "not_null": true, + "type": { + "name": "string" + }, "table": "events" }, { "name": "tag", - "type": "string", - "not_null": false, + "type": { + "name": "nullable", + "args": [ + { + "type": { + "name": "string" + } + } + ] + }, "table": "events" }, { "name": "amount", - "type": "float64", - "not_null": true, + "type": { + "name": "float64" + }, "table": "events" } ], @@ -95,14 +121,16 @@ "columns": [ { "name": "id", - "type": "uint64", - "not_null": true, + "type": { + "name": "uint64" + }, "table": "events" }, { "name": "name", - "type": "string", - "not_null": true, + "type": { + "name": "string" + }, "table": "events" } ], @@ -114,14 +142,23 @@ "columns": [ { "name": "id", - "type": "uint64", - "not_null": true, + "type": { + "name": "uint64" + }, "table": "events" }, { "name": "user_count", - "type": "uint64", - "not_null": false + "type": { + "name": "nullable", + "args": [ + { + "type": { + "name": "uint64" + } + } + ] + } } ], "params": [ @@ -129,8 +166,9 @@ "number": 1, "column": { "name": "email", - "type": "string", - "not_null": true, + "type": { + "name": "string" + }, "table": "users" } } diff --git a/internal/engine/clickhouse/testgen/testdata/types/stdout.txt b/internal/engine/clickhouse/testgen/testdata/types/stdout.txt index 350a8bbec6..e3ee33ffdf 100644 --- a/internal/engine/clickhouse/testgen/testdata/types/stdout.txt +++ b/internal/engine/clickhouse/testgen/testdata/types/stdout.txt @@ -5,122 +5,286 @@ "columns": [ { "name": "id", - "type": "uint64", - "not_null": true, + "type": { + "name": "uint64" + }, "table": "things" }, { "name": "name", - "type": "string", - "not_null": true, + "type": { + "name": "string" + }, "table": "things" }, { "name": "tag", - "type": "string", - "not_null": false, + "type": { + "name": "nullable", + "args": [ + { + "type": { + "name": "string" + } + } + ] + }, "table": "things" }, { "name": "amount", - "type": "float64", - "not_null": true, + "type": { + "name": "float64" + }, "table": "things" }, { "name": "tags", - "type": "array(string)", - "not_null": true, + "type": { + "name": "array", + "args": [ + { + "type": { + "name": "string" + } + } + ] + }, "table": "things" }, { "name": "labels", - "type": "array(nullable(string))", - "not_null": true, + "type": { + "name": "array", + "args": [ + { + "type": { + "name": "nullable", + "args": [ + { + "type": { + "name": "string" + } + } + ] + } + } + ] + }, "table": "things" }, { "name": "matrix", - "type": "array(array(uint8))", - "not_null": true, + "type": { + "name": "array", + "args": [ + { + "type": { + "name": "array", + "args": [ + { + "type": { + "name": "uint8" + } + } + ] + } + } + ] + }, "table": "things" }, { "name": "kind", - "type": "lowcardinality(string)", - "not_null": false, + "type": { + "name": "lowcardinality", + "args": [ + { + "type": { + "name": "nullable", + "args": [ + { + "type": { + "name": "string" + } + } + ] + } + } + ] + }, "table": "things" }, { "name": "created", - "type": "datetime", - "not_null": true, + "type": { + "name": "datetime" + }, "table": "things" }, { "name": "updated", - "type": "datetime64(3, 'UTC')", - "not_null": true, + "type": { + "name": "datetime64", + "args": [ + { + "int": 3 + }, + { + "string": "UTC" + } + ] + }, "table": "things" }, { "name": "price", - "type": "decimal(10, 2)", - "not_null": true, + "type": { + "name": "decimal", + "args": [ + { + "int": 10 + }, + { + "int": 2 + } + ] + }, "table": "things" }, { "name": "status", - "type": "enum8('active': 1, 'deleted': 2)", - "not_null": true, + "type": { + "name": "enum8", + "args": [ + { + "label": "active", + "int": 1 + }, + { + "label": "deleted", + "int": 2 + } + ] + }, "table": "things" }, { "name": "attrs", - "type": "map(string, uint32)", - "not_null": true, + "type": { + "name": "map", + "args": [ + { + "type": { + "name": "string" + } + }, + { + "type": { + "name": "uint32" + } + } + ] + }, "table": "things" }, { "name": "pos", - "type": "tuple(float64, float64)", - "not_null": true, + "type": { + "name": "tuple", + "args": [ + { + "type": { + "name": "float64" + } + }, + { + "type": { + "name": "float64" + } + } + ] + }, "table": "things" }, { "name": "geo", - "type": "tuple(lat: float64, lon: float64)", - "not_null": true, + "type": { + "name": "tuple", + "args": [ + { + "label": "lat", + "type": { + "name": "float64" + } + }, + { + "label": "lon", + "type": { + "name": "float64" + } + } + ] + }, "table": "things" }, { "name": "scores", - "type": "map(string, nullable(uint8))", - "not_null": true, + "type": { + "name": "map", + "args": [ + { + "type": { + "name": "string" + } + }, + { + "type": { + "name": "nullable", + "args": [ + { + "type": { + "name": "uint8" + } + } + ] + } + } + ] + }, "table": "things" }, { "name": "ip", - "type": "ipv4", - "not_null": true, + "type": { + "name": "ipv4" + }, "table": "things" }, { "name": "uid", - "type": "uuid", - "not_null": true, + "type": { + "name": "uuid" + }, "table": "things" }, { "name": "fixed", - "type": "fixedstring(4)", - "not_null": true, + "type": { + "name": "fixedstring", + "args": [ + { + "int": 4 + } + ] + }, "table": "things" }, { "name": "flag", - "type": "bool", - "not_null": true, + "type": { + "name": "bool" + }, "table": "things" } ], @@ -132,122 +296,286 @@ "columns": [ { "name": "id", - "type": "uint64", - "not_null": true, + "type": { + "name": "uint64" + }, "table": "things" }, { "name": "name", - "type": "string", - "not_null": true, + "type": { + "name": "string" + }, "table": "things" }, { "name": "tag", - "type": "string", - "not_null": false, + "type": { + "name": "nullable", + "args": [ + { + "type": { + "name": "string" + } + } + ] + }, "table": "things" }, { "name": "amount", - "type": "float64", - "not_null": true, + "type": { + "name": "float64" + }, "table": "things" }, { "name": "tags", - "type": "array(string)", - "not_null": true, + "type": { + "name": "array", + "args": [ + { + "type": { + "name": "string" + } + } + ] + }, "table": "things" }, { "name": "labels", - "type": "array(nullable(string))", - "not_null": true, + "type": { + "name": "array", + "args": [ + { + "type": { + "name": "nullable", + "args": [ + { + "type": { + "name": "string" + } + } + ] + } + } + ] + }, "table": "things" }, { "name": "matrix", - "type": "array(array(uint8))", - "not_null": true, + "type": { + "name": "array", + "args": [ + { + "type": { + "name": "array", + "args": [ + { + "type": { + "name": "uint8" + } + } + ] + } + } + ] + }, "table": "things" }, { "name": "kind", - "type": "lowcardinality(string)", - "not_null": false, + "type": { + "name": "lowcardinality", + "args": [ + { + "type": { + "name": "nullable", + "args": [ + { + "type": { + "name": "string" + } + } + ] + } + } + ] + }, "table": "things" }, { "name": "created", - "type": "datetime", - "not_null": true, + "type": { + "name": "datetime" + }, "table": "things" }, { "name": "updated", - "type": "datetime64(3, 'UTC')", - "not_null": true, + "type": { + "name": "datetime64", + "args": [ + { + "int": 3 + }, + { + "string": "UTC" + } + ] + }, "table": "things" }, { "name": "price", - "type": "decimal(10, 2)", - "not_null": true, + "type": { + "name": "decimal", + "args": [ + { + "int": 10 + }, + { + "int": 2 + } + ] + }, "table": "things" }, { "name": "status", - "type": "enum8('active': 1, 'deleted': 2)", - "not_null": true, + "type": { + "name": "enum8", + "args": [ + { + "label": "active", + "int": 1 + }, + { + "label": "deleted", + "int": 2 + } + ] + }, "table": "things" }, { "name": "attrs", - "type": "map(string, uint32)", - "not_null": true, + "type": { + "name": "map", + "args": [ + { + "type": { + "name": "string" + } + }, + { + "type": { + "name": "uint32" + } + } + ] + }, "table": "things" }, { "name": "pos", - "type": "tuple(float64, float64)", - "not_null": true, + "type": { + "name": "tuple", + "args": [ + { + "type": { + "name": "float64" + } + }, + { + "type": { + "name": "float64" + } + } + ] + }, "table": "things" }, { "name": "geo", - "type": "tuple(lat: float64, lon: float64)", - "not_null": true, + "type": { + "name": "tuple", + "args": [ + { + "label": "lat", + "type": { + "name": "float64" + } + }, + { + "label": "lon", + "type": { + "name": "float64" + } + } + ] + }, "table": "things" }, { "name": "scores", - "type": "map(string, nullable(uint8))", - "not_null": true, + "type": { + "name": "map", + "args": [ + { + "type": { + "name": "string" + } + }, + { + "type": { + "name": "nullable", + "args": [ + { + "type": { + "name": "uint8" + } + } + ] + } + } + ] + }, "table": "things" }, { "name": "ip", - "type": "ipv4", - "not_null": true, + "type": { + "name": "ipv4" + }, "table": "things" }, { "name": "uid", - "type": "uuid", - "not_null": true, + "type": { + "name": "uuid" + }, "table": "things" }, { "name": "fixed", - "type": "fixedstring(4)", - "not_null": true, + "type": { + "name": "fixedstring", + "args": [ + { + "int": 4 + } + ] + }, "table": "things" }, { "name": "flag", - "type": "bool", - "not_null": true, + "type": { + "name": "bool" + }, "table": "things" } ], diff --git a/internal/engine/clickhouse/testgen/types.go b/internal/engine/clickhouse/testgen/types.go index efbeb935d1..e1ae0068ba 100644 --- a/internal/engine/clickhouse/testgen/types.go +++ b/internal/engine/clickhouse/testgen/types.go @@ -1,79 +1,86 @@ package main -import "strings" +import ( + "strconv" + "strings" +) -// A type is written as a call expression, the way ClickHouse itself models -// one: a lowercased name applied to an ordered argument list in which each -// argument is a number, a quoted string, an identifier, another call, or -// one of those with a label. Nothing is special-cased, so Nullable, Array -// and LowCardinality are ordinary names, and the text form is the same for -// every engine that spells the same structure differently: +// A type is a call expression, the way ClickHouse itself models one: a +// lowercased name applied to an ordered argument list. Each argument is +// another type, an integer or a quoted string, optionally labelled, so +// Nullable, Array and LowCardinality are ordinary names and nothing about a +// nested type is lost. The shape maps one to one onto a protobuf message +// with a oneof for the argument value: // -// Map(String, Nullable(UInt32)) map(string, nullable(uint32)) -// Tuple(lat Float64, lon Float64) tuple(lat: float64, lon: float64) -// Enum8('a' = 1, 'b' = 2) enum8('a': 1, 'b': 2) -// DateTime64(3, 'UTC') datetime64(3, 'UTC') -// AggregateFunction(uniq, String) aggregatefunction(uniq, string) +// Map(String, Nullable(UInt32)) +// {"name": "map", "args": [ +// {"type": {"name": "string"}}, +// {"type": {"name": "nullable", "args": [{"type": {"name": "uint32"}}]}}]} // -// The catalog resolves the names afterwards; the output only records what -// was said. +// Tuple(lat Float64, lon Float64) +// {"name": "tuple", "args": [ +// {"label": "lat", "type": {"name": "float64"}}, +// {"label": "lon", "type": {"name": "float64"}}]} +// +// Enum8('a' = 1, 'b' = 2) +// {"name": "enum8", "args": [{"label": "a", "int": 1}, {"label": "b", "int": 2}]} +// +// DateTime64(3, 'UTC') +// {"name": "datetime64", "args": [{"int": 3}, {"string": "UTC"}]} +// +// An identifier argument such as the function in AggregateFunction(uniq, +// String) is a type with no arguments. Resolving names against the catalog +// is the reader's job; the output only records what was said. -// typeExpr renders a column's type and reports whether it is NOT NULL. An -// outer Nullable is the column's nullability rather than part of its type, -// so it is lifted into the flag, through LowCardinality when needed; a -// Nullable anywhere deeper stays in the expression. -func typeExpr(t string) (expr string, notNull bool) { - name, args := splitType(t) - switch strings.ToLower(name) { - case "nullable": - if len(args) == 1 { - expr, _ = typeExpr(args[0]) - return expr, false - } - case "lowcardinality": - if len(args) == 1 { - inner, notNull := typeExpr(args[0]) - return "lowcardinality(" + inner + ")", notNull - } - } - return renderCall(t), true +type typeExpr struct { + Name string `json:"name"` + Args []typeArg `json:"args,omitempty"` } -// renderCall renders a type as a call expression. -func renderCall(t string) string { +type typeArg struct { + Label string `json:"label,omitempty"` + Type *typeExpr `json:"type,omitempty"` + Int *int64 `json:"int,omitempty"` + String *string `json:"string,omitempty"` +} + +// parseType turns a ClickHouse type string into its expression. +func parseType(t string) *typeExpr { name, args := splitType(t) name = strings.ToLower(strings.TrimSpace(name)) if name == "" { - return "nothing" + name = "nothing" } - if args == nil { - return name + expr := &typeExpr{Name: name} + for _, a := range args { + expr.Args = append(expr.Args, parseArg(a)) } - parts := make([]string, len(args)) - for i, a := range args { - parts[i] = renderArg(a) - } - return name + "(" + strings.Join(parts, ", ") + ")" + return expr } -// renderArg renders one argument: a quoted string, a number, a labelled -// argument (`lat Float64` in a Tuple, `'a' = 1` in an Enum), or a call. -func renderArg(a string) string { +// parseArg parses one argument: a quoted string, an integer, a labelled +// argument (`lat Float64` in a Tuple, `'a' = 1` in an Enum), or a type. +func parseArg(a string) typeArg { a = strings.TrimSpace(a) if strings.HasPrefix(a, "'") { end := skipQuoted(a, 0) + lit := unquote(a[1 : end-1]) if rest := strings.TrimSpace(a[end:]); strings.HasPrefix(rest, "=") { - return a[:end] + ": " + renderArg(rest[1:]) + arg := parseArg(rest[1:]) + arg.Label = lit + return arg } - return a[:end] + return typeArg{String: &lit} } - if isNumber(a) { - return a + if n, err := strconv.ParseInt(a, 10, 64); err == nil { + return typeArg{Int: &n} } if i := labelEnd(a); i > 0 { - return a[:i] + ": " + renderArg(a[i+1:]) + arg := parseArg(a[i+1:]) + arg.Label = a[:i] + return arg } - return renderCall(a) + return typeArg{Type: parseType(a)} } // labelEnd returns the index of the space separating a label from the type @@ -87,19 +94,11 @@ func labelEnd(a string) int { return strings.IndexByte(head, ' ') } -func isNumber(s string) bool { - if s == "" { - return false - } - for i, c := range s { - if c == '-' && i == 0 && len(s) > 1 { - continue - } - if c < '0' || c > '9' { - return false - } - } - return true +// unquote undoes the escaping inside a single-quoted ClickHouse literal. +func unquote(s string) string { + s = strings.ReplaceAll(s, `\'`, `'`) + s = strings.ReplaceAll(s, `''`, `'`) + return strings.ReplaceAll(s, `\\`, `\`) } // splitType splits `Base(arg, arg)` into its base name and top-level From b6cd45d0e80b5c5f2c5e70b2a1273ae5504cb132 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 2 Sep 2026 19:24:46 +0000 Subject: [PATCH 04/10] Add a bool argument value to testgen type expressions Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01Y6XkyWnx7iJFEb8q3AnYps --- internal/engine/clickhouse/testgen/README.md | 2 +- internal/engine/clickhouse/testgen/types.go | 14 +++++++++++--- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/internal/engine/clickhouse/testgen/README.md b/internal/engine/clickhouse/testgen/README.md index 5984a5e670..90be352a2c 100644 --- a/internal/engine/clickhouse/testgen/README.md +++ b/internal/engine/clickhouse/testgen/README.md @@ -42,7 +42,7 @@ query is explained and then executed, and the process exits. The output has the shape of `sqlc analyze`, except that each column's type is one expression rather than a name and flags. A type is a call: a lowercased `name` applied to `args`, each of which carries an optional `label` and -exactly one of `type`, `int` or `string`. `Nullable`, `Array` and +exactly one of `type`, `int`, `bool` or `string`. `Nullable`, `Array` and `LowCardinality` are ordinary names in that grammar, so nothing about nesting is lost, and there is no separate nullability flag: a nullable column is one whose type is `nullable(...)`. diff --git a/internal/engine/clickhouse/testgen/types.go b/internal/engine/clickhouse/testgen/types.go index e1ae0068ba..a12bb542db 100644 --- a/internal/engine/clickhouse/testgen/types.go +++ b/internal/engine/clickhouse/testgen/types.go @@ -7,7 +7,8 @@ import ( // A type is a call expression, the way ClickHouse itself models one: a // lowercased name applied to an ordered argument list. Each argument is -// another type, an integer or a quoted string, optionally labelled, so +// another type, an integer, a boolean or a quoted string, optionally +// labelled, so // Nullable, Array and LowCardinality are ordinary names and nothing about a // nested type is lost. The shape maps one to one onto a protobuf message // with a oneof for the argument value: @@ -41,6 +42,7 @@ type typeArg struct { Label string `json:"label,omitempty"` Type *typeExpr `json:"type,omitempty"` Int *int64 `json:"int,omitempty"` + Bool *bool `json:"bool,omitempty"` String *string `json:"string,omitempty"` } @@ -58,8 +60,9 @@ func parseType(t string) *typeExpr { return expr } -// parseArg parses one argument: a quoted string, an integer, a labelled -// argument (`lat Float64` in a Tuple, `'a' = 1` in an Enum), or a type. +// parseArg parses one argument: a quoted string, an integer, a boolean, a +// labelled argument (`lat Float64` in a Tuple, `'a' = 1` in an Enum), or a +// type. func parseArg(a string) typeArg { a = strings.TrimSpace(a) if strings.HasPrefix(a, "'") { @@ -75,6 +78,11 @@ func parseArg(a string) typeArg { if n, err := strconv.ParseInt(a, 10, 64); err == nil { return typeArg{Int: &n} } + switch strings.ToLower(a) { + case "true", "false": + b := strings.EqualFold(a, "true") + return typeArg{Bool: &b} + } if i := labelEnd(a); i > 0 { arg := parseArg(a[i+1:]) arg.Label = a[:i] From b76757fbf39ad84c398f5631a02243656e9d1681 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 2 Sep 2026 19:33:54 +0000 Subject: [PATCH 05/10] Make nullability an attribute of a testgen type Nullable(T) is now T with nullable set, at whatever depth ClickHouse wrote it, instead of a call named nullable. Every engine has nullability and every consumer needs it, so an attribute on the type node spares readers from treating one name as special. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01Y6XkyWnx7iJFEb8q3AnYps --- internal/engine/clickhouse/testgen/README.md | 9 ++- .../testgen/testdata/exec/stdout.txt | 10 +-- .../testgen/testdata/expressions/stdout.txt | 70 ++++------------ .../testgen/testdata/subqueries/stdout.txt | 30 ++----- .../testgen/testdata/types/stdout.txt | 80 ++++--------------- internal/engine/clickhouse/testgen/types.go | 22 +++-- 6 files changed, 58 insertions(+), 163 deletions(-) diff --git a/internal/engine/clickhouse/testgen/README.md b/internal/engine/clickhouse/testgen/README.md index 90be352a2c..8f54d98fe2 100644 --- a/internal/engine/clickhouse/testgen/README.md +++ b/internal/engine/clickhouse/testgen/README.md @@ -42,15 +42,16 @@ query is explained and then executed, and the process exits. The output has the shape of `sqlc analyze`, except that each column's type is one expression rather than a name and flags. A type is a call: a lowercased `name` applied to `args`, each of which carries an optional `label` and -exactly one of `type`, `int`, `bool` or `string`. `Nullable`, `Array` and +exactly one of `type`, `int`, `bool` or `string`. `Array`, `Map` and `LowCardinality` are ordinary names in that grammar, so nothing about nesting -is lost, and there is no separate nullability flag: a nullable column is one -whose type is `nullable(...)`. +is lost. Nullability is an attribute of a type rather than a wrapper: +`Nullable(T)` becomes `T` with `nullable` set, at whatever depth ClickHouse +wrote it. ```json {"name": "map", "args": [ {"type": {"name": "string"}}, - {"type": {"name": "nullable", "args": [{"type": {"name": "uint8"}}]}}]} + {"type": {"name": "uint8", "nullable": true}}]} {"name": "tuple", "args": [ {"label": "lat", "type": {"name": "float64"}}, diff --git a/internal/engine/clickhouse/testgen/testdata/exec/stdout.txt b/internal/engine/clickhouse/testgen/testdata/exec/stdout.txt index 6d5aa0c14f..3709c4909b 100644 --- a/internal/engine/clickhouse/testgen/testdata/exec/stdout.txt +++ b/internal/engine/clickhouse/testgen/testdata/exec/stdout.txt @@ -56,14 +56,8 @@ "column": { "name": "tag", "type": { - "name": "nullable", - "args": [ - { - "type": { - "name": "string" - } - } - ] + "name": "string", + "nullable": true }, "table": "events" } diff --git a/internal/engine/clickhouse/testgen/testdata/expressions/stdout.txt b/internal/engine/clickhouse/testgen/testdata/expressions/stdout.txt index 80dbed9194..05eefc97c4 100644 --- a/internal/engine/clickhouse/testgen/testdata/expressions/stdout.txt +++ b/internal/engine/clickhouse/testgen/testdata/expressions/stdout.txt @@ -24,14 +24,8 @@ { "name": "first_tag", "type": { - "name": "nullable", - "args": [ - { - "type": { - "name": "string" - } - } - ] + "name": "string", + "nullable": true } }, { @@ -56,14 +50,8 @@ { "name": "maybe", "type": { - "name": "nullable", - "args": [ - { - "type": { - "name": "string" - } - } - ] + "name": "string", + "nullable": true } }, { @@ -93,14 +81,8 @@ { "name": "word", "type": { - "name": "nullable", - "args": [ - { - "type": { - "name": "string" - } - } - ] + "name": "string", + "nullable": true } } ], @@ -179,14 +161,8 @@ "column": { "name": "tag", "type": { - "name": "nullable", - "args": [ - { - "type": { - "name": "string" - } - } - ] + "name": "string", + "nullable": true }, "table": "events" } @@ -228,14 +204,8 @@ "column": { "name": "tag", "type": { - "name": "nullable", - "args": [ - { - "type": { - "name": "string" - } - } - ] + "name": "string", + "nullable": true }, "table": "events" } @@ -345,14 +315,8 @@ { "name": "echo", "type": { - "name": "nullable", - "args": [ - { - "type": { - "name": "nothing" - } - } - ] + "name": "nothing", + "nullable": true } }, { @@ -368,14 +332,8 @@ "column": { "name": "echo", "type": { - "name": "nullable", - "args": [ - { - "type": { - "name": "nothing" - } - } - ] + "name": "nothing", + "nullable": true } } } diff --git a/internal/engine/clickhouse/testgen/testdata/subqueries/stdout.txt b/internal/engine/clickhouse/testgen/testdata/subqueries/stdout.txt index 70bd8c33f1..92a4c2fa6a 100644 --- a/internal/engine/clickhouse/testgen/testdata/subqueries/stdout.txt +++ b/internal/engine/clickhouse/testgen/testdata/subqueries/stdout.txt @@ -13,14 +13,8 @@ { "name": "tag", "type": { - "name": "nullable", - "args": [ - { - "type": { - "name": "string" - } - } - ] + "name": "string", + "nullable": true }, "table": "events" }, @@ -94,14 +88,8 @@ { "name": "tag", "type": { - "name": "nullable", - "args": [ - { - "type": { - "name": "string" - } - } - ] + "name": "string", + "nullable": true }, "table": "events" }, @@ -150,14 +138,8 @@ { "name": "user_count", "type": { - "name": "nullable", - "args": [ - { - "type": { - "name": "uint64" - } - } - ] + "name": "uint64", + "nullable": true } } ], diff --git a/internal/engine/clickhouse/testgen/testdata/types/stdout.txt b/internal/engine/clickhouse/testgen/testdata/types/stdout.txt index e3ee33ffdf..5ef32d6b93 100644 --- a/internal/engine/clickhouse/testgen/testdata/types/stdout.txt +++ b/internal/engine/clickhouse/testgen/testdata/types/stdout.txt @@ -20,14 +20,8 @@ { "name": "tag", "type": { - "name": "nullable", - "args": [ - { - "type": { - "name": "string" - } - } - ] + "name": "string", + "nullable": true }, "table": "things" }, @@ -59,14 +53,8 @@ "args": [ { "type": { - "name": "nullable", - "args": [ - { - "type": { - "name": "string" - } - } - ] + "name": "string", + "nullable": true } } ] @@ -101,14 +89,8 @@ "args": [ { "type": { - "name": "nullable", - "args": [ - { - "type": { - "name": "string" - } - } - ] + "name": "string", + "nullable": true } } ] @@ -240,14 +222,8 @@ }, { "type": { - "name": "nullable", - "args": [ - { - "type": { - "name": "uint8" - } - } - ] + "name": "uint8", + "nullable": true } } ] @@ -311,14 +287,8 @@ { "name": "tag", "type": { - "name": "nullable", - "args": [ - { - "type": { - "name": "string" - } - } - ] + "name": "string", + "nullable": true }, "table": "things" }, @@ -350,14 +320,8 @@ "args": [ { "type": { - "name": "nullable", - "args": [ - { - "type": { - "name": "string" - } - } - ] + "name": "string", + "nullable": true } } ] @@ -392,14 +356,8 @@ "args": [ { "type": { - "name": "nullable", - "args": [ - { - "type": { - "name": "string" - } - } - ] + "name": "string", + "nullable": true } } ] @@ -531,14 +489,8 @@ }, { "type": { - "name": "nullable", - "args": [ - { - "type": { - "name": "uint8" - } - } - ] + "name": "uint8", + "nullable": true } } ] diff --git a/internal/engine/clickhouse/testgen/types.go b/internal/engine/clickhouse/testgen/types.go index a12bb542db..f396d78f20 100644 --- a/internal/engine/clickhouse/testgen/types.go +++ b/internal/engine/clickhouse/testgen/types.go @@ -8,15 +8,17 @@ import ( // A type is a call expression, the way ClickHouse itself models one: a // lowercased name applied to an ordered argument list. Each argument is // another type, an integer, a boolean or a quoted string, optionally -// labelled, so -// Nullable, Array and LowCardinality are ordinary names and nothing about a -// nested type is lost. The shape maps one to one onto a protobuf message -// with a oneof for the argument value: +// labelled, so Array, Map and LowCardinality are ordinary names and nothing +// about a nested type is lost. Nullability is an attribute of a type rather +// than a wrapper, since every engine has it and only ClickHouse spells it as +// a type: Nullable(T) becomes T with nullable set, at whatever depth it +// appears. The shape maps one to one onto a protobuf message with a oneof +// for the argument value: // // Map(String, Nullable(UInt32)) // {"name": "map", "args": [ // {"type": {"name": "string"}}, -// {"type": {"name": "nullable", "args": [{"type": {"name": "uint32"}}]}}]} +// {"type": {"name": "uint32", "nullable": true}}]} // // Tuple(lat Float64, lon Float64) // {"name": "tuple", "args": [ @@ -34,8 +36,9 @@ import ( // is the reader's job; the output only records what was said. type typeExpr struct { - Name string `json:"name"` - Args []typeArg `json:"args,omitempty"` + Name string `json:"name"` + Nullable bool `json:"nullable,omitempty"` + Args []typeArg `json:"args,omitempty"` } type typeArg struct { @@ -50,6 +53,11 @@ type typeArg struct { func parseType(t string) *typeExpr { name, args := splitType(t) name = strings.ToLower(strings.TrimSpace(name)) + if name == "nullable" && len(args) == 1 { + expr := parseType(args[0]) + expr.Nullable = true + return expr + } if name == "" { name = "nothing" } From 5e121a167a3b07f71f975b50c591c3a2a179595a Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 2 Sep 2026 19:39:42 +0000 Subject: [PATCH 06/10] Name testgen goldens analyze.json so editors highlight them as JSON Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01Y6XkyWnx7iJFEb8q3AnYps --- internal/engine/clickhouse/testgen/README.md | 4 ++-- .../testdata/analyze_params/{stdout.txt => analyze.json} | 0 .../testgen/testdata/exec/{stdout.txt => analyze.json} | 0 .../testdata/expressions/{stdout.txt => analyze.json} | 0 .../testdata/subqueries/{stdout.txt => analyze.json} | 0 .../testgen/testdata/types/{stdout.txt => analyze.json} | 0 internal/engine/clickhouse/testgen/testgen_test.go | 6 +++--- 7 files changed, 5 insertions(+), 5 deletions(-) rename internal/engine/clickhouse/testgen/testdata/analyze_params/{stdout.txt => analyze.json} (100%) rename internal/engine/clickhouse/testgen/testdata/exec/{stdout.txt => analyze.json} (100%) rename internal/engine/clickhouse/testgen/testdata/expressions/{stdout.txt => analyze.json} (100%) rename internal/engine/clickhouse/testgen/testdata/subqueries/{stdout.txt => analyze.json} (100%) rename internal/engine/clickhouse/testgen/testdata/types/{stdout.txt => analyze.json} (100%) diff --git a/internal/engine/clickhouse/testgen/README.md b/internal/engine/clickhouse/testgen/README.md index 8f54d98fe2..87b32a2f40 100644 --- a/internal/engine/clickhouse/testgen/README.md +++ b/internal/engine/clickhouse/testgen/README.md @@ -69,11 +69,11 @@ whoever reads the output. ## Tests `testdata//` holds a `schema.sql`, `query.sql`, an optional -`fixture.sql` and the expected `stdout.txt`. The test skips unless a binary is +`fixture.sql` and the expected `analyze.json`. The test skips unless a binary is available. ```bash go run . install go test . -go test . -update # rewrite every stdout.txt +go test . -update # rewrite every analyze.json ``` diff --git a/internal/engine/clickhouse/testgen/testdata/analyze_params/stdout.txt b/internal/engine/clickhouse/testgen/testdata/analyze_params/analyze.json similarity index 100% rename from internal/engine/clickhouse/testgen/testdata/analyze_params/stdout.txt rename to internal/engine/clickhouse/testgen/testdata/analyze_params/analyze.json diff --git a/internal/engine/clickhouse/testgen/testdata/exec/stdout.txt b/internal/engine/clickhouse/testgen/testdata/exec/analyze.json similarity index 100% rename from internal/engine/clickhouse/testgen/testdata/exec/stdout.txt rename to internal/engine/clickhouse/testgen/testdata/exec/analyze.json diff --git a/internal/engine/clickhouse/testgen/testdata/expressions/stdout.txt b/internal/engine/clickhouse/testgen/testdata/expressions/analyze.json similarity index 100% rename from internal/engine/clickhouse/testgen/testdata/expressions/stdout.txt rename to internal/engine/clickhouse/testgen/testdata/expressions/analyze.json diff --git a/internal/engine/clickhouse/testgen/testdata/subqueries/stdout.txt b/internal/engine/clickhouse/testgen/testdata/subqueries/analyze.json similarity index 100% rename from internal/engine/clickhouse/testgen/testdata/subqueries/stdout.txt rename to internal/engine/clickhouse/testgen/testdata/subqueries/analyze.json diff --git a/internal/engine/clickhouse/testgen/testdata/types/stdout.txt b/internal/engine/clickhouse/testgen/testdata/types/analyze.json similarity index 100% rename from internal/engine/clickhouse/testgen/testdata/types/stdout.txt rename to internal/engine/clickhouse/testgen/testdata/types/analyze.json diff --git a/internal/engine/clickhouse/testgen/testgen_test.go b/internal/engine/clickhouse/testgen/testgen_test.go index 0cca493058..a7d0e588ae 100644 --- a/internal/engine/clickhouse/testgen/testgen_test.go +++ b/internal/engine/clickhouse/testgen/testgen_test.go @@ -9,11 +9,11 @@ import ( "testing" ) -var update = flag.Bool("update", false, "rewrite the expected stdout.txt of every case") +var update = flag.Bool("update", false, "rewrite the expected analyze.json of every case") // TestAnalyze runs the CLI over each directory under testdata, which holds // the same files a sqlc analyze case does plus a fixture, and compares the -// output with the committed stdout.txt. It needs the clickhouse binary and +// output with the committed analyze.json. It needs the clickhouse binary and // skips when none is installed. func TestAnalyze(t *testing.T) { if _, err := Locate(); err != nil { @@ -36,7 +36,7 @@ func TestAnalyze(t *testing.T) { t.Fatalf("%v\n%s", err, stderr.String()) } - golden := filepath.Join(dir, "stdout.txt") + golden := filepath.Join(dir, "analyze.json") if *update { if err := os.WriteFile(golden, stdout.Bytes(), 0o644); err != nil { t.Fatal(err) From 480021831e164a0cbaacf7e672a578fd198c5f25 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 2 Sep 2026 19:45:13 +0000 Subject: [PATCH 07/10] Verify testgen downloads against a SHA-512 asset table releaseAsset now looks a build up in a table of version, platform, file name and SHA-512, and Install hashes every byte off the wire, including the tail of a tarball past the binary, and discards a download whose digest does not match. A version missing from the table cannot be installed. The tarball digests are the ones ClickHouse publishes in its .sha512 sidecar files; the macOS binaries have no published digest, so theirs were computed from the downloads. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01Y6XkyWnx7iJFEb8q3AnYps --- internal/engine/clickhouse/testgen/README.md | 9 +- internal/engine/clickhouse/testgen/install.go | 105 +++++++++++++----- 2 files changed, 83 insertions(+), 31 deletions(-) diff --git a/internal/engine/clickhouse/testgen/README.md b/internal/engine/clickhouse/testgen/README.md index 87b32a2f40..6ca09c6b59 100644 --- a/internal/engine/clickhouse/testgen/README.md +++ b/internal/engine/clickhouse/testgen/README.md @@ -16,9 +16,12 @@ go run . analyze --schema schema.sql --fixture fixture.sql query.sql ``` The binary is looked up in the `CLICKHOUSE` environment variable first, then -in the cache populated by `install`. The version is pinned in `install.go`; -bumping it can change the query tree format and type inference, so regenerate -and review the goldens afterwards. +in the cache populated by `install`. The version is pinned in `install.go`, +whose asset table lists each platform's download and its SHA-512; a download +that does not match is discarded. Bumping the version means adding the new +release's assets and checksums to the table, and since a release can change +the query tree format and type inference, regenerating and reviewing the +goldens afterwards. ## How it works diff --git a/internal/engine/clickhouse/testgen/install.go b/internal/engine/clickhouse/testgen/install.go index 6eb7dde49e..82cd53a55e 100644 --- a/internal/engine/clickhouse/testgen/install.go +++ b/internal/engine/clickhouse/testgen/install.go @@ -4,6 +4,8 @@ import ( "archive/tar" "compress/gzip" "context" + "crypto/sha512" + "encoding/hex" "errors" "fmt" "io" @@ -17,9 +19,34 @@ import ( // DefaultVersion is the ClickHouse release the goldens are generated with. // Bumping it is a deliberate change: the query tree format and type // inference can shift between releases, so regenerate and review the goldens -// after changing it. +// after changing it, and add the new release's assets to the table below. const DefaultVersion = "25.8.2.29" +// asset is one downloadable build of ClickHouse. Linux builds are published +// as clickhouse-common-static tarballs holding the binary at +// usr/bin/clickhouse; macOS builds are published as bare binaries. +type asset struct { + Version string + OS string + Arch string + Name string + SHA512 string +} + +// assets lists every build Install knows how to fetch, with the SHA-512 of +// the download. A version that is not in this table cannot be installed: +// verifying the download is the point of the table. +// +// The tarball checksums are the ones in the .sha512 files ClickHouse +// publishes next to them. ClickHouse publishes no checksum for the macOS +// binaries, so those were computed from the downloads. +var assets = []asset{ + {"25.8.2.29", "linux", "amd64", "clickhouse-common-static-25.8.2.29-amd64.tgz", "6ff0aa1ffac6e564970174422ecde0d645cdb96812247a6e544d39cad6d78a514265f90a2bc7b4bad49903cea96eddd16a415a45b2aeaf9164461be76331bdee"}, + {"25.8.2.29", "linux", "arm64", "clickhouse-common-static-25.8.2.29-arm64.tgz", "68204ca4d4e472790f808ee376251fae82e58066a31f35a40d15d442ce5988d697f18a1208d28b8bb8e2dfad4b20b7fcb5107e2178472abcd97251b8de7f058e"}, + {"25.8.2.29", "darwin", "amd64", "clickhouse-macos", "2805805ad2506e37a3e71b4ae9e797bdc010a9368dc28e99bcaaa2c70a72cfdd031c0fce8fc304248fad73211d28d645f75c6432cfae1e1e54d72d04e8626cd4"}, + {"25.8.2.29", "darwin", "arm64", "clickhouse-macos-aarch64", "4c9237e85c8d4e1aced2b339b32e086b4f23fa14f99754b056a7e33dda88c4fb5d52e09e29d20181ec5b580caadb00f36613802e71755ff34927535eb8babf79"}, +} + // releaseTag returns the GitHub release tag for a version. ClickHouse tags // its March and August releases as LTS and everything else as stable. func releaseTag(version string) (string, error) { @@ -38,27 +65,33 @@ func releaseTag(version string) (string, error) { return "v" + version + suffix, nil } -// releaseAsset returns the download URL for a platform and whether it is a -// tarball holding the binary at usr/bin/clickhouse rather than the bare -// binary. Linux builds are only published as tarballs; macOS builds only as -// bare binaries. -func releaseAsset(version, goos, goarch string) (url string, tarball bool, err error) { - tag, err := releaseTag(version) +// releaseAsset finds the build for a platform in the table. +func releaseAsset(version, goos, goarch string) (asset, error) { + for _, a := range assets { + if a.Version == version && a.OS == goos && a.Arch == goarch { + return a, nil + } + } + for _, a := range assets { + if a.Version == version { + return asset{}, fmt.Errorf("no ClickHouse %s build is listed for %s/%s", version, goos, goarch) + } + } + return asset{}, fmt.Errorf("ClickHouse %s is not in the asset table; add its downloads and checksums to install.go", version) +} + +// url is the asset's download address on GitHub. +func (a asset) url() (string, error) { + tag, err := releaseTag(a.Version) if err != nil { - return "", false, err - } - base := "https://github.com/ClickHouse/ClickHouse/releases/download/" + tag + "/" - switch goos + "/" + goarch { - case "linux/amd64": - return base + "clickhouse-common-static-" + version + "-amd64.tgz", true, nil - case "linux/arm64": - return base + "clickhouse-common-static-" + version + "-arm64.tgz", true, nil - case "darwin/amd64": - return base + "clickhouse-macos", false, nil - case "darwin/arm64": - return base + "clickhouse-macos-aarch64", false, nil - } - return "", false, fmt.Errorf("no ClickHouse build is published for %s/%s", goos, goarch) + return "", err + } + return "https://github.com/ClickHouse/ClickHouse/releases/download/" + tag + "/" + a.Name, nil +} + +// tarball reports whether the download is an archive rather than the binary. +func (a asset) tarball() bool { + return strings.HasSuffix(a.Name, ".tgz") } // cachedBinary is where Install puts the binary for a version. @@ -87,7 +120,8 @@ func Locate() (string, error) { } // Install downloads the clickhouse binary for a version into the cache and -// returns its path. It is a no-op when the version is already cached. +// returns its path. It is a no-op when the version is already cached. The +// download is checked against the table's SHA-512 before it is installed. func Install(ctx context.Context, version, goos, goarch string, progress io.Writer) (string, error) { dest, err := cachedBinary(version) if err != nil { @@ -96,7 +130,11 @@ func Install(ctx context.Context, version, goos, goarch string, progress io.Writ if _, err := os.Stat(dest); err == nil { return dest, nil } - url, tarball, err := releaseAsset(version, goos, goarch) + a, err := releaseAsset(version, goos, goarch) + if err != nil { + return "", err + } + url, err := a.url() if err != nil { return "", err } @@ -118,17 +156,21 @@ func Install(ctx context.Context, version, goos, goarch string, progress io.Writ return "", fmt.Errorf("downloading %s: %s", url, resp.Status) } - // Write next to the destination and rename so a partial download never - // masquerades as an installed binary. + // Write next to the destination and rename so a partial or corrupt + // download never masquerades as an installed binary. tmp, err := os.CreateTemp(filepath.Dir(dest), "clickhouse-*.partial") if err != nil { return "", err } defer os.Remove(tmp.Name()) - var src io.Reader = resp.Body - if tarball { - src, err = binaryInTarball(resp.Body) + // Hash every byte that comes off the wire, including the parts of a + // tarball after the binary, which extraction would otherwise not read. + sum := sha512.New() + body := io.TeeReader(resp.Body, sum) + var src io.Reader = body + if a.tarball() { + src, err = binaryInTarball(body) if err != nil { return "", fmt.Errorf("downloading %s: %w", url, err) } @@ -137,9 +179,16 @@ func Install(ctx context.Context, version, goos, goarch string, progress io.Writ tmp.Close() return "", err } + if _, err := io.Copy(io.Discard, body); err != nil { + tmp.Close() + return "", err + } if err := tmp.Close(); err != nil { return "", err } + if got := hex.EncodeToString(sum.Sum(nil)); got != a.SHA512 { + return "", fmt.Errorf("downloading %s: SHA-512 mismatch: got %s, want %s", url, got, a.SHA512) + } if err := os.Chmod(tmp.Name(), 0o755); err != nil { return "", err } From 9d0a4e8ea517dd25205c8eef413e90f6161bec18 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 2 Sep 2026 20:22:47 +0000 Subject: [PATCH 08/10] Move ClickHouse testgen into internal/testcheck and verify end-to-end cases testcheck replaces testgen. It generates nothing: each engine package finds the analyze cases under internal/endtoend/testdata, loads a case's schema, fixture and queries into a real database, and compares what the database reports with the output.json the case committed, byte for byte. The ClickHouse package is the testgen code moved over; other engines get their own package alongside it. sqlc analyze now prints each column's type as a call expression instead of data_type, not_null and is_array, so its output and the database's answer share one format. The compiler's flat column description maps onto it as the data type wrapped in one array node per dimension with the column's nullability on the outermost node. Every analyze case's expected output is renamed from stdout.txt to output.json, which the end-to-end harness now reads first, and regenerated. The cases from testgen's testdata become analyze_types, analyze_expressions, analyze_subqueries and analyze_exec under the ClickHouse dialect, with fixture.sql next to the schema, and the existing analyze_basic and analyze_params ClickHouse cases gain fixtures. Two queries from the subqueries case, a CTE and a SELECT * over a join, are left out because sqlc cannot analyze them yet. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01Y6XkyWnx7iJFEb8q3AnYps --- CLAUDE.md | 10 +- docs/howto/analyze.md | 32 ++- internal/cmd/analyze.go | 53 +++- internal/endtoend/case_test.go | 21 +- .../postgresql/{stdout.txt => output.json} | 12 +- .../analyze_basic/clickhouse/fixture.sql | 1 + .../clickhouse/{stdout.txt => output.json} | 31 +-- .../duckdb/{stdout.txt => output.json} | 31 +-- .../googlesql/{stdout.txt => output.json} | 18 +- .../mssql/{stdout.txt => output.json} | 31 +-- .../{sqlite/stdout.txt => mysql/output.json} | 18 +- .../postgresql/{stdout.txt => output.json} | 25 +- .../{mysql/stdout.txt => sqlite/output.json} | 18 +- .../duckdb/{stdout.txt => output.json} | 69 ++--- .../googlesql/{stdout.txt => output.json} | 98 +++---- .../mssql/{stdout.txt => output.json} | 50 ++-- .../mysql/{stdout.txt => output.json} | 44 ++-- .../postgresql/{stdout.txt => output.json} | 87 ++++--- .../sqlite/{stdout.txt => output.json} | 51 ++-- .../analyze_exec/clickhouse/exec.json | 5 + .../analyze_exec/clickhouse}/fixture.sql | 0 .../analyze_exec/clickhouse/output.json} | 0 .../analyze_exec/clickhouse}/query.sql | 0 .../analyze_exec/clickhouse}/schema.sql | 0 .../analyze_expressions/clickhouse/exec.json | 5 + .../clickhouse}/fixture.sql | 0 .../clickhouse/output.json} | 137 ++-------- .../analyze_expressions/clickhouse}/query.sql | 0 .../clickhouse}/schema.sql | 0 .../postgresql/{stdout.txt => output.json} | 42 +-- .../analyze_params/clickhouse}/fixture.sql | 0 .../analyze_params/clickhouse/output.json} | 0 .../analyze_params/clickhouse/stdout.txt | 76 ------ .../duckdb/{stdout.txt => output.json} | 61 ++--- .../mssql/{stdout.txt => output.json} | 49 ++-- .../mysql/{stdout.txt => output.json} | 61 ++--- .../postgresql/{stdout.txt => output.json} | 86 +++--- .../sqlite/{stdout.txt => output.json} | 55 ++-- .../duckdb/{stdout.txt => output.json} | 74 +++--- .../googlesql/{stdout.txt => output.json} | 62 ++--- .../mysql/{stdout.txt => output.json} | 100 +++---- .../postgresql/{stdout.txt => output.json} | 100 +++---- .../sqlite/{stdout.txt => output.json} | 88 ++++--- .../analyze_subqueries/clickhouse/exec.json | 5 + .../clickhouse}/fixture.sql | 0 .../analyze_subqueries/clickhouse/output.json | 63 +++++ .../analyze_subqueries/clickhouse}/query.sql | 10 +- .../analyze_subqueries/clickhouse}/schema.sql | 0 .../postgresql/{stdout.txt => output.json} | 40 +-- .../analyze_types/clickhouse/exec.json | 5 + .../analyze_types/clickhouse}/fixture.sql | 0 .../analyze_types/clickhouse/output.json} | 244 ++---------------- .../analyze_types/clickhouse}/query.sql | 0 .../analyze_types/clickhouse}/schema.sql | 0 internal/engine/clickhouse/testgen/README.md | 82 ------ internal/engine/clickhouse/testgen/go.mod | 3 - internal/engine/clickhouse/testgen/main.go | 132 ---------- .../testgen/testdata/analyze_params/query.sql | 5 - .../testdata/analyze_params/schema.sql | 7 - .../testgen/testdata/subqueries/analyze.json | 159 ------------ .../engine/clickhouse/testgen/testgen_test.go | 55 ---- internal/testcheck/README.md | 40 +++ .../clickhouse}/analyze.go | 6 +- internal/testcheck/clickhouse/check.go | 68 +++++ .../testcheck/clickhouse/clickhouse_test.go | 35 +++ .../clickhouse}/install.go | 4 +- .../testgen => testcheck/clickhouse}/local.go | 2 +- .../clickhouse}/queries.go | 2 +- .../testgen => testcheck/clickhouse}/tree.go | 2 +- .../testgen => testcheck/clickhouse}/types.go | 2 +- internal/testcheck/endtoend/endtoend.go | 180 +++++++++++++ internal/testcheck/go.mod | 3 + internal/testcheck/main.go | 116 +++++++++ 73 files changed, 1383 insertions(+), 1588 deletions(-) rename internal/endtoend/testdata/analyze_ast/postgresql/{stdout.txt => output.json} (93%) create mode 100644 internal/endtoend/testdata/analyze_basic/clickhouse/fixture.sql rename internal/endtoend/testdata/analyze_basic/clickhouse/{stdout.txt => output.json} (50%) rename internal/endtoend/testdata/analyze_basic/duckdb/{stdout.txt => output.json} (51%) rename internal/endtoend/testdata/analyze_basic/googlesql/{stdout.txt => output.json} (57%) rename internal/endtoend/testdata/analyze_basic/mssql/{stdout.txt => output.json} (50%) rename internal/endtoend/testdata/analyze_basic/{sqlite/stdout.txt => mysql/output.json} (57%) rename internal/endtoend/testdata/analyze_basic/postgresql/{stdout.txt => output.json} (55%) rename internal/endtoend/testdata/analyze_basic/{mysql/stdout.txt => sqlite/output.json} (57%) rename internal/endtoend/testdata/analyze_dml/duckdb/{stdout.txt => output.json} (62%) rename internal/endtoend/testdata/analyze_dml/googlesql/{stdout.txt => output.json} (61%) rename internal/endtoend/testdata/analyze_dml/mssql/{stdout.txt => output.json} (64%) rename internal/endtoend/testdata/analyze_dml/mysql/{stdout.txt => output.json} (66%) rename internal/endtoend/testdata/analyze_dml/postgresql/{stdout.txt => output.json} (60%) rename internal/endtoend/testdata/analyze_dml/sqlite/{stdout.txt => output.json} (60%) create mode 100644 internal/endtoend/testdata/analyze_exec/clickhouse/exec.json rename internal/{engine/clickhouse/testgen/testdata/exec => endtoend/testdata/analyze_exec/clickhouse}/fixture.sql (100%) rename internal/{engine/clickhouse/testgen/testdata/exec/analyze.json => endtoend/testdata/analyze_exec/clickhouse/output.json} (100%) rename internal/{engine/clickhouse/testgen/testdata/exec => endtoend/testdata/analyze_exec/clickhouse}/query.sql (100%) rename internal/{engine/clickhouse/testgen/testdata/exec => endtoend/testdata/analyze_exec/clickhouse}/schema.sql (100%) create mode 100644 internal/endtoend/testdata/analyze_expressions/clickhouse/exec.json rename internal/{engine/clickhouse/testgen/testdata/expressions => endtoend/testdata/analyze_expressions/clickhouse}/fixture.sql (100%) rename internal/{engine/clickhouse/testgen/testdata/expressions/analyze.json => endtoend/testdata/analyze_expressions/clickhouse/output.json} (58%) rename internal/{engine/clickhouse/testgen/testdata/expressions => endtoend/testdata/analyze_expressions/clickhouse}/query.sql (100%) rename internal/{engine/clickhouse/testgen/testdata/expressions => endtoend/testdata/analyze_expressions/clickhouse}/schema.sql (100%) rename internal/endtoend/testdata/analyze_extension/postgresql/{stdout.txt => output.json} (57%) rename internal/{engine/clickhouse/testgen/testdata/analyze_params => endtoend/testdata/analyze_params/clickhouse}/fixture.sql (100%) rename internal/{engine/clickhouse/testgen/testdata/analyze_params/analyze.json => endtoend/testdata/analyze_params/clickhouse/output.json} (100%) delete mode 100644 internal/endtoend/testdata/analyze_params/clickhouse/stdout.txt rename internal/endtoend/testdata/analyze_params/duckdb/{stdout.txt => output.json} (58%) rename internal/endtoend/testdata/analyze_params/mssql/{stdout.txt => output.json} (56%) rename internal/endtoend/testdata/analyze_params/mysql/{stdout.txt => output.json} (57%) rename internal/endtoend/testdata/analyze_params/postgresql/{stdout.txt => output.json} (57%) rename internal/endtoend/testdata/analyze_params/sqlite/{stdout.txt => output.json} (58%) rename internal/endtoend/testdata/analyze_select/duckdb/{stdout.txt => output.json} (54%) rename internal/endtoend/testdata/analyze_select/googlesql/{stdout.txt => output.json} (53%) rename internal/endtoend/testdata/analyze_select/mysql/{stdout.txt => output.json} (54%) rename internal/endtoend/testdata/analyze_select/postgresql/{stdout.txt => output.json} (54%) rename internal/endtoend/testdata/analyze_select/sqlite/{stdout.txt => output.json} (55%) create mode 100644 internal/endtoend/testdata/analyze_subqueries/clickhouse/exec.json rename internal/{engine/clickhouse/testgen/testdata/subqueries => endtoend/testdata/analyze_subqueries/clickhouse}/fixture.sql (100%) create mode 100644 internal/endtoend/testdata/analyze_subqueries/clickhouse/output.json rename internal/{engine/clickhouse/testgen/testdata/subqueries => endtoend/testdata/analyze_subqueries/clickhouse}/query.sql (64%) rename internal/{engine/clickhouse/testgen/testdata/subqueries => endtoend/testdata/analyze_subqueries/clickhouse}/schema.sql (100%) rename internal/endtoend/testdata/analyze_system_catalog/postgresql/{stdout.txt => output.json} (56%) create mode 100644 internal/endtoend/testdata/analyze_types/clickhouse/exec.json rename internal/{engine/clickhouse/testgen/testdata/types => endtoend/testdata/analyze_types/clickhouse}/fixture.sql (100%) rename internal/{engine/clickhouse/testgen/testdata/types/analyze.json => endtoend/testdata/analyze_types/clickhouse/output.json} (54%) rename internal/{engine/clickhouse/testgen/testdata/types => endtoend/testdata/analyze_types/clickhouse}/query.sql (100%) rename internal/{engine/clickhouse/testgen/testdata/types => endtoend/testdata/analyze_types/clickhouse}/schema.sql (100%) delete mode 100644 internal/engine/clickhouse/testgen/README.md delete mode 100644 internal/engine/clickhouse/testgen/go.mod delete mode 100644 internal/engine/clickhouse/testgen/main.go delete mode 100644 internal/engine/clickhouse/testgen/testdata/analyze_params/query.sql delete mode 100644 internal/engine/clickhouse/testgen/testdata/analyze_params/schema.sql delete mode 100644 internal/engine/clickhouse/testgen/testdata/subqueries/analyze.json delete mode 100644 internal/engine/clickhouse/testgen/testgen_test.go create mode 100644 internal/testcheck/README.md rename internal/{engine/clickhouse/testgen => testcheck/clickhouse}/analyze.go (98%) create mode 100644 internal/testcheck/clickhouse/check.go create mode 100644 internal/testcheck/clickhouse/clickhouse_test.go rename internal/{engine/clickhouse/testgen => testcheck/clickhouse}/install.go (97%) rename internal/{engine/clickhouse/testgen => testcheck/clickhouse}/local.go (99%) rename internal/{engine/clickhouse/testgen => testcheck/clickhouse}/queries.go (99%) rename internal/{engine/clickhouse/testgen => testcheck/clickhouse}/tree.go (99%) rename internal/{engine/clickhouse/testgen => testcheck/clickhouse}/types.go (99%) create mode 100644 internal/testcheck/endtoend/endtoend.go create mode 100644 internal/testcheck/go.mod create mode 100644 internal/testcheck/main.go diff --git a/CLAUDE.md b/CLAUDE.md index 6ac48badce..cbfa9892a2 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -120,8 +120,8 @@ A case is a directory holding the inputs and the expected output. `exec.json` names the command and its arguments — omit it and the case runs `generate`, comparing the generated files against the ones committed alongside; give it `{"command": "analyze", "args": [...]}` and the case compares the command's -stdout against `stdout.txt`. A case that is expected to fail commits its -`stderr.txt`. Regenerate a golden by running the command in its directory and +stdout against `output.json` (or `stdout.txt` for a command that does not +print JSON). A case that is expected to fail commits its `stderr.txt`. Regenerate a golden by running the command in its directory and writing the output back over the committed file. `TestReplay` runs the whole corpus once per *context*. `base` runs each case as @@ -233,9 +233,6 @@ MYSQL_SERVER_URI="root:mysecretpassword@tcp(127.0.0.1:3306)/mysql?multiStatement - `/postgresql/` - PostgreSQL parser and converter - `/dolphin/` - MySQL parser (uses TiDB parser) - `/sqlite/` - SQLite parser - - `/clickhouse/testgen/` - Nested module that records what a real - ClickHouse reports about a schema, fixture and queries, in the shape of - `sqlc analyze` output with types as call expressions; see its README - `/duckdb/` - DuckDB 2.0 parser (uses darkwing, the pure Go port of DuckDB's PEG parser) - `/dialect/` - The engine's type system and standard library, as @@ -249,6 +246,9 @@ MYSQL_SERVER_URI="root:mysecretpassword@tcp(127.0.0.1:3306)/mysql?multiStatement - `/internal/codegen/` - Code generation for different languages - `/internal/config/` - Configuration file parsing - `/internal/endtoend/` - End-to-end tests +- `/internal/testcheck/` - Nested module that verifies the analyze cases under + `/internal/endtoend/testdata/` against a real database, one package per + engine; see its README - `/internal/sqltest/` - Test database setup (Docker, native, local detection) - `/examples/` - Example projects for testing diff --git a/docs/howto/analyze.md b/docs/howto/analyze.md index 87541d093e..325acbd6f6 100644 --- a/docs/howto/analyze.md +++ b/docs/howto/analyze.md @@ -70,23 +70,24 @@ reports the result columns and parameters: "columns": [ { "name": "id", - "data_type": "bigserial", - "not_null": true, - "is_array": false, + "type": { + "name": "bigserial" + }, "table": "authors" }, { "name": "name", - "data_type": "text", - "not_null": true, - "is_array": false, + "type": { + "name": "text" + }, "table": "authors" }, { "name": "bio", - "data_type": "text", - "not_null": false, - "is_array": false, + "type": { + "name": "text", + "nullable": true + }, "table": "authors" } ], @@ -95,9 +96,9 @@ reports the result columns and parameters: "number": 1, "column": { "name": "id", - "data_type": "bigserial", - "not_null": true, - "is_array": false, + "type": { + "name": "bigserial" + }, "table": "authors" } } @@ -106,6 +107,13 @@ reports the result columns and parameters: ] ``` +A column's `type` is written as a call expression: a `name` applied to +`args`, each of which carries an optional `label` and exactly one of `type`, +`int`, `bool` or `string`, with `nullable` set at whatever depth it applies. +An array of text is `array` applied to `text`; a `Map(String, Nullable(UInt8))` +in ClickHouse is `map` applied to `string` and a nullable `uint8`. Names are +recorded as the engine reports them. + Pass `--ast` to also include each statement's parsed AST under an `ast` key. It has the same shape as the output of [`parse`](parse.md), with every node tagged by type. diff --git a/internal/cmd/analyze.go b/internal/cmd/analyze.go index df71bae0b4..7d52b3fe2f 100644 --- a/internal/cmd/analyze.go +++ b/internal/cmd/analyze.go @@ -204,11 +204,9 @@ type analyzedQuery struct { } type analyzedColumn struct { - Name string `json:"name"` - DataType string `json:"data_type"` - NotNull bool `json:"not_null"` - IsArray bool `json:"is_array"` - Table string `json:"table,omitempty"` + Name string `json:"name"` + Type *analyzedType `json:"type,omitempty"` + Table string `json:"table,omitempty"` } type analyzedParam struct { @@ -216,6 +214,26 @@ type analyzedParam struct { Column analyzedColumn `json:"column"` } +// analyzedType writes a type as a call expression: a name applied to +// arguments that are other types, integers, booleans or strings, each with +// an optional label, and a nullable flag at whatever depth it applies. An +// array of text is array(text); a nullable column of it has nullable set on +// the array node. Names are recorded as the engine reports them and resolve +// against the catalog afterwards. +type analyzedType struct { + Name string `json:"name"` + Nullable bool `json:"nullable,omitempty"` + Args []analyzedArg `json:"args,omitempty"` +} + +type analyzedArg struct { + Label string `json:"label,omitempty"` + Type *analyzedType `json:"type,omitempty"` + Int *int64 `json:"int,omitempty"` + Bool *bool `json:"bool,omitempty"` + String *string `json:"string,omitempty"` +} + func newAnalyzedQuery(q *compiler.Query, includeAST bool) analyzedQuery { aq := analyzedQuery{ Name: q.Metadata.Name, @@ -243,13 +261,30 @@ func newAnalyzedColumn(col *compiler.Column) analyzedColumn { return analyzedColumn{} } ac := analyzedColumn{ - Name: col.Name, - DataType: col.DataType, - NotNull: col.NotNull, - IsArray: col.IsArray, + Name: col.Name, + Type: newAnalyzedType(col), } if col.Table != nil { ac.Table = col.Table.Name } return ac } + +// newAnalyzedType builds the type expression the compiler's flat column +// description amounts to: the data type wrapped in one array node per +// dimension, with the column's nullability on the outermost node. +func newAnalyzedType(col *compiler.Column) *analyzedType { + if col.DataType == "" { + return nil + } + t := &analyzedType{Name: col.DataType} + dims := col.ArrayDims + if col.IsArray && dims == 0 { + dims = 1 + } + for i := 0; i < dims; i++ { + t = &analyzedType{Name: "array", Args: []analyzedArg{{Type: t}}} + } + t.Nullable = !col.NotNull + return t +} diff --git a/internal/endtoend/case_test.go b/internal/endtoend/case_test.go index 183b965a2a..0fb5e8f100 100644 --- a/internal/endtoend/case_test.go +++ b/internal/endtoend/case_test.go @@ -52,17 +52,22 @@ func parseStderr(t *testing.T, dir, testctx string) []byte { return nil } +// parseStdout reads the command's expected output: output.json for a +// command that prints JSON, so editors highlight it, otherwise stdout.txt. func parseStdout(t *testing.T, dir string) []byte { t.Helper() - path := filepath.Join(dir, "stdout.txt") - if _, err := os.Stat(path); os.IsNotExist(err) { - return nil - } - blob, err := os.ReadFile(path) - if err != nil { - t.Fatal(err) + for _, name := range []string{"output.json", "stdout.txt"} { + path := filepath.Join(dir, name) + if _, err := os.Stat(path); os.IsNotExist(err) { + continue + } + blob, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + return blob } - return blob + return nil } // hasSQLCConfig reports whether dir contains an sqlc configuration file. diff --git a/internal/endtoend/testdata/analyze_ast/postgresql/stdout.txt b/internal/endtoend/testdata/analyze_ast/postgresql/output.json similarity index 93% rename from internal/endtoend/testdata/analyze_ast/postgresql/stdout.txt rename to internal/endtoend/testdata/analyze_ast/postgresql/output.json index 1473a99cc5..427fe5af92 100644 --- a/internal/endtoend/testdata/analyze_ast/postgresql/stdout.txt +++ b/internal/endtoend/testdata/analyze_ast/postgresql/output.json @@ -5,9 +5,9 @@ "columns": [ { "name": "name", - "data_type": "text", - "not_null": true, - "is_array": false, + "type": { + "name": "text" + }, "table": "authors" } ], @@ -16,9 +16,9 @@ "number": 1, "column": { "name": "id", - "data_type": "bigserial", - "not_null": true, - "is_array": false, + "type": { + "name": "bigserial" + }, "table": "authors" } } diff --git a/internal/endtoend/testdata/analyze_basic/clickhouse/fixture.sql b/internal/endtoend/testdata/analyze_basic/clickhouse/fixture.sql new file mode 100644 index 0000000000..88c30875e9 --- /dev/null +++ b/internal/endtoend/testdata/analyze_basic/clickhouse/fixture.sql @@ -0,0 +1 @@ +INSERT INTO events (id, name, tag, amount, created) VALUES (1, 'signup', NULL, 9.5, '2024-01-01 00:00:00'); diff --git a/internal/endtoend/testdata/analyze_basic/clickhouse/stdout.txt b/internal/endtoend/testdata/analyze_basic/clickhouse/output.json similarity index 50% rename from internal/endtoend/testdata/analyze_basic/clickhouse/stdout.txt rename to internal/endtoend/testdata/analyze_basic/clickhouse/output.json index 11e3a90d5d..ad9963d7e7 100644 --- a/internal/endtoend/testdata/analyze_basic/clickhouse/stdout.txt +++ b/internal/endtoend/testdata/analyze_basic/clickhouse/output.json @@ -5,37 +5,38 @@ "columns": [ { "name": "id", - "data_type": "uint64", - "not_null": true, - "is_array": false, + "type": { + "name": "uint64" + }, "table": "events" }, { "name": "name", - "data_type": "string", - "not_null": true, - "is_array": false, + "type": { + "name": "string" + }, "table": "events" }, { "name": "tag", - "data_type": "string", - "not_null": false, - "is_array": false, + "type": { + "name": "string", + "nullable": true + }, "table": "events" }, { "name": "amount", - "data_type": "float64", - "not_null": true, - "is_array": false, + "type": { + "name": "float64" + }, "table": "events" }, { "name": "created", - "data_type": "datetime", - "not_null": true, - "is_array": false, + "type": { + "name": "datetime" + }, "table": "events" } ], diff --git a/internal/endtoend/testdata/analyze_basic/duckdb/stdout.txt b/internal/endtoend/testdata/analyze_basic/duckdb/output.json similarity index 51% rename from internal/endtoend/testdata/analyze_basic/duckdb/stdout.txt rename to internal/endtoend/testdata/analyze_basic/duckdb/output.json index 597820269b..6b7631c5de 100644 --- a/internal/endtoend/testdata/analyze_basic/duckdb/stdout.txt +++ b/internal/endtoend/testdata/analyze_basic/duckdb/output.json @@ -5,37 +5,38 @@ "columns": [ { "name": "id", - "data_type": "bigint", - "not_null": true, - "is_array": false, + "type": { + "name": "bigint" + }, "table": "authors" }, { "name": "name", - "data_type": "text", - "not_null": true, - "is_array": false, + "type": { + "name": "text" + }, "table": "authors" }, { "name": "bio", - "data_type": "text", - "not_null": false, - "is_array": false, + "type": { + "name": "text", + "nullable": true + }, "table": "authors" }, { "name": "royalties", - "data_type": "decimal", - "not_null": true, - "is_array": false, + "type": { + "name": "decimal" + }, "table": "authors" }, { "name": "created", - "data_type": "timestamp", - "not_null": true, - "is_array": false, + "type": { + "name": "timestamp" + }, "table": "authors" } ], diff --git a/internal/endtoend/testdata/analyze_basic/googlesql/stdout.txt b/internal/endtoend/testdata/analyze_basic/googlesql/output.json similarity index 57% rename from internal/endtoend/testdata/analyze_basic/googlesql/stdout.txt rename to internal/endtoend/testdata/analyze_basic/googlesql/output.json index 4075f80992..b14d4a249e 100644 --- a/internal/endtoend/testdata/analyze_basic/googlesql/stdout.txt +++ b/internal/endtoend/testdata/analyze_basic/googlesql/output.json @@ -5,16 +5,16 @@ "columns": [ { "name": "id", - "data_type": "int64", - "not_null": true, - "is_array": false, + "type": { + "name": "int64" + }, "table": "users" }, { "name": "name", - "data_type": "string", - "not_null": true, - "is_array": false, + "type": { + "name": "string" + }, "table": "users" } ], @@ -23,9 +23,9 @@ "number": 1, "column": { "name": "id", - "data_type": "int64", - "not_null": true, - "is_array": false, + "type": { + "name": "int64" + }, "table": "users" } } diff --git a/internal/endtoend/testdata/analyze_basic/mssql/stdout.txt b/internal/endtoend/testdata/analyze_basic/mssql/output.json similarity index 50% rename from internal/endtoend/testdata/analyze_basic/mssql/stdout.txt rename to internal/endtoend/testdata/analyze_basic/mssql/output.json index 7226354b09..181660561a 100644 --- a/internal/endtoend/testdata/analyze_basic/mssql/stdout.txt +++ b/internal/endtoend/testdata/analyze_basic/mssql/output.json @@ -5,37 +5,38 @@ "columns": [ { "name": "id", - "data_type": "bigint", - "not_null": true, - "is_array": false, + "type": { + "name": "bigint" + }, "table": "authors" }, { "name": "name", - "data_type": "nvarchar", - "not_null": true, - "is_array": false, + "type": { + "name": "nvarchar" + }, "table": "authors" }, { "name": "bio", - "data_type": "nvarchar", - "not_null": false, - "is_array": false, + "type": { + "name": "nvarchar", + "nullable": true + }, "table": "authors" }, { "name": "royalties", - "data_type": "decimal", - "not_null": true, - "is_array": false, + "type": { + "name": "decimal" + }, "table": "authors" }, { "name": "created", - "data_type": "datetime2", - "not_null": true, - "is_array": false, + "type": { + "name": "datetime2" + }, "table": "authors" } ], diff --git a/internal/endtoend/testdata/analyze_basic/sqlite/stdout.txt b/internal/endtoend/testdata/analyze_basic/mysql/output.json similarity index 57% rename from internal/endtoend/testdata/analyze_basic/sqlite/stdout.txt rename to internal/endtoend/testdata/analyze_basic/mysql/output.json index aa26281658..83dde3e339 100644 --- a/internal/endtoend/testdata/analyze_basic/sqlite/stdout.txt +++ b/internal/endtoend/testdata/analyze_basic/mysql/output.json @@ -5,16 +5,16 @@ "columns": [ { "name": "id", - "data_type": "integer", - "not_null": true, - "is_array": false, + "type": { + "name": "bigint" + }, "table": "users" }, { "name": "name", - "data_type": "text", - "not_null": true, - "is_array": false, + "type": { + "name": "varchar" + }, "table": "users" } ], @@ -23,9 +23,9 @@ "number": 1, "column": { "name": "id", - "data_type": "integer", - "not_null": true, - "is_array": false, + "type": { + "name": "bigint" + }, "table": "users" } } diff --git a/internal/endtoend/testdata/analyze_basic/postgresql/stdout.txt b/internal/endtoend/testdata/analyze_basic/postgresql/output.json similarity index 55% rename from internal/endtoend/testdata/analyze_basic/postgresql/stdout.txt rename to internal/endtoend/testdata/analyze_basic/postgresql/output.json index b93421c32a..36356b73c3 100644 --- a/internal/endtoend/testdata/analyze_basic/postgresql/stdout.txt +++ b/internal/endtoend/testdata/analyze_basic/postgresql/output.json @@ -5,23 +5,24 @@ "columns": [ { "name": "id", - "data_type": "bigserial", - "not_null": true, - "is_array": false, + "type": { + "name": "bigserial" + }, "table": "authors" }, { "name": "name", - "data_type": "text", - "not_null": true, - "is_array": false, + "type": { + "name": "text" + }, "table": "authors" }, { "name": "bio", - "data_type": "text", - "not_null": false, - "is_array": false, + "type": { + "name": "text", + "nullable": true + }, "table": "authors" } ], @@ -30,9 +31,9 @@ "number": 1, "column": { "name": "id", - "data_type": "bigserial", - "not_null": true, - "is_array": false, + "type": { + "name": "bigserial" + }, "table": "authors" } } diff --git a/internal/endtoend/testdata/analyze_basic/mysql/stdout.txt b/internal/endtoend/testdata/analyze_basic/sqlite/output.json similarity index 57% rename from internal/endtoend/testdata/analyze_basic/mysql/stdout.txt rename to internal/endtoend/testdata/analyze_basic/sqlite/output.json index e599e249aa..8799d17310 100644 --- a/internal/endtoend/testdata/analyze_basic/mysql/stdout.txt +++ b/internal/endtoend/testdata/analyze_basic/sqlite/output.json @@ -5,16 +5,16 @@ "columns": [ { "name": "id", - "data_type": "bigint", - "not_null": true, - "is_array": false, + "type": { + "name": "integer" + }, "table": "users" }, { "name": "name", - "data_type": "varchar", - "not_null": true, - "is_array": false, + "type": { + "name": "text" + }, "table": "users" } ], @@ -23,9 +23,9 @@ "number": 1, "column": { "name": "id", - "data_type": "bigint", - "not_null": true, - "is_array": false, + "type": { + "name": "integer" + }, "table": "users" } } diff --git a/internal/endtoend/testdata/analyze_dml/duckdb/stdout.txt b/internal/endtoend/testdata/analyze_dml/duckdb/output.json similarity index 62% rename from internal/endtoend/testdata/analyze_dml/duckdb/stdout.txt rename to internal/endtoend/testdata/analyze_dml/duckdb/output.json index 1ac3372918..afdd96cfd7 100644 --- a/internal/endtoend/testdata/analyze_dml/duckdb/stdout.txt +++ b/internal/endtoend/testdata/analyze_dml/duckdb/output.json @@ -5,9 +5,9 @@ "columns": [ { "name": "id", - "data_type": "bigint", - "not_null": true, - "is_array": false, + "type": { + "name": "bigint" + }, "table": "authors" } ], @@ -16,9 +16,9 @@ "number": 1, "column": { "name": "id", - "data_type": "bigint", - "not_null": true, - "is_array": false, + "type": { + "name": "bigint" + }, "table": "authors" } }, @@ -26,9 +26,9 @@ "number": 2, "column": { "name": "name", - "data_type": "text", - "not_null": true, - "is_array": false, + "type": { + "name": "text" + }, "table": "authors" } }, @@ -36,9 +36,10 @@ "number": 3, "column": { "name": "bio", - "data_type": "text", - "not_null": false, - "is_array": false, + "type": { + "name": "text", + "nullable": true + }, "table": "authors" } } @@ -53,9 +54,9 @@ "number": 1, "column": { "name": "name", - "data_type": "text", - "not_null": true, - "is_array": false, + "type": { + "name": "text" + }, "table": "authors" } }, @@ -63,9 +64,10 @@ "number": 2, "column": { "name": "bio", - "data_type": "text", - "not_null": false, - "is_array": false, + "type": { + "name": "text", + "nullable": true + }, "table": "authors" } }, @@ -73,9 +75,9 @@ "number": 3, "column": { "name": "id", - "data_type": "bigint", - "not_null": true, - "is_array": false, + "type": { + "name": "bigint" + }, "table": "authors" } } @@ -90,9 +92,9 @@ "number": 1, "column": { "name": "name", - "data_type": "text", - "not_null": true, - "is_array": false, + "type": { + "name": "text" + }, "table": "authors" } } @@ -104,16 +106,16 @@ "columns": [ { "name": "id", - "data_type": "bigint", - "not_null": true, - "is_array": false, + "type": { + "name": "bigint" + }, "table": "books" }, { "name": "title", - "data_type": "text", - "not_null": true, - "is_array": false, + "type": { + "name": "text" + }, "table": "books" } ], @@ -122,9 +124,10 @@ "number": 1, "column": { "name": "price", - "data_type": "decimal", - "not_null": false, - "is_array": false, + "type": { + "name": "decimal", + "nullable": true + }, "table": "books" } } diff --git a/internal/endtoend/testdata/analyze_dml/googlesql/stdout.txt b/internal/endtoend/testdata/analyze_dml/googlesql/output.json similarity index 61% rename from internal/endtoend/testdata/analyze_dml/googlesql/stdout.txt rename to internal/endtoend/testdata/analyze_dml/googlesql/output.json index 5148457d6d..4e696578c2 100644 --- a/internal/endtoend/testdata/analyze_dml/googlesql/stdout.txt +++ b/internal/endtoend/testdata/analyze_dml/googlesql/output.json @@ -8,9 +8,9 @@ "number": 1, "column": { "name": "id", - "data_type": "int64", - "not_null": true, - "is_array": false, + "type": { + "name": "int64" + }, "table": "users" } }, @@ -18,9 +18,9 @@ "number": 2, "column": { "name": "name", - "data_type": "string", - "not_null": true, - "is_array": false, + "type": { + "name": "string" + }, "table": "users" } }, @@ -28,9 +28,10 @@ "number": 3, "column": { "name": "bio", - "data_type": "string", - "not_null": false, - "is_array": false, + "type": { + "name": "string", + "nullable": true + }, "table": "users" } } @@ -42,16 +43,16 @@ "columns": [ { "name": "id", - "data_type": "int64", - "not_null": true, - "is_array": false, + "type": { + "name": "int64" + }, "table": "users" }, { "name": "name", - "data_type": "string", - "not_null": true, - "is_array": false, + "type": { + "name": "string" + }, "table": "users" } ], @@ -60,9 +61,9 @@ "number": 1, "column": { "name": "id", - "data_type": "int64", - "not_null": true, - "is_array": false, + "type": { + "name": "int64" + }, "table": "users" } }, @@ -70,9 +71,9 @@ "number": 2, "column": { "name": "name", - "data_type": "string", - "not_null": true, - "is_array": false, + "type": { + "name": "string" + }, "table": "users" } } @@ -87,9 +88,10 @@ "number": 1, "column": { "name": "bio", - "data_type": "string", - "not_null": false, - "is_array": false, + "type": { + "name": "string", + "nullable": true + }, "table": "users" } }, @@ -97,9 +99,9 @@ "number": 2, "column": { "name": "id", - "data_type": "int64", - "not_null": true, - "is_array": false, + "type": { + "name": "int64" + }, "table": "users" } } @@ -111,16 +113,16 @@ "columns": [ { "name": "id", - "data_type": "int64", - "not_null": true, - "is_array": false, + "type": { + "name": "int64" + }, "table": "users" }, { "name": "name", - "data_type": "string", - "not_null": true, - "is_array": false, + "type": { + "name": "string" + }, "table": "users" } ], @@ -129,9 +131,9 @@ "number": 1, "column": { "name": "name", - "data_type": "string", - "not_null": true, - "is_array": false, + "type": { + "name": "string" + }, "table": "users" } }, @@ -139,9 +141,9 @@ "number": 2, "column": { "name": "id", - "data_type": "int64", - "not_null": true, - "is_array": false, + "type": { + "name": "int64" + }, "table": "users" } } @@ -156,9 +158,9 @@ "number": 1, "column": { "name": "id", - "data_type": "int64", - "not_null": true, - "is_array": false, + "type": { + "name": "int64" + }, "table": "users" } } @@ -170,9 +172,9 @@ "columns": [ { "name": "id", - "data_type": "int64", - "not_null": true, - "is_array": false, + "type": { + "name": "int64" + }, "table": "users" } ], @@ -181,9 +183,9 @@ "number": 1, "column": { "name": "id", - "data_type": "int64", - "not_null": true, - "is_array": false, + "type": { + "name": "int64" + }, "table": "users" } } diff --git a/internal/endtoend/testdata/analyze_dml/mssql/stdout.txt b/internal/endtoend/testdata/analyze_dml/mssql/output.json similarity index 64% rename from internal/endtoend/testdata/analyze_dml/mssql/stdout.txt rename to internal/endtoend/testdata/analyze_dml/mssql/output.json index d2ed5f27e0..24e2efb3e4 100644 --- a/internal/endtoend/testdata/analyze_dml/mssql/stdout.txt +++ b/internal/endtoend/testdata/analyze_dml/mssql/output.json @@ -5,9 +5,9 @@ "columns": [ { "name": "id", - "data_type": "bigint", - "not_null": true, - "is_array": false, + "type": { + "name": "bigint" + }, "table": "authors" } ], @@ -16,9 +16,9 @@ "number": 1, "column": { "name": "name", - "data_type": "nvarchar", - "not_null": true, - "is_array": false, + "type": { + "name": "nvarchar" + }, "table": "authors" } }, @@ -26,9 +26,10 @@ "number": 2, "column": { "name": "bio", - "data_type": "nvarchar", - "not_null": false, - "is_array": false, + "type": { + "name": "nvarchar", + "nullable": true + }, "table": "authors" } } @@ -43,9 +44,9 @@ "number": 1, "column": { "name": "name", - "data_type": "nvarchar", - "not_null": true, - "is_array": false, + "type": { + "name": "nvarchar" + }, "table": "authors" } }, @@ -53,9 +54,9 @@ "number": 2, "column": { "name": "id", - "data_type": "bigint", - "not_null": true, - "is_array": false, + "type": { + "name": "bigint" + }, "table": "authors" } } @@ -70,9 +71,10 @@ "number": 1, "column": { "name": "price", - "data_type": "decimal", - "not_null": false, - "is_array": false, + "type": { + "name": "decimal", + "nullable": true + }, "table": "books" } }, @@ -80,9 +82,9 @@ "number": 2, "column": { "name": "name", - "data_type": "nvarchar", - "not_null": true, - "is_array": false, + "type": { + "name": "nvarchar" + }, "table": "authors" } } @@ -97,9 +99,9 @@ "number": 1, "column": { "name": "name", - "data_type": "nvarchar", - "not_null": true, - "is_array": false, + "type": { + "name": "nvarchar" + }, "table": "authors" } } diff --git a/internal/endtoend/testdata/analyze_dml/mysql/stdout.txt b/internal/endtoend/testdata/analyze_dml/mysql/output.json similarity index 66% rename from internal/endtoend/testdata/analyze_dml/mysql/stdout.txt rename to internal/endtoend/testdata/analyze_dml/mysql/output.json index abaa295663..e474c04a04 100644 --- a/internal/endtoend/testdata/analyze_dml/mysql/stdout.txt +++ b/internal/endtoend/testdata/analyze_dml/mysql/output.json @@ -8,9 +8,9 @@ "number": 1, "column": { "name": "id", - "data_type": "bigint", - "not_null": true, - "is_array": false, + "type": { + "name": "bigint" + }, "table": "users" } }, @@ -18,9 +18,9 @@ "number": 2, "column": { "name": "name", - "data_type": "varchar", - "not_null": true, - "is_array": false, + "type": { + "name": "varchar" + }, "table": "users" } }, @@ -28,9 +28,10 @@ "number": 3, "column": { "name": "bio", - "data_type": "text", - "not_null": false, - "is_array": false, + "type": { + "name": "text", + "nullable": true + }, "table": "users" } } @@ -45,9 +46,10 @@ "number": 1, "column": { "name": "bio", - "data_type": "text", - "not_null": false, - "is_array": false, + "type": { + "name": "text", + "nullable": true + }, "table": "users" } }, @@ -55,9 +57,9 @@ "number": 2, "column": { "name": "id", - "data_type": "bigint", - "not_null": true, - "is_array": false, + "type": { + "name": "bigint" + }, "table": "users" } } @@ -72,9 +74,9 @@ "number": 1, "column": { "name": "id", - "data_type": "bigint", - "not_null": true, - "is_array": false, + "type": { + "name": "bigint" + }, "table": "users" } } @@ -89,9 +91,9 @@ "number": 1, "column": { "name": "name", - "data_type": "varchar", - "not_null": true, - "is_array": false, + "type": { + "name": "varchar" + }, "table": "users" } } diff --git a/internal/endtoend/testdata/analyze_dml/postgresql/stdout.txt b/internal/endtoend/testdata/analyze_dml/postgresql/output.json similarity index 60% rename from internal/endtoend/testdata/analyze_dml/postgresql/stdout.txt rename to internal/endtoend/testdata/analyze_dml/postgresql/output.json index bcfce7b15a..876a1ed726 100644 --- a/internal/endtoend/testdata/analyze_dml/postgresql/stdout.txt +++ b/internal/endtoend/testdata/analyze_dml/postgresql/output.json @@ -5,23 +5,24 @@ "columns": [ { "name": "id", - "data_type": "bigserial", - "not_null": true, - "is_array": false, + "type": { + "name": "bigserial" + }, "table": "users" }, { "name": "name", - "data_type": "text", - "not_null": true, - "is_array": false, + "type": { + "name": "text" + }, "table": "users" }, { "name": "bio", - "data_type": "text", - "not_null": false, - "is_array": false, + "type": { + "name": "text", + "nullable": true + }, "table": "users" } ], @@ -30,9 +31,9 @@ "number": 1, "column": { "name": "name", - "data_type": "text", - "not_null": true, - "is_array": false, + "type": { + "name": "text" + }, "table": "users" } }, @@ -40,9 +41,10 @@ "number": 2, "column": { "name": "bio", - "data_type": "text", - "not_null": false, - "is_array": false, + "type": { + "name": "text", + "nullable": true + }, "table": "users" } } @@ -57,9 +59,10 @@ "number": 1, "column": { "name": "bio", - "data_type": "text", - "not_null": false, - "is_array": false, + "type": { + "name": "text", + "nullable": true + }, "table": "users" } }, @@ -67,9 +70,9 @@ "number": 2, "column": { "name": "id", - "data_type": "bigserial", - "not_null": true, - "is_array": false, + "type": { + "name": "bigserial" + }, "table": "users" } } @@ -81,16 +84,16 @@ "columns": [ { "name": "id", - "data_type": "bigserial", - "not_null": true, - "is_array": false, + "type": { + "name": "bigserial" + }, "table": "users" }, { "name": "name", - "data_type": "text", - "not_null": true, - "is_array": false, + "type": { + "name": "text" + }, "table": "users" } ], @@ -99,9 +102,9 @@ "number": 1, "column": { "name": "name", - "data_type": "text", - "not_null": true, - "is_array": false, + "type": { + "name": "text" + }, "table": "users" } }, @@ -109,9 +112,9 @@ "number": 2, "column": { "name": "id", - "data_type": "bigserial", - "not_null": true, - "is_array": false, + "type": { + "name": "bigserial" + }, "table": "users" } } @@ -126,9 +129,9 @@ "number": 1, "column": { "name": "id", - "data_type": "bigserial", - "not_null": true, - "is_array": false, + "type": { + "name": "bigserial" + }, "table": "users" } } @@ -140,9 +143,9 @@ "columns": [ { "name": "id", - "data_type": "bigserial", - "not_null": true, - "is_array": false, + "type": { + "name": "bigserial" + }, "table": "users" } ], @@ -151,9 +154,9 @@ "number": 1, "column": { "name": "id", - "data_type": "bigserial", - "not_null": true, - "is_array": false, + "type": { + "name": "bigserial" + }, "table": "users" } } diff --git a/internal/endtoend/testdata/analyze_dml/sqlite/stdout.txt b/internal/endtoend/testdata/analyze_dml/sqlite/output.json similarity index 60% rename from internal/endtoend/testdata/analyze_dml/sqlite/stdout.txt rename to internal/endtoend/testdata/analyze_dml/sqlite/output.json index 78d512d3e7..06863367fa 100644 --- a/internal/endtoend/testdata/analyze_dml/sqlite/stdout.txt +++ b/internal/endtoend/testdata/analyze_dml/sqlite/output.json @@ -5,23 +5,24 @@ "columns": [ { "name": "id", - "data_type": "integer", - "not_null": true, - "is_array": false, + "type": { + "name": "integer" + }, "table": "users" }, { "name": "name", - "data_type": "text", - "not_null": true, - "is_array": false, + "type": { + "name": "text" + }, "table": "users" }, { "name": "bio", - "data_type": "text", - "not_null": false, - "is_array": false, + "type": { + "name": "text", + "nullable": true + }, "table": "users" } ], @@ -30,9 +31,9 @@ "number": 1, "column": { "name": "name", - "data_type": "text", - "not_null": true, - "is_array": false, + "type": { + "name": "text" + }, "table": "users" } }, @@ -40,9 +41,10 @@ "number": 2, "column": { "name": "bio", - "data_type": "text", - "not_null": false, - "is_array": false, + "type": { + "name": "text", + "nullable": true + }, "table": "users" } } @@ -57,9 +59,10 @@ "number": 1, "column": { "name": "bio", - "data_type": "text", - "not_null": false, - "is_array": false, + "type": { + "name": "text", + "nullable": true + }, "table": "users" } }, @@ -67,9 +70,9 @@ "number": 2, "column": { "name": "id", - "data_type": "integer", - "not_null": true, - "is_array": false, + "type": { + "name": "integer" + }, "table": "users" } } @@ -84,9 +87,9 @@ "number": 1, "column": { "name": "id", - "data_type": "integer", - "not_null": true, - "is_array": false, + "type": { + "name": "integer" + }, "table": "users" } } diff --git a/internal/endtoend/testdata/analyze_exec/clickhouse/exec.json b/internal/endtoend/testdata/analyze_exec/clickhouse/exec.json new file mode 100644 index 0000000000..c92bfa3dab --- /dev/null +++ b/internal/endtoend/testdata/analyze_exec/clickhouse/exec.json @@ -0,0 +1,5 @@ +{ + "command": "analyze", + "args": ["--dialect", "clickhouse", "--schema", "schema.sql", "query.sql"], + "contexts": ["base"] +} diff --git a/internal/engine/clickhouse/testgen/testdata/exec/fixture.sql b/internal/endtoend/testdata/analyze_exec/clickhouse/fixture.sql similarity index 100% rename from internal/engine/clickhouse/testgen/testdata/exec/fixture.sql rename to internal/endtoend/testdata/analyze_exec/clickhouse/fixture.sql diff --git a/internal/engine/clickhouse/testgen/testdata/exec/analyze.json b/internal/endtoend/testdata/analyze_exec/clickhouse/output.json similarity index 100% rename from internal/engine/clickhouse/testgen/testdata/exec/analyze.json rename to internal/endtoend/testdata/analyze_exec/clickhouse/output.json diff --git a/internal/engine/clickhouse/testgen/testdata/exec/query.sql b/internal/endtoend/testdata/analyze_exec/clickhouse/query.sql similarity index 100% rename from internal/engine/clickhouse/testgen/testdata/exec/query.sql rename to internal/endtoend/testdata/analyze_exec/clickhouse/query.sql diff --git a/internal/engine/clickhouse/testgen/testdata/exec/schema.sql b/internal/endtoend/testdata/analyze_exec/clickhouse/schema.sql similarity index 100% rename from internal/engine/clickhouse/testgen/testdata/exec/schema.sql rename to internal/endtoend/testdata/analyze_exec/clickhouse/schema.sql diff --git a/internal/endtoend/testdata/analyze_expressions/clickhouse/exec.json b/internal/endtoend/testdata/analyze_expressions/clickhouse/exec.json new file mode 100644 index 0000000000..c92bfa3dab --- /dev/null +++ b/internal/endtoend/testdata/analyze_expressions/clickhouse/exec.json @@ -0,0 +1,5 @@ +{ + "command": "analyze", + "args": ["--dialect", "clickhouse", "--schema", "schema.sql", "query.sql"], + "contexts": ["base"] +} diff --git a/internal/engine/clickhouse/testgen/testdata/expressions/fixture.sql b/internal/endtoend/testdata/analyze_expressions/clickhouse/fixture.sql similarity index 100% rename from internal/engine/clickhouse/testgen/testdata/expressions/fixture.sql rename to internal/endtoend/testdata/analyze_expressions/clickhouse/fixture.sql diff --git a/internal/engine/clickhouse/testgen/testdata/expressions/analyze.json b/internal/endtoend/testdata/analyze_expressions/clickhouse/output.json similarity index 58% rename from internal/engine/clickhouse/testgen/testdata/expressions/analyze.json rename to internal/endtoend/testdata/analyze_expressions/clickhouse/output.json index 05eefc97c4..0553ffac14 100644 --- a/internal/engine/clickhouse/testgen/testdata/expressions/analyze.json +++ b/internal/endtoend/testdata/analyze_expressions/clickhouse/output.json @@ -10,29 +10,16 @@ } }, { - "name": "top", - "type": { - "name": "float64" - } + "name": "top" }, { - "name": "sum(amount)", - "type": { - "name": "float64" - } + "name": "sum" }, { - "name": "first_tag", - "type": { - "name": "string", - "nullable": true - } + "name": "first_tag" }, { - "name": "uniq(name)", - "type": { - "name": "uint64" - } + "name": "uniq" } ], "params": [] @@ -48,42 +35,25 @@ } }, { - "name": "maybe", - "type": { - "name": "string", - "nullable": true - } + "name": "maybe" }, { - "name": "tag_or_none", - "type": { - "name": "string" - } + "name": "tag_or_none" }, { - "name": "lower(name)", - "type": { - "name": "string" - } + "name": "lower" }, { - "name": "today", - "type": { - "name": "date" - } + "name": "today" }, { "name": "big", "type": { - "name": "uint8" + "name": "bool" } }, { - "name": "word", - "type": { - "name": "string", - "nullable": true - } + "name": "word" } ], "params": [] @@ -188,49 +158,7 @@ "table": "events" } ], - "params": [ - { - "number": 1, - "column": { - "name": "name", - "type": { - "name": "string" - }, - "table": "events" - } - }, - { - "number": 2, - "column": { - "name": "tag", - "type": { - "name": "string", - "nullable": true - }, - "table": "events" - } - }, - { - "number": 3, - "column": { - "name": "amount", - "type": { - "name": "float64" - }, - "table": "events" - } - }, - { - "number": 3, - "column": { - "name": "amount", - "type": { - "name": "float64" - }, - "table": "events" - } - } - ] + "params": [] }, { "name": "Functions", @@ -248,10 +176,7 @@ { "number": 1, "column": { - "name": "lower", - "type": { - "name": "string" - } + "name": "" } }, { @@ -267,10 +192,7 @@ { "number": 3, "column": { - "name": "", - "type": { - "name": "date" - } + "name": "" } } ] @@ -287,37 +209,14 @@ "table": "events" } ], - "params": [ - { - "number": 1, - "column": { - "name": "", - "type": { - "name": "uint64" - } - } - }, - { - "number": 2, - "column": { - "name": "", - "type": { - "name": "uint64" - } - } - } - ] + "params": [] }, { "name": "Projected", "cmd": ":one", "columns": [ { - "name": "echo", - "type": { - "name": "nothing", - "nullable": true - } + "name": "echo" }, { "name": "lit", @@ -330,11 +229,7 @@ { "number": 1, "column": { - "name": "echo", - "type": { - "name": "nothing", - "nullable": true - } + "name": "" } } ] diff --git a/internal/engine/clickhouse/testgen/testdata/expressions/query.sql b/internal/endtoend/testdata/analyze_expressions/clickhouse/query.sql similarity index 100% rename from internal/engine/clickhouse/testgen/testdata/expressions/query.sql rename to internal/endtoend/testdata/analyze_expressions/clickhouse/query.sql diff --git a/internal/engine/clickhouse/testgen/testdata/expressions/schema.sql b/internal/endtoend/testdata/analyze_expressions/clickhouse/schema.sql similarity index 100% rename from internal/engine/clickhouse/testgen/testdata/expressions/schema.sql rename to internal/endtoend/testdata/analyze_expressions/clickhouse/schema.sql diff --git a/internal/endtoend/testdata/analyze_extension/postgresql/stdout.txt b/internal/endtoend/testdata/analyze_extension/postgresql/output.json similarity index 57% rename from internal/endtoend/testdata/analyze_extension/postgresql/stdout.txt rename to internal/endtoend/testdata/analyze_extension/postgresql/output.json index 9bf76757ac..af10afec06 100644 --- a/internal/endtoend/testdata/analyze_extension/postgresql/stdout.txt +++ b/internal/endtoend/testdata/analyze_extension/postgresql/output.json @@ -5,16 +5,16 @@ "columns": [ { "name": "id", - "data_type": "bigserial", - "not_null": true, - "is_array": false, + "type": { + "name": "bigserial" + }, "table": "users" }, { "name": "email", - "data_type": "citext", - "not_null": true, - "is_array": false, + "type": { + "name": "citext" + }, "table": "users" } ], @@ -23,9 +23,9 @@ "number": 1, "column": { "name": "email", - "data_type": "citext", - "not_null": true, - "is_array": false, + "type": { + "name": "citext" + }, "table": "users" } } @@ -37,9 +37,9 @@ "columns": [ { "name": "id", - "data_type": "bigserial", - "not_null": true, - "is_array": false, + "type": { + "name": "bigserial" + }, "table": "users" } ], @@ -51,16 +51,16 @@ "columns": [ { "name": "id", - "data_type": "bigserial", - "not_null": true, - "is_array": false, + "type": { + "name": "bigserial" + }, "table": "users" }, { "name": "score", - "data_type": "real", - "not_null": true, - "is_array": false + "type": { + "name": "real" + } } ], "params": [ @@ -68,9 +68,9 @@ "number": 1, "column": { "name": "", - "data_type": "text", - "not_null": true, - "is_array": false + "type": { + "name": "text" + } } } ] diff --git a/internal/engine/clickhouse/testgen/testdata/analyze_params/fixture.sql b/internal/endtoend/testdata/analyze_params/clickhouse/fixture.sql similarity index 100% rename from internal/engine/clickhouse/testgen/testdata/analyze_params/fixture.sql rename to internal/endtoend/testdata/analyze_params/clickhouse/fixture.sql diff --git a/internal/engine/clickhouse/testgen/testdata/analyze_params/analyze.json b/internal/endtoend/testdata/analyze_params/clickhouse/output.json similarity index 100% rename from internal/engine/clickhouse/testgen/testdata/analyze_params/analyze.json rename to internal/endtoend/testdata/analyze_params/clickhouse/output.json diff --git a/internal/endtoend/testdata/analyze_params/clickhouse/stdout.txt b/internal/endtoend/testdata/analyze_params/clickhouse/stdout.txt deleted file mode 100644 index 6fb0cc654b..0000000000 --- a/internal/endtoend/testdata/analyze_params/clickhouse/stdout.txt +++ /dev/null @@ -1,76 +0,0 @@ -[ - { - "name": "GetEvent", - "cmd": ":one", - "columns": [ - { - "name": "id", - "data_type": "uint64", - "not_null": true, - "is_array": false, - "table": "events" - }, - { - "name": "name", - "data_type": "string", - "not_null": true, - "is_array": false, - "table": "events" - } - ], - "params": [ - { - "number": 1, - "column": { - "name": "id", - "data_type": "uint64", - "not_null": true, - "is_array": false, - "table": "events" - } - } - ] - }, - { - "name": "FilterEvents", - "cmd": ":many", - "columns": [ - { - "name": "id", - "data_type": "uint64", - "not_null": true, - "is_array": false, - "table": "events" - }, - { - "name": "name", - "data_type": "string", - "not_null": true, - "is_array": false, - "table": "events" - } - ], - "params": [ - { - "number": 1, - "column": { - "name": "name", - "data_type": "string", - "not_null": true, - "is_array": false, - "table": "events" - } - }, - { - "number": 2, - "column": { - "name": "amount", - "data_type": "float64", - "not_null": true, - "is_array": false, - "table": "events" - } - } - ] - } -] diff --git a/internal/endtoend/testdata/analyze_params/duckdb/stdout.txt b/internal/endtoend/testdata/analyze_params/duckdb/output.json similarity index 58% rename from internal/endtoend/testdata/analyze_params/duckdb/stdout.txt rename to internal/endtoend/testdata/analyze_params/duckdb/output.json index baacd76bbc..0a85a39358 100644 --- a/internal/endtoend/testdata/analyze_params/duckdb/stdout.txt +++ b/internal/endtoend/testdata/analyze_params/duckdb/output.json @@ -5,23 +5,24 @@ "columns": [ { "name": "id", - "data_type": "bigint", - "not_null": true, - "is_array": false, + "type": { + "name": "bigint" + }, "table": "authors" }, { "name": "name", - "data_type": "text", - "not_null": true, - "is_array": false, + "type": { + "name": "text" + }, "table": "authors" }, { "name": "bio", - "data_type": "text", - "not_null": false, - "is_array": false, + "type": { + "name": "text", + "nullable": true + }, "table": "authors" } ], @@ -30,9 +31,9 @@ "number": 1, "column": { "name": "id", - "data_type": "bigint", - "not_null": true, - "is_array": false, + "type": { + "name": "bigint" + }, "table": "authors" } } @@ -44,16 +45,16 @@ "columns": [ { "name": "id", - "data_type": "bigint", - "not_null": true, - "is_array": false, + "type": { + "name": "bigint" + }, "table": "authors" }, { "name": "name", - "data_type": "text", - "not_null": true, - "is_array": false, + "type": { + "name": "text" + }, "table": "authors" } ], @@ -62,9 +63,9 @@ "number": 1, "column": { "name": "name", - "data_type": "text", - "not_null": true, - "is_array": false, + "type": { + "name": "text" + }, "table": "authors" } }, @@ -72,9 +73,9 @@ "number": 2, "column": { "name": "royalties", - "data_type": "decimal", - "not_null": true, - "is_array": false, + "type": { + "name": "decimal" + }, "table": "authors" } } @@ -86,9 +87,9 @@ "columns": [ { "name": "id", - "data_type": "bigint", - "not_null": true, - "is_array": false, + "type": { + "name": "bigint" + }, "table": "authors" } ], @@ -97,9 +98,9 @@ "number": 1, "column": { "name": "name", - "data_type": "text", - "not_null": true, - "is_array": false, + "type": { + "name": "text" + }, "table": "authors" } } diff --git a/internal/endtoend/testdata/analyze_params/mssql/stdout.txt b/internal/endtoend/testdata/analyze_params/mssql/output.json similarity index 56% rename from internal/endtoend/testdata/analyze_params/mssql/stdout.txt rename to internal/endtoend/testdata/analyze_params/mssql/output.json index c5765c94c3..1483213369 100644 --- a/internal/endtoend/testdata/analyze_params/mssql/stdout.txt +++ b/internal/endtoend/testdata/analyze_params/mssql/output.json @@ -5,23 +5,24 @@ "columns": [ { "name": "id", - "data_type": "bigint", - "not_null": true, - "is_array": false, + "type": { + "name": "bigint" + }, "table": "authors" }, { "name": "name", - "data_type": "nvarchar", - "not_null": true, - "is_array": false, + "type": { + "name": "nvarchar" + }, "table": "authors" }, { "name": "bio", - "data_type": "nvarchar", - "not_null": false, - "is_array": false, + "type": { + "name": "nvarchar", + "nullable": true + }, "table": "authors" } ], @@ -30,9 +31,9 @@ "number": 1, "column": { "name": "id", - "data_type": "bigint", - "not_null": true, - "is_array": false, + "type": { + "name": "bigint" + }, "table": "authors" } } @@ -44,16 +45,16 @@ "columns": [ { "name": "id", - "data_type": "bigint", - "not_null": true, - "is_array": false, + "type": { + "name": "bigint" + }, "table": "authors" }, { "name": "name", - "data_type": "nvarchar", - "not_null": true, - "is_array": false, + "type": { + "name": "nvarchar" + }, "table": "authors" } ], @@ -62,9 +63,9 @@ "number": 1, "column": { "name": "name", - "data_type": "nvarchar", - "not_null": true, - "is_array": false, + "type": { + "name": "nvarchar" + }, "table": "authors" } }, @@ -72,9 +73,9 @@ "number": 2, "column": { "name": "royalties", - "data_type": "decimal", - "not_null": true, - "is_array": false, + "type": { + "name": "decimal" + }, "table": "authors" } } diff --git a/internal/endtoend/testdata/analyze_params/mysql/stdout.txt b/internal/endtoend/testdata/analyze_params/mysql/output.json similarity index 57% rename from internal/endtoend/testdata/analyze_params/mysql/stdout.txt rename to internal/endtoend/testdata/analyze_params/mysql/output.json index e5ef23ba6f..7ff80429a7 100644 --- a/internal/endtoend/testdata/analyze_params/mysql/stdout.txt +++ b/internal/endtoend/testdata/analyze_params/mysql/output.json @@ -5,16 +5,16 @@ "columns": [ { "name": "id", - "data_type": "bigint", - "not_null": true, - "is_array": false, + "type": { + "name": "bigint" + }, "table": "users" }, { "name": "name", - "data_type": "varchar", - "not_null": true, - "is_array": false, + "type": { + "name": "varchar" + }, "table": "users" } ], @@ -23,9 +23,9 @@ "number": 1, "column": { "name": "id", - "data_type": "bigint", - "not_null": true, - "is_array": false, + "type": { + "name": "bigint" + }, "table": "users" } } @@ -37,16 +37,16 @@ "columns": [ { "name": "id", - "data_type": "bigint", - "not_null": true, - "is_array": false, + "type": { + "name": "bigint" + }, "table": "users" }, { "name": "name", - "data_type": "varchar", - "not_null": true, - "is_array": false, + "type": { + "name": "varchar" + }, "table": "users" } ], @@ -55,9 +55,9 @@ "number": 1, "column": { "name": "name", - "data_type": "varchar", - "not_null": true, - "is_array": false, + "type": { + "name": "varchar" + }, "table": "users" } }, @@ -65,9 +65,10 @@ "number": 2, "column": { "name": "bio", - "data_type": "text", - "not_null": false, - "is_array": false, + "type": { + "name": "text", + "nullable": true + }, "table": "users" } } @@ -79,16 +80,16 @@ "columns": [ { "name": "id", - "data_type": "bigint", - "not_null": true, - "is_array": false, + "type": { + "name": "bigint" + }, "table": "users" }, { "name": "name", - "data_type": "varchar", - "not_null": true, - "is_array": false, + "type": { + "name": "varchar" + }, "table": "users" } ], @@ -97,9 +98,9 @@ "number": 1, "column": { "name": "ids", - "data_type": "bigint", - "not_null": true, - "is_array": false, + "type": { + "name": "bigint" + }, "table": "users" } } diff --git a/internal/endtoend/testdata/analyze_params/postgresql/stdout.txt b/internal/endtoend/testdata/analyze_params/postgresql/output.json similarity index 57% rename from internal/endtoend/testdata/analyze_params/postgresql/stdout.txt rename to internal/endtoend/testdata/analyze_params/postgresql/output.json index 6bbcf899ff..3d98c7ff65 100644 --- a/internal/endtoend/testdata/analyze_params/postgresql/stdout.txt +++ b/internal/endtoend/testdata/analyze_params/postgresql/output.json @@ -5,16 +5,16 @@ "columns": [ { "name": "id", - "data_type": "bigserial", - "not_null": true, - "is_array": false, + "type": { + "name": "bigserial" + }, "table": "users" }, { "name": "name", - "data_type": "text", - "not_null": true, - "is_array": false, + "type": { + "name": "text" + }, "table": "users" } ], @@ -23,9 +23,9 @@ "number": 1, "column": { "name": "id", - "data_type": "bigserial", - "not_null": true, - "is_array": false, + "type": { + "name": "bigserial" + }, "table": "users" } } @@ -37,16 +37,16 @@ "columns": [ { "name": "id", - "data_type": "bigserial", - "not_null": true, - "is_array": false, + "type": { + "name": "bigserial" + }, "table": "users" }, { "name": "name", - "data_type": "text", - "not_null": true, - "is_array": false, + "type": { + "name": "text" + }, "table": "users" } ], @@ -55,9 +55,9 @@ "number": 1, "column": { "name": "name", - "data_type": "text", - "not_null": true, - "is_array": false, + "type": { + "name": "text" + }, "table": "users" } }, @@ -65,9 +65,10 @@ "number": 2, "column": { "name": "bio", - "data_type": "text", - "not_null": false, - "is_array": false, + "type": { + "name": "text", + "nullable": true + }, "table": "users" } } @@ -79,16 +80,16 @@ "columns": [ { "name": "id", - "data_type": "bigserial", - "not_null": true, - "is_array": false, + "type": { + "name": "bigserial" + }, "table": "users" }, { "name": "name", - "data_type": "text", - "not_null": true, - "is_array": false, + "type": { + "name": "text" + }, "table": "users" } ], @@ -97,9 +98,9 @@ "number": 1, "column": { "name": "ids", - "data_type": "bigserial", - "not_null": true, - "is_array": false, + "type": { + "name": "bigserial" + }, "table": "users" } } @@ -111,16 +112,17 @@ "columns": [ { "name": "id", - "data_type": "bigserial", - "not_null": true, - "is_array": false, + "type": { + "name": "bigserial" + }, "table": "posts" }, { "name": "title", - "data_type": "varchar", - "not_null": false, - "is_array": false, + "type": { + "name": "varchar", + "nullable": true + }, "table": "posts" } ], @@ -129,9 +131,9 @@ "number": 1, "column": { "name": "created", - "data_type": "timestamptz", - "not_null": true, - "is_array": false, + "type": { + "name": "timestamptz" + }, "table": "posts" } }, @@ -139,9 +141,9 @@ "number": 2, "column": { "name": "created", - "data_type": "timestamptz", - "not_null": true, - "is_array": false, + "type": { + "name": "timestamptz" + }, "table": "posts" } } diff --git a/internal/endtoend/testdata/analyze_params/sqlite/stdout.txt b/internal/endtoend/testdata/analyze_params/sqlite/output.json similarity index 58% rename from internal/endtoend/testdata/analyze_params/sqlite/stdout.txt rename to internal/endtoend/testdata/analyze_params/sqlite/output.json index 067fe197be..fd0e721573 100644 --- a/internal/endtoend/testdata/analyze_params/sqlite/stdout.txt +++ b/internal/endtoend/testdata/analyze_params/sqlite/output.json @@ -5,16 +5,16 @@ "columns": [ { "name": "id", - "data_type": "integer", - "not_null": true, - "is_array": false, + "type": { + "name": "integer" + }, "table": "users" }, { "name": "name", - "data_type": "text", - "not_null": true, - "is_array": false, + "type": { + "name": "text" + }, "table": "users" } ], @@ -23,9 +23,9 @@ "number": 1, "column": { "name": "id", - "data_type": "integer", - "not_null": true, - "is_array": false, + "type": { + "name": "integer" + }, "table": "users" } } @@ -37,16 +37,16 @@ "columns": [ { "name": "id", - "data_type": "integer", - "not_null": true, - "is_array": false, + "type": { + "name": "integer" + }, "table": "users" }, { "name": "name", - "data_type": "text", - "not_null": true, - "is_array": false, + "type": { + "name": "text" + }, "table": "users" } ], @@ -55,9 +55,9 @@ "number": 1, "column": { "name": "name", - "data_type": "text", - "not_null": true, - "is_array": false, + "type": { + "name": "text" + }, "table": "users" } }, @@ -65,9 +65,10 @@ "number": 2, "column": { "name": "bio", - "data_type": "text", - "not_null": false, - "is_array": false, + "type": { + "name": "text", + "nullable": true + }, "table": "users" } } @@ -79,9 +80,9 @@ "columns": [ { "name": "total", - "data_type": "integer", - "not_null": true, - "is_array": false + "type": { + "name": "integer" + } } ], "params": [ @@ -89,9 +90,9 @@ "number": 1, "column": { "name": "user_id", - "data_type": "integer", - "not_null": true, - "is_array": false, + "type": { + "name": "integer" + }, "table": "posts" } } diff --git a/internal/endtoend/testdata/analyze_select/duckdb/stdout.txt b/internal/endtoend/testdata/analyze_select/duckdb/output.json similarity index 54% rename from internal/endtoend/testdata/analyze_select/duckdb/stdout.txt rename to internal/endtoend/testdata/analyze_select/duckdb/output.json index 88d31141e5..2b0e161be2 100644 --- a/internal/endtoend/testdata/analyze_select/duckdb/stdout.txt +++ b/internal/endtoend/testdata/analyze_select/duckdb/output.json @@ -5,23 +5,24 @@ "columns": [ { "name": "id", - "data_type": "bigint", - "not_null": true, - "is_array": false, + "type": { + "name": "bigint" + }, "table": "users" }, { "name": "name", - "data_type": "text", - "not_null": true, - "is_array": false, + "type": { + "name": "text" + }, "table": "users" }, { "name": "age", - "data_type": "integer", - "not_null": false, - "is_array": false, + "type": { + "name": "integer", + "nullable": true + }, "table": "users" } ], @@ -33,9 +34,9 @@ "columns": [ { "name": "total", - "data_type": "bigint", - "not_null": true, - "is_array": false + "type": { + "name": "bigint" + } } ], "params": [] @@ -46,37 +47,38 @@ "columns": [ { "name": "name", - "data_type": "text", - "not_null": true, - "is_array": false, + "type": { + "name": "text" + }, "table": "users" }, { "name": "id", - "data_type": "bigint", - "not_null": true, - "is_array": false, + "type": { + "name": "bigint" + }, "table": "posts" }, { "name": "user_id", - "data_type": "bigint", - "not_null": true, - "is_array": false, + "type": { + "name": "bigint" + }, "table": "posts" }, { "name": "title", - "data_type": "text", - "not_null": true, - "is_array": false, + "type": { + "name": "text" + }, "table": "posts" }, { "name": "body", - "data_type": "text", - "not_null": false, - "is_array": false, + "type": { + "name": "text", + "nullable": true + }, "table": "posts" } ], @@ -85,9 +87,9 @@ "number": 1, "column": { "name": "name", - "data_type": "text", - "not_null": true, - "is_array": false, + "type": { + "name": "text" + }, "table": "users" } } @@ -99,16 +101,16 @@ "columns": [ { "name": "name", - "data_type": "text", - "not_null": true, - "is_array": false, + "type": { + "name": "text" + }, "table": "users" }, { "name": "n", - "data_type": "bigint", - "not_null": true, - "is_array": false + "type": { + "name": "bigint" + } } ], "params": [] diff --git a/internal/endtoend/testdata/analyze_select/googlesql/stdout.txt b/internal/endtoend/testdata/analyze_select/googlesql/output.json similarity index 53% rename from internal/endtoend/testdata/analyze_select/googlesql/stdout.txt rename to internal/endtoend/testdata/analyze_select/googlesql/output.json index b2b19ce73f..0bd71ba16a 100644 --- a/internal/endtoend/testdata/analyze_select/googlesql/stdout.txt +++ b/internal/endtoend/testdata/analyze_select/googlesql/output.json @@ -5,23 +5,24 @@ "columns": [ { "name": "id", - "data_type": "int64", - "not_null": true, - "is_array": false, + "type": { + "name": "int64" + }, "table": "users" }, { "name": "name", - "data_type": "string", - "not_null": true, - "is_array": false, + "type": { + "name": "string" + }, "table": "users" }, { "name": "bio", - "data_type": "string", - "not_null": false, - "is_array": false, + "type": { + "name": "string", + "nullable": true + }, "table": "users" } ], @@ -33,9 +34,9 @@ "columns": [ { "name": "total", - "data_type": "int64", - "not_null": true, - "is_array": false + "type": { + "name": "int64" + } } ], "params": [] @@ -46,37 +47,38 @@ "columns": [ { "name": "name", - "data_type": "string", - "not_null": true, - "is_array": false, + "type": { + "name": "string" + }, "table": "users" }, { "name": "id", - "data_type": "int64", - "not_null": true, - "is_array": false, + "type": { + "name": "int64" + }, "table": "posts" }, { "name": "user_id", - "data_type": "int64", - "not_null": true, - "is_array": false, + "type": { + "name": "int64" + }, "table": "posts" }, { "name": "title", - "data_type": "string", - "not_null": false, - "is_array": false, + "type": { + "name": "string", + "nullable": true + }, "table": "posts" }, { "name": "created", - "data_type": "timestamp", - "not_null": true, - "is_array": false, + "type": { + "name": "timestamp" + }, "table": "posts" } ], @@ -85,9 +87,9 @@ "number": 1, "column": { "name": "name", - "data_type": "string", - "not_null": true, - "is_array": false, + "type": { + "name": "string" + }, "table": "users" } } diff --git a/internal/endtoend/testdata/analyze_select/mysql/stdout.txt b/internal/endtoend/testdata/analyze_select/mysql/output.json similarity index 54% rename from internal/endtoend/testdata/analyze_select/mysql/stdout.txt rename to internal/endtoend/testdata/analyze_select/mysql/output.json index 14fac17e42..2ef9380633 100644 --- a/internal/endtoend/testdata/analyze_select/mysql/stdout.txt +++ b/internal/endtoend/testdata/analyze_select/mysql/output.json @@ -5,23 +5,24 @@ "columns": [ { "name": "id", - "data_type": "bigint", - "not_null": true, - "is_array": false, + "type": { + "name": "bigint" + }, "table": "users" }, { "name": "name", - "data_type": "varchar", - "not_null": true, - "is_array": false, + "type": { + "name": "varchar" + }, "table": "users" }, { "name": "bio", - "data_type": "text", - "not_null": false, - "is_array": false, + "type": { + "name": "text", + "nullable": true + }, "table": "users" } ], @@ -33,9 +34,9 @@ "columns": [ { "name": "total", - "data_type": "bigint", - "not_null": true, - "is_array": false + "type": { + "name": "bigint" + } } ], "params": [] @@ -46,37 +47,38 @@ "columns": [ { "name": "name", - "data_type": "varchar", - "not_null": true, - "is_array": false, + "type": { + "name": "varchar" + }, "table": "users" }, { "name": "id", - "data_type": "bigint", - "not_null": true, - "is_array": false, + "type": { + "name": "bigint" + }, "table": "posts" }, { "name": "user_id", - "data_type": "bigint", - "not_null": true, - "is_array": false, + "type": { + "name": "bigint" + }, "table": "posts" }, { "name": "title", - "data_type": "varchar", - "not_null": false, - "is_array": false, + "type": { + "name": "varchar", + "nullable": true + }, "table": "posts" }, { "name": "created", - "data_type": "datetime", - "not_null": true, - "is_array": false, + "type": { + "name": "datetime" + }, "table": "posts" } ], @@ -85,9 +87,9 @@ "number": 1, "column": { "name": "name", - "data_type": "varchar", - "not_null": true, - "is_array": false, + "type": { + "name": "varchar" + }, "table": "users" } } @@ -99,16 +101,17 @@ "columns": [ { "name": "id", - "data_type": "bigint", - "not_null": true, - "is_array": false, + "type": { + "name": "bigint" + }, "table": "posts" }, { "name": "title", - "data_type": "varchar", - "not_null": false, - "is_array": false, + "type": { + "name": "varchar", + "nullable": true + }, "table": "posts" } ], @@ -117,9 +120,10 @@ "number": 1, "column": { "name": "title", - "data_type": "varchar", - "not_null": false, - "is_array": false, + "type": { + "name": "varchar", + "nullable": true + }, "table": "posts" } }, @@ -127,9 +131,9 @@ "number": 2, "column": { "name": "created", - "data_type": "datetime", - "not_null": true, - "is_array": false, + "type": { + "name": "datetime" + }, "table": "posts" } } @@ -141,16 +145,16 @@ "columns": [ { "name": "name", - "data_type": "varchar", - "not_null": true, - "is_array": false, + "type": { + "name": "varchar" + }, "table": "users" }, { "name": "created", - "data_type": "datetime", - "not_null": true, - "is_array": false + "type": { + "name": "datetime" + } } ], "params": [] diff --git a/internal/endtoend/testdata/analyze_select/postgresql/stdout.txt b/internal/endtoend/testdata/analyze_select/postgresql/output.json similarity index 54% rename from internal/endtoend/testdata/analyze_select/postgresql/stdout.txt rename to internal/endtoend/testdata/analyze_select/postgresql/output.json index d424775aa1..75ae4486c3 100644 --- a/internal/endtoend/testdata/analyze_select/postgresql/stdout.txt +++ b/internal/endtoend/testdata/analyze_select/postgresql/output.json @@ -5,23 +5,24 @@ "columns": [ { "name": "id", - "data_type": "bigserial", - "not_null": true, - "is_array": false, + "type": { + "name": "bigserial" + }, "table": "users" }, { "name": "name", - "data_type": "text", - "not_null": true, - "is_array": false, + "type": { + "name": "text" + }, "table": "users" }, { "name": "bio", - "data_type": "text", - "not_null": false, - "is_array": false, + "type": { + "name": "text", + "nullable": true + }, "table": "users" } ], @@ -33,9 +34,9 @@ "columns": [ { "name": "total", - "data_type": "bigint", - "not_null": true, - "is_array": false + "type": { + "name": "bigint" + } } ], "params": [] @@ -46,37 +47,38 @@ "columns": [ { "name": "name", - "data_type": "text", - "not_null": true, - "is_array": false, + "type": { + "name": "text" + }, "table": "users" }, { "name": "id", - "data_type": "bigserial", - "not_null": true, - "is_array": false, + "type": { + "name": "bigserial" + }, "table": "posts" }, { "name": "user_id", - "data_type": "int8", - "not_null": true, - "is_array": false, + "type": { + "name": "int8" + }, "table": "posts" }, { "name": "title", - "data_type": "varchar", - "not_null": false, - "is_array": false, + "type": { + "name": "varchar", + "nullable": true + }, "table": "posts" }, { "name": "created", - "data_type": "timestamptz", - "not_null": true, - "is_array": false, + "type": { + "name": "timestamptz" + }, "table": "posts" } ], @@ -85,9 +87,9 @@ "number": 1, "column": { "name": "name", - "data_type": "text", - "not_null": true, - "is_array": false, + "type": { + "name": "text" + }, "table": "users" } } @@ -99,16 +101,17 @@ "columns": [ { "name": "id", - "data_type": "bigserial", - "not_null": true, - "is_array": false, + "type": { + "name": "bigserial" + }, "table": "posts" }, { "name": "title", - "data_type": "varchar", - "not_null": false, - "is_array": false, + "type": { + "name": "varchar", + "nullable": true + }, "table": "posts" } ], @@ -117,9 +120,10 @@ "number": 1, "column": { "name": "title", - "data_type": "varchar", - "not_null": false, - "is_array": false, + "type": { + "name": "varchar", + "nullable": true + }, "table": "posts" } }, @@ -127,9 +131,9 @@ "number": 2, "column": { "name": "created", - "data_type": "timestamptz", - "not_null": true, - "is_array": false, + "type": { + "name": "timestamptz" + }, "table": "posts" } } @@ -141,16 +145,16 @@ "columns": [ { "name": "name", - "data_type": "text", - "not_null": true, - "is_array": false, + "type": { + "name": "text" + }, "table": "users" }, { "name": "created", - "data_type": "timestamptz", - "not_null": true, - "is_array": false + "type": { + "name": "timestamptz" + } } ], "params": [] diff --git a/internal/endtoend/testdata/analyze_select/sqlite/stdout.txt b/internal/endtoend/testdata/analyze_select/sqlite/output.json similarity index 55% rename from internal/endtoend/testdata/analyze_select/sqlite/stdout.txt rename to internal/endtoend/testdata/analyze_select/sqlite/output.json index 7c525df07c..032d947426 100644 --- a/internal/endtoend/testdata/analyze_select/sqlite/stdout.txt +++ b/internal/endtoend/testdata/analyze_select/sqlite/output.json @@ -5,23 +5,24 @@ "columns": [ { "name": "id", - "data_type": "integer", - "not_null": true, - "is_array": false, + "type": { + "name": "integer" + }, "table": "users" }, { "name": "name", - "data_type": "text", - "not_null": true, - "is_array": false, + "type": { + "name": "text" + }, "table": "users" }, { "name": "bio", - "data_type": "text", - "not_null": false, - "is_array": false, + "type": { + "name": "text", + "nullable": true + }, "table": "users" } ], @@ -33,9 +34,9 @@ "columns": [ { "name": "total", - "data_type": "integer", - "not_null": true, - "is_array": false + "type": { + "name": "integer" + } } ], "params": [] @@ -46,37 +47,38 @@ "columns": [ { "name": "name", - "data_type": "text", - "not_null": true, - "is_array": false, + "type": { + "name": "text" + }, "table": "users" }, { "name": "id", - "data_type": "integer", - "not_null": true, - "is_array": false, + "type": { + "name": "integer" + }, "table": "posts" }, { "name": "user_id", - "data_type": "integer", - "not_null": true, - "is_array": false, + "type": { + "name": "integer" + }, "table": "posts" }, { "name": "title", - "data_type": "text", - "not_null": false, - "is_array": false, + "type": { + "name": "text", + "nullable": true + }, "table": "posts" }, { "name": "created", - "data_type": "text", - "not_null": true, - "is_array": false, + "type": { + "name": "text" + }, "table": "posts" } ], @@ -85,9 +87,9 @@ "number": 1, "column": { "name": "name", - "data_type": "text", - "not_null": true, - "is_array": false, + "type": { + "name": "text" + }, "table": "users" } } @@ -99,16 +101,17 @@ "columns": [ { "name": "id", - "data_type": "integer", - "not_null": true, - "is_array": false, + "type": { + "name": "integer" + }, "table": "posts" }, { "name": "title", - "data_type": "text", - "not_null": false, - "is_array": false, + "type": { + "name": "text", + "nullable": true + }, "table": "posts" } ], @@ -117,9 +120,10 @@ "number": 1, "column": { "name": "title", - "data_type": "text", - "not_null": false, - "is_array": false, + "type": { + "name": "text", + "nullable": true + }, "table": "posts" } }, @@ -127,9 +131,9 @@ "number": 2, "column": { "name": "created", - "data_type": "text", - "not_null": true, - "is_array": false, + "type": { + "name": "text" + }, "table": "posts" } } diff --git a/internal/endtoend/testdata/analyze_subqueries/clickhouse/exec.json b/internal/endtoend/testdata/analyze_subqueries/clickhouse/exec.json new file mode 100644 index 0000000000..c92bfa3dab --- /dev/null +++ b/internal/endtoend/testdata/analyze_subqueries/clickhouse/exec.json @@ -0,0 +1,5 @@ +{ + "command": "analyze", + "args": ["--dialect", "clickhouse", "--schema", "schema.sql", "query.sql"], + "contexts": ["base"] +} diff --git a/internal/engine/clickhouse/testgen/testdata/subqueries/fixture.sql b/internal/endtoend/testdata/analyze_subqueries/clickhouse/fixture.sql similarity index 100% rename from internal/engine/clickhouse/testgen/testdata/subqueries/fixture.sql rename to internal/endtoend/testdata/analyze_subqueries/clickhouse/fixture.sql diff --git a/internal/endtoend/testdata/analyze_subqueries/clickhouse/output.json b/internal/endtoend/testdata/analyze_subqueries/clickhouse/output.json new file mode 100644 index 0000000000..db310e24a1 --- /dev/null +++ b/internal/endtoend/testdata/analyze_subqueries/clickhouse/output.json @@ -0,0 +1,63 @@ +[ + { + "name": "Aliased", + "cmd": ":many", + "columns": [ + { + "name": "x", + "type": { + "name": "uint64" + }, + "table": "events" + }, + { + "name": "total" + }, + { + "name": "email" + } + ], + "params": [] + }, + { + "name": "Union", + "cmd": ":many", + "columns": [ + { + "name": "id", + "type": { + "name": "uint64" + }, + "table": "events" + }, + { + "name": "name", + "type": { + "name": "string" + }, + "table": "events" + } + ], + "params": [] + }, + { + "name": "ScalarSubquery", + "cmd": ":many", + "columns": [ + { + "name": "id", + "type": { + "name": "uint64" + }, + "table": "events" + }, + { + "name": "?column?", + "type": { + "name": "bool" + } + } + ], + "params": [] + } +] diff --git a/internal/engine/clickhouse/testgen/testdata/subqueries/query.sql b/internal/endtoend/testdata/analyze_subqueries/clickhouse/query.sql similarity index 64% rename from internal/engine/clickhouse/testgen/testdata/subqueries/query.sql rename to internal/endtoend/testdata/analyze_subqueries/clickhouse/query.sql index 297e082951..8633227102 100644 --- a/internal/engine/clickhouse/testgen/testdata/subqueries/query.sql +++ b/internal/endtoend/testdata/analyze_subqueries/clickhouse/query.sql @@ -1,9 +1,3 @@ --- name: Cte :many -WITH t AS (SELECT id, tag FROM events) -SELECT t.id, t.tag, s.cnt -FROM t -JOIN (SELECT id, count() AS cnt FROM events GROUP BY id) s ON s.id = t.id; - -- name: Aliased :many SELECT s.x, s.total, s.email FROM ( @@ -12,9 +6,6 @@ FROM ( GROUP BY e.id ) s; --- name: Star :many -SELECT * FROM users u JOIN events e ON e.id = u.id; - -- name: Union :many SELECT id, name FROM events UNION ALL @@ -24,3 +15,4 @@ SELECT id, email FROM users; SELECT id, (SELECT count() FROM users) AS user_count FROM events WHERE id IN (SELECT id FROM users WHERE email = ?); + diff --git a/internal/engine/clickhouse/testgen/testdata/subqueries/schema.sql b/internal/endtoend/testdata/analyze_subqueries/clickhouse/schema.sql similarity index 100% rename from internal/engine/clickhouse/testgen/testdata/subqueries/schema.sql rename to internal/endtoend/testdata/analyze_subqueries/clickhouse/schema.sql diff --git a/internal/endtoend/testdata/analyze_system_catalog/postgresql/stdout.txt b/internal/endtoend/testdata/analyze_system_catalog/postgresql/output.json similarity index 56% rename from internal/endtoend/testdata/analyze_system_catalog/postgresql/stdout.txt rename to internal/endtoend/testdata/analyze_system_catalog/postgresql/output.json index ee5f980921..0432abd33e 100644 --- a/internal/endtoend/testdata/analyze_system_catalog/postgresql/stdout.txt +++ b/internal/endtoend/testdata/analyze_system_catalog/postgresql/output.json @@ -5,23 +5,26 @@ "columns": [ { "name": "table_name", - "data_type": "sql_identifier", - "not_null": false, - "is_array": false, + "type": { + "name": "sql_identifier", + "nullable": true + }, "table": "columns" }, { "name": "column_name", - "data_type": "sql_identifier", - "not_null": false, - "is_array": false, + "type": { + "name": "sql_identifier", + "nullable": true + }, "table": "columns" }, { "name": "data_type", - "data_type": "character_data", - "not_null": false, - "is_array": false, + "type": { + "name": "character_data", + "nullable": true + }, "table": "columns" } ], @@ -30,9 +33,10 @@ "number": 1, "column": { "name": "table_schema", - "data_type": "sql_identifier", - "not_null": false, - "is_array": false, + "type": { + "name": "sql_identifier", + "nullable": true + }, "table": "columns" } } @@ -44,9 +48,9 @@ "columns": [ { "name": "total", - "data_type": "bigint", - "not_null": true, - "is_array": false + "type": { + "name": "bigint" + } } ], "params": [ @@ -54,9 +58,9 @@ "number": 1, "column": { "name": "relkind", - "data_type": "char", - "not_null": true, - "is_array": false, + "type": { + "name": "char" + }, "table": "pg_class" } } diff --git a/internal/endtoend/testdata/analyze_types/clickhouse/exec.json b/internal/endtoend/testdata/analyze_types/clickhouse/exec.json new file mode 100644 index 0000000000..c92bfa3dab --- /dev/null +++ b/internal/endtoend/testdata/analyze_types/clickhouse/exec.json @@ -0,0 +1,5 @@ +{ + "command": "analyze", + "args": ["--dialect", "clickhouse", "--schema", "schema.sql", "query.sql"], + "contexts": ["base"] +} diff --git a/internal/engine/clickhouse/testgen/testdata/types/fixture.sql b/internal/endtoend/testdata/analyze_types/clickhouse/fixture.sql similarity index 100% rename from internal/engine/clickhouse/testgen/testdata/types/fixture.sql rename to internal/endtoend/testdata/analyze_types/clickhouse/fixture.sql diff --git a/internal/engine/clickhouse/testgen/testdata/types/analyze.json b/internal/endtoend/testdata/analyze_types/clickhouse/output.json similarity index 54% rename from internal/engine/clickhouse/testgen/testdata/types/analyze.json rename to internal/endtoend/testdata/analyze_types/clickhouse/output.json index 5ef32d6b93..d9a4d7934c 100644 --- a/internal/engine/clickhouse/testgen/testdata/types/analyze.json +++ b/internal/endtoend/testdata/analyze_types/clickhouse/output.json @@ -50,11 +50,11 @@ "name": "labels", "type": { "name": "array", + "nullable": true, "args": [ { "type": { - "name": "string", - "nullable": true + "name": "string" } } ] @@ -68,14 +68,7 @@ "args": [ { "type": { - "name": "array", - "args": [ - { - "type": { - "name": "uint8" - } - } - ] + "name": "uint8" } } ] @@ -85,15 +78,8 @@ { "name": "kind", "type": { - "name": "lowcardinality", - "args": [ - { - "type": { - "name": "string", - "nullable": true - } - } - ] + "name": "string", + "nullable": true }, "table": "things" }, @@ -107,126 +93,49 @@ { "name": "updated", "type": { - "name": "datetime64", - "args": [ - { - "int": 3 - }, - { - "string": "UTC" - } - ] + "name": "datetime64" }, "table": "things" }, { "name": "price", "type": { - "name": "decimal", - "args": [ - { - "int": 10 - }, - { - "int": 2 - } - ] + "name": "decimal" }, "table": "things" }, { "name": "status", "type": { - "name": "enum8", - "args": [ - { - "label": "active", - "int": 1 - }, - { - "label": "deleted", - "int": 2 - } - ] + "name": "enum8" }, "table": "things" }, { "name": "attrs", "type": { - "name": "map", - "args": [ - { - "type": { - "name": "string" - } - }, - { - "type": { - "name": "uint32" - } - } - ] + "name": "map" }, "table": "things" }, { "name": "pos", "type": { - "name": "tuple", - "args": [ - { - "type": { - "name": "float64" - } - }, - { - "type": { - "name": "float64" - } - } - ] + "name": "tuple" }, "table": "things" }, { "name": "geo", "type": { - "name": "tuple", - "args": [ - { - "label": "lat", - "type": { - "name": "float64" - } - }, - { - "label": "lon", - "type": { - "name": "float64" - } - } - ] + "name": "tuple" }, "table": "things" }, { "name": "scores", "type": { - "name": "map", - "args": [ - { - "type": { - "name": "string" - } - }, - { - "type": { - "name": "uint8", - "nullable": true - } - } - ] + "name": "map" }, "table": "things" }, @@ -247,12 +156,7 @@ { "name": "fixed", "type": { - "name": "fixedstring", - "args": [ - { - "int": 4 - } - ] + "name": "fixedstring" }, "table": "things" }, @@ -317,11 +221,11 @@ "name": "labels", "type": { "name": "array", + "nullable": true, "args": [ { "type": { - "name": "string", - "nullable": true + "name": "string" } } ] @@ -335,14 +239,7 @@ "args": [ { "type": { - "name": "array", - "args": [ - { - "type": { - "name": "uint8" - } - } - ] + "name": "uint8" } } ] @@ -352,15 +249,8 @@ { "name": "kind", "type": { - "name": "lowcardinality", - "args": [ - { - "type": { - "name": "string", - "nullable": true - } - } - ] + "name": "string", + "nullable": true }, "table": "things" }, @@ -374,126 +264,49 @@ { "name": "updated", "type": { - "name": "datetime64", - "args": [ - { - "int": 3 - }, - { - "string": "UTC" - } - ] + "name": "datetime64" }, "table": "things" }, { "name": "price", "type": { - "name": "decimal", - "args": [ - { - "int": 10 - }, - { - "int": 2 - } - ] + "name": "decimal" }, "table": "things" }, { "name": "status", "type": { - "name": "enum8", - "args": [ - { - "label": "active", - "int": 1 - }, - { - "label": "deleted", - "int": 2 - } - ] + "name": "enum8" }, "table": "things" }, { "name": "attrs", "type": { - "name": "map", - "args": [ - { - "type": { - "name": "string" - } - }, - { - "type": { - "name": "uint32" - } - } - ] + "name": "map" }, "table": "things" }, { "name": "pos", "type": { - "name": "tuple", - "args": [ - { - "type": { - "name": "float64" - } - }, - { - "type": { - "name": "float64" - } - } - ] + "name": "tuple" }, "table": "things" }, { "name": "geo", "type": { - "name": "tuple", - "args": [ - { - "label": "lat", - "type": { - "name": "float64" - } - }, - { - "label": "lon", - "type": { - "name": "float64" - } - } - ] + "name": "tuple" }, "table": "things" }, { "name": "scores", "type": { - "name": "map", - "args": [ - { - "type": { - "name": "string" - } - }, - { - "type": { - "name": "uint8", - "nullable": true - } - } - ] + "name": "map" }, "table": "things" }, @@ -514,12 +327,7 @@ { "name": "fixed", "type": { - "name": "fixedstring", - "args": [ - { - "int": 4 - } - ] + "name": "fixedstring" }, "table": "things" }, diff --git a/internal/engine/clickhouse/testgen/testdata/types/query.sql b/internal/endtoend/testdata/analyze_types/clickhouse/query.sql similarity index 100% rename from internal/engine/clickhouse/testgen/testdata/types/query.sql rename to internal/endtoend/testdata/analyze_types/clickhouse/query.sql diff --git a/internal/engine/clickhouse/testgen/testdata/types/schema.sql b/internal/endtoend/testdata/analyze_types/clickhouse/schema.sql similarity index 100% rename from internal/engine/clickhouse/testgen/testdata/types/schema.sql rename to internal/endtoend/testdata/analyze_types/clickhouse/schema.sql diff --git a/internal/engine/clickhouse/testgen/README.md b/internal/engine/clickhouse/testgen/README.md deleted file mode 100644 index 6ca09c6b59..0000000000 --- a/internal/engine/clickhouse/testgen/README.md +++ /dev/null @@ -1,82 +0,0 @@ -# ClickHouse testgen - -`testgen` records what ClickHouse itself reports about a set of sqlc queries, -so the answer can be committed as a golden file and held against what sqlc's -own analysis produces for the same schema and queries. - -It is a nested Go module with no dependencies beyond the standard library. -Run it from this directory: - -```bash -# Download the pinned clickhouse binary into the user cache directory. -go run . install - -# Analyze every query in query.sql against schema.sql with fixture.sql loaded. -go run . analyze --schema schema.sql --fixture fixture.sql query.sql -``` - -The binary is looked up in the `CLICKHOUSE` environment variable first, then -in the cache populated by `install`. The version is pinned in `install.go`, -whose asset table lists each platform's download and its SHA-512; a download -that does not match is discarded. Bumping the version means adding the new -release's assets and checksums to the table, and since a release can change -the query tree format and type inference, regenerating and reviewing the -goldens afterwards. - -## How it works - -Each query runs in its own `clickhouse local` process, which needs no server, -no network and no configuration: the schema and fixture are loaded fresh, the -query is explained and then executed, and the process exits. - -- **Types and nullability** come from the executed query's result header, - exactly as a driver would see them. -- **Provenance** comes from `EXPLAIN QUERY TREE`, whose resolved columns point - at the table expression they read from. References are followed through - subqueries, CTEs and unions to the base table. -- **Parameters** are invisible to ClickHouse, which substitutes `?` on the - client. Each `?`, `sqlc.arg()` and `sqlc.narg()` is replaced by a constant - expression that carries its ordinal, `(NULL + k)`, or `toUInt64(4294967295 + k)` - after `LIMIT` and `OFFSET`. The query tree prints the expression each folded - constant came from, so the placeholder is found again and described by the - operand it is compared with. Parameters of `INSERT ... VALUES` map onto the - target columns reported by `DESCRIBE TABLE`. - -The output has the shape of `sqlc analyze`, except that each column's type is -one expression rather than a name and flags. A type is a call: a lowercased -`name` applied to `args`, each of which carries an optional `label` and -exactly one of `type`, `int`, `bool` or `string`. `Array`, `Map` and -`LowCardinality` are ordinary names in that grammar, so nothing about nesting -is lost. Nullability is an attribute of a type rather than a wrapper: -`Nullable(T)` becomes `T` with `nullable` set, at whatever depth ClickHouse -wrote it. - -```json -{"name": "map", "args": [ - {"type": {"name": "string"}}, - {"type": {"name": "uint8", "nullable": true}}]} - -{"name": "tuple", "args": [ - {"label": "lat", "type": {"name": "float64"}}, - {"label": "lon", "type": {"name": "float64"}}]} - -{"name": "enum8", "args": [{"label": "active", "int": 1}, {"label": "deleted", "int": 2}]} - -{"name": "datetime64", "args": [{"int": 3}, {"string": "UTC"}]} -``` - -An identifier argument, such as the function in `AggregateFunction(uniq, -String)`, is a type with no arguments. Resolving the names is left to -whoever reads the output. - -## Tests - -`testdata//` holds a `schema.sql`, `query.sql`, an optional -`fixture.sql` and the expected `analyze.json`. The test skips unless a binary is -available. - -```bash -go run . install -go test . -go test . -update # rewrite every analyze.json -``` diff --git a/internal/engine/clickhouse/testgen/go.mod b/internal/engine/clickhouse/testgen/go.mod deleted file mode 100644 index fdace64397..0000000000 --- a/internal/engine/clickhouse/testgen/go.mod +++ /dev/null @@ -1,3 +0,0 @@ -module github.com/sqlc-dev/sqlc/internal/engine/clickhouse/testgen - -go 1.24.0 diff --git a/internal/engine/clickhouse/testgen/main.go b/internal/engine/clickhouse/testgen/main.go deleted file mode 100644 index d798a65cc9..0000000000 --- a/internal/engine/clickhouse/testgen/main.go +++ /dev/null @@ -1,132 +0,0 @@ -// Command testgen records what ClickHouse itself says about a set of sqlc -// queries, so the answer can be committed as a golden file and compared with -// what sqlc's own analysis produces. -// -// It loads a schema and a fixture into an ephemeral `clickhouse local` -// process, runs each query found in a sqlc query file against that data, and -// prints the result column types, nullability and source tables along with -// the parameters each query binds, in the JSON shape `sqlc analyze` prints -// with each type written as a call expression. -// -// The clickhouse binary is downloaded once per pinned version with -// `testgen install`, or supplied through the CLICKHOUSE environment variable. -// -// Usage: -// -// go run . install -// go run . analyze --schema schema.sql --fixture fixture.sql query.sql -package main - -import ( - "context" - "encoding/json" - "errors" - "flag" - "fmt" - "io" - "os" - "runtime" -) - -func main() { - if err := run(context.Background(), os.Args[1:], os.Stdout, os.Stderr); err != nil { - fmt.Fprintln(os.Stderr, "testgen:", err) - os.Exit(1) - } -} - -const usage = `usage: - testgen install [-version V] - download the pinned clickhouse binary into the user cache directory - testgen analyze [-clickhouse PATH] --schema FILE [--fixture FILE] QUERY_FILE - analyze every query in QUERY_FILE and print the result as JSON - -The binary is looked up in the CLICKHOUSE environment variable first, then in -the cache directory populated by "testgen install".` - -func run(ctx context.Context, args []string, stdout, stderr io.Writer) error { - if len(args) == 0 { - fmt.Fprintln(stderr, usage) - return errors.New("a command is required") - } - switch args[0] { - case "install": - return runInstall(ctx, args[1:], stdout, stderr) - case "analyze": - return runAnalyze(ctx, args[1:], stdout, stderr) - case "help", "-h", "--help": - fmt.Fprintln(stdout, usage) - return nil - default: - fmt.Fprintln(stderr, usage) - return fmt.Errorf("unknown command %q", args[0]) - } -} - -func runInstall(ctx context.Context, args []string, stdout, stderr io.Writer) error { - fs := flag.NewFlagSet("install", flag.ContinueOnError) - fs.SetOutput(stderr) - version := fs.String("version", DefaultVersion, "ClickHouse release to install") - if err := fs.Parse(args); err != nil { - return err - } - path, err := Install(ctx, *version, runtime.GOOS, runtime.GOARCH, stderr) - if err != nil { - return err - } - fmt.Fprintln(stdout, path) - return nil -} - -func runAnalyze(ctx context.Context, args []string, stdout, stderr io.Writer) error { - fs := flag.NewFlagSet("analyze", flag.ContinueOnError) - fs.SetOutput(stderr) - binary := fs.String("clickhouse", "", "path to the clickhouse binary (defaults to $CLICKHOUSE, then the cache)") - schemaPath := fs.String("schema", "", "path to the schema file") - fixturePath := fs.String("fixture", "", "path to the fixture file loaded after the schema") - if err := fs.Parse(args); err != nil { - return err - } - if fs.NArg() != 1 { - fmt.Fprintln(stderr, usage) - return errors.New("analyze takes exactly one query file") - } - if *schemaPath == "" { - return errors.New("--schema is required") - } - if *binary == "" { - path, err := Locate() - if err != nil { - return err - } - *binary = path - } - - schema, err := os.ReadFile(*schemaPath) - if err != nil { - return err - } - var fixture []byte - if *fixturePath != "" { - fixture, err = os.ReadFile(*fixturePath) - if err != nil { - return err - } - } - querySrc, err := os.ReadFile(fs.Arg(0)) - if err != nil { - return err - } - queries, err := parseQueries(string(querySrc)) - if err != nil { - return fmt.Errorf("%s: %w", fs.Arg(0), err) - } - - out, err := analyze(ctx, local{binary: *binary}, string(schema), string(fixture), queries) - if err != nil { - return err - } - enc := json.NewEncoder(stdout) - enc.SetIndent("", " ") - return enc.Encode(out) -} diff --git a/internal/engine/clickhouse/testgen/testdata/analyze_params/query.sql b/internal/engine/clickhouse/testgen/testdata/analyze_params/query.sql deleted file mode 100644 index 5405a5baa8..0000000000 --- a/internal/engine/clickhouse/testgen/testdata/analyze_params/query.sql +++ /dev/null @@ -1,5 +0,0 @@ --- name: GetEvent :one -SELECT id, name FROM events WHERE id = ?; - --- name: FilterEvents :many -SELECT id, name FROM events WHERE name = ? AND amount > ?; diff --git a/internal/engine/clickhouse/testgen/testdata/analyze_params/schema.sql b/internal/engine/clickhouse/testgen/testdata/analyze_params/schema.sql deleted file mode 100644 index 29960ee63d..0000000000 --- a/internal/engine/clickhouse/testgen/testdata/analyze_params/schema.sql +++ /dev/null @@ -1,7 +0,0 @@ -CREATE TABLE events ( - id UInt64, - name String, - tag Nullable(String), - amount Float64, - created DateTime -) ENGINE = MergeTree ORDER BY id; diff --git a/internal/engine/clickhouse/testgen/testdata/subqueries/analyze.json b/internal/engine/clickhouse/testgen/testdata/subqueries/analyze.json deleted file mode 100644 index 92a4c2fa6a..0000000000 --- a/internal/engine/clickhouse/testgen/testdata/subqueries/analyze.json +++ /dev/null @@ -1,159 +0,0 @@ -[ - { - "name": "Cte", - "cmd": ":many", - "columns": [ - { - "name": "id", - "type": { - "name": "uint64" - }, - "table": "events" - }, - { - "name": "tag", - "type": { - "name": "string", - "nullable": true - }, - "table": "events" - }, - { - "name": "cnt", - "type": { - "name": "uint64" - } - } - ], - "params": [] - }, - { - "name": "Aliased", - "cmd": ":many", - "columns": [ - { - "name": "x", - "type": { - "name": "uint64" - }, - "table": "events" - }, - { - "name": "total", - "type": { - "name": "float64" - } - }, - { - "name": "email", - "type": { - "name": "string" - } - } - ], - "params": [] - }, - { - "name": "Star", - "cmd": ":many", - "columns": [ - { - "name": "id", - "type": { - "name": "uint64" - }, - "table": "users" - }, - { - "name": "email", - "type": { - "name": "string" - }, - "table": "users" - }, - { - "name": "e.id", - "type": { - "name": "uint64" - }, - "table": "events" - }, - { - "name": "name", - "type": { - "name": "string" - }, - "table": "events" - }, - { - "name": "tag", - "type": { - "name": "string", - "nullable": true - }, - "table": "events" - }, - { - "name": "amount", - "type": { - "name": "float64" - }, - "table": "events" - } - ], - "params": [] - }, - { - "name": "Union", - "cmd": ":many", - "columns": [ - { - "name": "id", - "type": { - "name": "uint64" - }, - "table": "events" - }, - { - "name": "name", - "type": { - "name": "string" - }, - "table": "events" - } - ], - "params": [] - }, - { - "name": "ScalarSubquery", - "cmd": ":many", - "columns": [ - { - "name": "id", - "type": { - "name": "uint64" - }, - "table": "events" - }, - { - "name": "user_count", - "type": { - "name": "uint64", - "nullable": true - } - } - ], - "params": [ - { - "number": 1, - "column": { - "name": "email", - "type": { - "name": "string" - }, - "table": "users" - } - } - ] - } -] diff --git a/internal/engine/clickhouse/testgen/testgen_test.go b/internal/engine/clickhouse/testgen/testgen_test.go deleted file mode 100644 index a7d0e588ae..0000000000 --- a/internal/engine/clickhouse/testgen/testgen_test.go +++ /dev/null @@ -1,55 +0,0 @@ -package main - -import ( - "bytes" - "context" - "flag" - "os" - "path/filepath" - "testing" -) - -var update = flag.Bool("update", false, "rewrite the expected analyze.json of every case") - -// TestAnalyze runs the CLI over each directory under testdata, which holds -// the same files a sqlc analyze case does plus a fixture, and compares the -// output with the committed analyze.json. It needs the clickhouse binary and -// skips when none is installed. -func TestAnalyze(t *testing.T) { - if _, err := Locate(); err != nil { - t.Skip(err) - } - dirs, err := filepath.Glob("testdata/*") - if err != nil { - t.Fatal(err) - } - for _, dir := range dirs { - t.Run(filepath.Base(dir), func(t *testing.T) { - args := []string{"analyze", "--schema", filepath.Join(dir, "schema.sql")} - if _, err := os.Stat(filepath.Join(dir, "fixture.sql")); err == nil { - args = append(args, "--fixture", filepath.Join(dir, "fixture.sql")) - } - args = append(args, filepath.Join(dir, "query.sql")) - - var stdout, stderr bytes.Buffer - if err := run(context.Background(), args, &stdout, &stderr); err != nil { - t.Fatalf("%v\n%s", err, stderr.String()) - } - - golden := filepath.Join(dir, "analyze.json") - if *update { - if err := os.WriteFile(golden, stdout.Bytes(), 0o644); err != nil { - t.Fatal(err) - } - return - } - want, err := os.ReadFile(golden) - if err != nil { - t.Fatal(err) - } - if !bytes.Equal(want, stdout.Bytes()) { - t.Errorf("output differs from %s (run with -update to rewrite)\n--- want\n%s\n--- got\n%s", golden, want, stdout.Bytes()) - } - }) - } -} diff --git a/internal/testcheck/README.md b/internal/testcheck/README.md new file mode 100644 index 0000000000..b56553bd55 --- /dev/null +++ b/internal/testcheck/README.md @@ -0,0 +1,40 @@ +# testcheck + +`testcheck` verifies the analyze cases under `internal/endtoend/testdata` +against a real database. It generates nothing. Each engine package reads a +case's `schema.sql`, `fixture.sql` and `query.sql`, asks the database what it +makes of the queries, prints the answer in the JSON shape `sqlc analyze` +prints, and compares it with the `output.json` the case committed, byte for +byte. A difference means sqlc's analysis disagrees with the database. + +It is a nested Go module with no dependencies beyond the standard library, +so it never shares code with the analysis it checks. Run it from this +directory: + +```bash +go run . install clickhouse # download the pinned clickhouse binary once +go run . check # check every engine whose database is available +go run . check clickhouse # check one engine +go test ./... # the same checks as tests; engines without a database skip +``` + +## Cases + +A case is an `analyze_/` directory whose `exec.json` runs the +analyze command. `fixture.sql` is optional and is loaded after the schema, so +the queries run against real rows. A case that asks for `--ast` is skipped, +since only sqlc can print that. + +## Engines + +Each engine is its own package. + +- **`clickhouse`** needs no server. Each case runs in an ephemeral + `clickhouse local` process, downloaded once per pinned version by + `install` into the user cache directory, or supplied through the + `CLICKHOUSE` environment variable. The pinned version and the SHA-512 of + each platform's download live in `clickhouse/install.go`. Column types + come from the executed query's result header, provenance from + `EXPLAIN QUERY TREE`, and parameters from sentinel constants substituted + for `?`, `sqlc.arg()` and `sqlc.narg()`, since ClickHouse itself never + sees a placeholder. diff --git a/internal/engine/clickhouse/testgen/analyze.go b/internal/testcheck/clickhouse/analyze.go similarity index 98% rename from internal/engine/clickhouse/testgen/analyze.go rename to internal/testcheck/clickhouse/analyze.go index b8d6cc5263..36b08a0a97 100644 --- a/internal/engine/clickhouse/testgen/analyze.go +++ b/internal/testcheck/clickhouse/analyze.go @@ -1,4 +1,4 @@ -package main +package clickhouse import ( "context" @@ -9,8 +9,8 @@ import ( "strings" ) -// The output follows the JSON `sqlc analyze` prints, with each type written -// as a call expression (see types.go) instead of a name and flags. +// The output is the JSON `sqlc analyze` prints, so a case's committed +// output.json can be compared with it byte for byte. type analyzedQuery struct { Name string `json:"name"` diff --git a/internal/testcheck/clickhouse/check.go b/internal/testcheck/clickhouse/check.go new file mode 100644 index 0000000000..983bd9bc8f --- /dev/null +++ b/internal/testcheck/clickhouse/check.go @@ -0,0 +1,68 @@ +// Package clickhouse verifies the ClickHouse analyze cases under +// internal/endtoend/testdata against what ClickHouse itself reports. +// +// Each case's schema and fixture are loaded into an ephemeral +// `clickhouse local` process and its queries are run there. Result column +// types come from the executed query's result header, provenance from +// EXPLAIN QUERY TREE, and parameters from sentinel constants substituted +// for the placeholders, since ClickHouse itself never sees a ?. The answer +// is printed in the JSON shape sqlc analyze prints and compared with the +// case's committed output.json byte for byte. +package clickhouse + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "os" + + "github.com/sqlc-dev/sqlc/internal/testcheck/endtoend" +) + +// Engine is the name of this engine's directory under each analyze case. +const Engine = "clickhouse" + +// Analyze runs a case's queries through the clickhouse binary and returns +// the analysis in the JSON shape sqlc analyze prints. +func Analyze(ctx context.Context, binary string, c endtoend.Case) ([]byte, error) { + schema, err := os.ReadFile(c.Schema) + if err != nil { + return nil, err + } + var fixture []byte + if c.Fixture != "" { + if fixture, err = os.ReadFile(c.Fixture); err != nil { + return nil, err + } + } + src, err := os.ReadFile(c.Query) + if err != nil { + return nil, err + } + queries, err := parseQueries(string(src)) + if err != nil { + return nil, fmt.Errorf("%s: %w", c.Query, err) + } + out, err := analyze(ctx, local{binary: binary}, string(schema), string(fixture), queries) + if err != nil { + return nil, err + } + var buf bytes.Buffer + enc := json.NewEncoder(&buf) + enc.SetIndent("", " ") + if err := enc.Encode(out); err != nil { + return nil, err + } + return buf.Bytes(), nil +} + +// Check compares what ClickHouse reports for a case with the output the +// case committed, returning a diff when they differ. +func Check(ctx context.Context, binary string, c endtoend.Case) (string, error) { + got, err := Analyze(ctx, binary, c) + if err != nil { + return "", err + } + return c.Compare(got) +} diff --git a/internal/testcheck/clickhouse/clickhouse_test.go b/internal/testcheck/clickhouse/clickhouse_test.go new file mode 100644 index 0000000000..f9181ed9dc --- /dev/null +++ b/internal/testcheck/clickhouse/clickhouse_test.go @@ -0,0 +1,35 @@ +package clickhouse + +import ( + "context" + "testing" + + "github.com/sqlc-dev/sqlc/internal/testcheck/endtoend" +) + +// TestEndToEnd verifies every ClickHouse analyze case against ClickHouse. +// It skips unless the clickhouse binary is installed. +func TestEndToEnd(t *testing.T) { + binary, err := Locate() + if err != nil { + t.Skip(err) + } + cases, err := endtoend.Cases(Engine) + if err != nil { + t.Fatal(err) + } + if len(cases) == 0 { + t.Fatal("no clickhouse analyze cases found") + } + for _, c := range cases { + t.Run(c.Name, func(t *testing.T) { + diff, err := Check(context.Background(), binary, c) + if err != nil { + t.Fatal(err) + } + if diff != "" { + t.Errorf("%s does not match what ClickHouse reports (-committed +clickhouse):\n%s", c.Output, diff) + } + }) + } +} diff --git a/internal/engine/clickhouse/testgen/install.go b/internal/testcheck/clickhouse/install.go similarity index 97% rename from internal/engine/clickhouse/testgen/install.go rename to internal/testcheck/clickhouse/install.go index 82cd53a55e..0af8e2e436 100644 --- a/internal/engine/clickhouse/testgen/install.go +++ b/internal/testcheck/clickhouse/install.go @@ -1,4 +1,4 @@ -package main +package clickhouse import ( "archive/tar" @@ -114,7 +114,7 @@ func Locate() (string, error) { return "", err } if _, err := os.Stat(path); err != nil { - return "", fmt.Errorf("clickhouse %s is not installed: run `testgen install` or set CLICKHOUSE to a clickhouse binary", DefaultVersion) + return "", fmt.Errorf("clickhouse %s is not installed: run `go run . install clickhouse` in internal/testcheck, or set CLICKHOUSE to a clickhouse binary", DefaultVersion) } return path, nil } diff --git a/internal/engine/clickhouse/testgen/local.go b/internal/testcheck/clickhouse/local.go similarity index 99% rename from internal/engine/clickhouse/testgen/local.go rename to internal/testcheck/clickhouse/local.go index 36f899b510..5539fc79ce 100644 --- a/internal/engine/clickhouse/testgen/local.go +++ b/internal/testcheck/clickhouse/local.go @@ -1,4 +1,4 @@ -package main +package clickhouse import ( "bytes" diff --git a/internal/engine/clickhouse/testgen/queries.go b/internal/testcheck/clickhouse/queries.go similarity index 99% rename from internal/engine/clickhouse/testgen/queries.go rename to internal/testcheck/clickhouse/queries.go index c4880c284a..40f2ecdc36 100644 --- a/internal/engine/clickhouse/testgen/queries.go +++ b/internal/testcheck/clickhouse/queries.go @@ -1,4 +1,4 @@ -package main +package clickhouse import ( "fmt" diff --git a/internal/engine/clickhouse/testgen/tree.go b/internal/testcheck/clickhouse/tree.go similarity index 99% rename from internal/engine/clickhouse/testgen/tree.go rename to internal/testcheck/clickhouse/tree.go index 9ff0d329fc..d0a8a38924 100644 --- a/internal/engine/clickhouse/testgen/tree.go +++ b/internal/testcheck/clickhouse/tree.go @@ -1,4 +1,4 @@ -package main +package clickhouse import ( "fmt" diff --git a/internal/engine/clickhouse/testgen/types.go b/internal/testcheck/clickhouse/types.go similarity index 99% rename from internal/engine/clickhouse/testgen/types.go rename to internal/testcheck/clickhouse/types.go index f396d78f20..77ed8d8c19 100644 --- a/internal/engine/clickhouse/testgen/types.go +++ b/internal/testcheck/clickhouse/types.go @@ -1,4 +1,4 @@ -package main +package clickhouse import ( "strconv" diff --git a/internal/testcheck/endtoend/endtoend.go b/internal/testcheck/endtoend/endtoend.go new file mode 100644 index 0000000000..51d033e981 --- /dev/null +++ b/internal/testcheck/endtoend/endtoend.go @@ -0,0 +1,180 @@ +// Package endtoend finds the analyze cases under internal/endtoend/testdata +// and compares an engine's own answer with the output a case committed. +package endtoend + +import ( + "bytes" + "encoding/json" + "errors" + "fmt" + "os" + "path/filepath" + "runtime" + "strings" +) + +// Case is one analyze case: the files sqlc analyze ran with, the fixture +// loaded before the queries run against a real database, and the output +// sqlc committed. +type Case struct { + Name string // analyze_params/clickhouse + Dir string + Schema string + Query string + Fixture string // empty when the case has no fixture.sql + Output string +} + +// Testdata returns the end-to-end testdata directory, found relative to this +// source file so the working directory does not matter. +func Testdata() (string, error) { + _, file, _, ok := runtime.Caller(0) + if !ok { + return "", errors.New("cannot locate the testcheck source directory") + } + dir := filepath.Join(filepath.Dir(file), "..", "..", "endtoend", "testdata") + if _, err := os.Stat(dir); err != nil { + return "", err + } + return filepath.Clean(dir), nil +} + +// Cases lists the analyze cases for an engine: every analyze_*/ +// directory whose exec.json runs the analyze command. A case that asks for +// the AST is skipped, since only sqlc can print that. +func Cases(engine string) ([]Case, error) { + root, err := Testdata() + if err != nil { + return nil, err + } + dirs, err := filepath.Glob(filepath.Join(root, "analyze_*", engine)) + if err != nil { + return nil, err + } + var cases []Case + for _, dir := range dirs { + c, ok, err := load(dir) + if err != nil { + return nil, fmt.Errorf("%s: %w", dir, err) + } + if ok { + cases = append(cases, c) + } + } + return cases, nil +} + +func load(dir string) (Case, bool, error) { + blob, err := os.ReadFile(filepath.Join(dir, "exec.json")) + if errors.Is(err, os.ErrNotExist) { + return Case{}, false, nil + } + if err != nil { + return Case{}, false, err + } + var exec struct { + Command string `json:"command"` + Args []string `json:"args"` + } + if err := json.Unmarshal(blob, &exec); err != nil { + return Case{}, false, fmt.Errorf("exec.json: %w", err) + } + if exec.Command != "analyze" { + return Case{}, false, nil + } + var schema, query string + for i := 0; i < len(exec.Args); i++ { + switch arg := exec.Args[i]; arg { + case "--schema", "-s", "--dialect", "-d": + i++ + if arg == "--schema" || arg == "-s" { + if i < len(exec.Args) { + schema = exec.Args[i] + } + } + case "--ast": + return Case{}, false, nil + default: + if !strings.HasPrefix(arg, "-") { + query = arg + } + } + } + if schema == "" || query == "" { + return Case{}, false, errors.New("exec.json: analyze needs --schema and a query file") + } + c := Case{ + Name: filepath.Join(filepath.Base(filepath.Dir(dir)), filepath.Base(dir)), + Dir: dir, + Schema: filepath.Join(dir, schema), + Query: filepath.Join(dir, query), + Output: filepath.Join(dir, "output.json"), + } + if fixture := filepath.Join(dir, "fixture.sql"); fileExists(fixture) { + c.Fixture = fixture + } + return c, true, nil +} + +func fileExists(path string) bool { + _, err := os.Stat(path) + return err == nil +} + +// Compare checks an engine's answer against the case's committed output +// byte for byte and returns a line diff when they differ, or "" when they +// match. +func (c Case) Compare(got []byte) (string, error) { + want, err := os.ReadFile(c.Output) + if err != nil { + return "", err + } + if bytes.Equal(want, got) { + return "", nil + } + return Diff(string(want), string(got)), nil +} + +// Diff is a line diff of two texts, marking lines only in want with "-" and +// lines only in got with "+". Outputs are small, so a plain longest common +// subsequence is fine. +func Diff(want, got string) string { + a := strings.Split(strings.TrimSuffix(want, "\n"), "\n") + b := strings.Split(strings.TrimSuffix(got, "\n"), "\n") + lcs := make([][]int, len(a)+1) + for i := range lcs { + lcs[i] = make([]int, len(b)+1) + } + for i := len(a) - 1; i >= 0; i-- { + for j := len(b) - 1; j >= 0; j-- { + if a[i] == b[j] { + lcs[i][j] = lcs[i+1][j+1] + 1 + } else { + lcs[i][j] = max(lcs[i+1][j], lcs[i][j+1]) + } + } + } + var out strings.Builder + i, j := 0, 0 + for i < len(a) && j < len(b) { + switch { + case a[i] == b[j]: + fmt.Fprintf(&out, " %s\n", a[i]) + i++ + j++ + case lcs[i+1][j] >= lcs[i][j+1]: + fmt.Fprintf(&out, "- %s\n", a[i]) + i++ + default: + fmt.Fprintf(&out, "+ %s\n", b[j]) + j++ + } + } + for ; i < len(a); i++ { + fmt.Fprintf(&out, "- %s\n", a[i]) + } + for ; j < len(b); j++ { + fmt.Fprintf(&out, "+ %s\n", b[j]) + } + return out.String() +} diff --git a/internal/testcheck/go.mod b/internal/testcheck/go.mod new file mode 100644 index 0000000000..154e10028e --- /dev/null +++ b/internal/testcheck/go.mod @@ -0,0 +1,3 @@ +module github.com/sqlc-dev/sqlc/internal/testcheck + +go 1.24.0 diff --git a/internal/testcheck/main.go b/internal/testcheck/main.go new file mode 100644 index 0000000000..3d77d4f508 --- /dev/null +++ b/internal/testcheck/main.go @@ -0,0 +1,116 @@ +// Command testcheck verifies the analyze cases under internal/endtoend/testdata +// against a real database. It generates nothing: each engine package reads a +// case's schema, fixture and queries, asks the database what it makes of +// them, and compares the answer with the output.json the case committed. +// +// Usage, from this directory: +// +// go run . install clickhouse # download the pinned clickhouse binary +// go run . check [engine] # check every case, or one engine's +// +// `go test ./...` runs the same checks as tests, skipping engines whose +// database is not available. +package main + +import ( + "context" + "errors" + "flag" + "fmt" + "io" + "os" + "runtime" + + "github.com/sqlc-dev/sqlc/internal/testcheck/clickhouse" + "github.com/sqlc-dev/sqlc/internal/testcheck/endtoend" +) + +func main() { + if err := run(context.Background(), os.Args[1:], os.Stdout, os.Stderr); err != nil { + fmt.Fprintln(os.Stderr, "testcheck:", err) + os.Exit(1) + } +} + +const usage = `usage: + testcheck install clickhouse [-version V] + download the pinned clickhouse binary into the user cache directory + testcheck check [engine] + verify the analyze cases against the database, for every engine or one` + +func run(ctx context.Context, args []string, stdout, stderr io.Writer) error { + if len(args) == 0 { + fmt.Fprintln(stderr, usage) + return errors.New("a command is required") + } + switch args[0] { + case "install": + return install(ctx, args[1:], stdout, stderr) + case "check": + return check(ctx, args[1:], stdout, stderr) + case "help", "-h", "--help": + fmt.Fprintln(stdout, usage) + return nil + } + fmt.Fprintln(stderr, usage) + return fmt.Errorf("unknown command %q", args[0]) +} + +func install(ctx context.Context, args []string, stdout, stderr io.Writer) error { + if len(args) == 0 || args[0] != clickhouse.Engine { + return errors.New("install takes the engine to install: clickhouse") + } + fs := flag.NewFlagSet("install", flag.ContinueOnError) + fs.SetOutput(stderr) + version := fs.String("version", clickhouse.DefaultVersion, "ClickHouse release to install") + if err := fs.Parse(args[1:]); err != nil { + return err + } + path, err := clickhouse.Install(ctx, *version, runtime.GOOS, runtime.GOARCH, stderr) + if err != nil { + return err + } + fmt.Fprintln(stdout, path) + return nil +} + +func check(ctx context.Context, args []string, stdout, stderr io.Writer) error { + engine := "" + if len(args) > 0 { + engine = args[0] + } + failed := 0 + if engine == "" || engine == clickhouse.Engine { + binary, err := clickhouse.Locate() + if err != nil { + if engine != "" { + return err + } + fmt.Fprintf(stderr, "skipping clickhouse: %v\n", err) + } else { + cases, err := endtoend.Cases(clickhouse.Engine) + if err != nil { + return err + } + for _, c := range cases { + diff, err := clickhouse.Check(ctx, binary, c) + switch { + case err != nil: + failed++ + fmt.Fprintf(stdout, "ERROR %s: %v\n", c.Name, err) + case diff != "": + failed++ + fmt.Fprintf(stdout, "FAIL %s (-committed +clickhouse)\n%s\n", c.Name, diff) + default: + fmt.Fprintf(stdout, "ok %s\n", c.Name) + } + } + } + } else { + return fmt.Errorf("unknown engine %q", engine) + } + if failed > 0 { + return fmt.Errorf("%d case(s) did not match", failed) + } + return nil +} From dd37cbc765e72a9deb118cdc3daacfb79bba1c23 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 2 Sep 2026 21:02:02 +0000 Subject: [PATCH 09/10] Make sqlc's ClickHouse analysis agree with ClickHouse testcheck showed sqlc disagreeing with ClickHouse on every analyze case that went beyond plain column references. This makes the six ClickHouse cases match the database byte for byte, and moves the testcheck command under cmd/testcheck. Types are carried as expressions. The ClickHouse converter keeps a column's full spelling in the type name's Spelling, the schema stores it as the attribute's declared type, and the analysis core writes each column and parameter a TypeExpr from it, so Array(Nullable(String)), Map(String, Nullable(UInt8)), Tuple(lat Float64, lon Float64) and Decimal(10, 2) survive intact. The analyze command prints that expression when the core produced one. Functions are typed. A ClickHouse function seed of 581 signatures replaces the single count() entry, with "$n" naming the type of the nth argument and never_null marking results that stay non-null. The dialect declares that functions propagate nullability, that comparisons yield UInt8 while true is Bool, that LIMIT counts are UInt64, and that an unconstrained placeholder is Nothing. The analyzer scores overloads, types ORDER BY, LIMIT and OFFSET, follows IN (SELECT ...) into the subquery, makes COALESCE null only when every argument is, and names a placeholder compared with a function call after the function. The converter gives unaliased expression columns ClickHouse's own names, such as sum(amount) and plus(id, 1), captures aliases on every node kind, converts a scalar subquery as a value rather than EXISTS, converts coalesce and ifNull to COALESCE, handles WITH elements as the parser produces them, and counts positions from zero so star expansion and parameter renumbering line up. ClickHouse joins the preprocessed engines, so sqlc.arg() and sqlc.narg() work with ? binding. A dialect that names the second id of a join e.id says so in its seed and the analyzer qualifies such columns. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01Y6XkyWnx7iJFEb8q3AnYps --- internal/cmd/analyze.go | 43 +- internal/compiler/parse_core.go | 2 + internal/compiler/query.go | 6 + internal/core/analysis.go | 8 +- internal/core/analyzer/analyzer.go | 44 ++ internal/core/analyzer/expr.go | 177 +++++- internal/core/analyzer/projection.go | 72 +++ internal/core/dialect.go | 81 +++ internal/core/proc.go | 36 +- internal/core/schema/schema.go | 16 +- internal/core/seed/seed.go | 55 +- internal/core/typeexpr.go | 198 ++++++ .../clickhouse/output.json | 137 ++++- .../analyze_subqueries/clickhouse/output.json | 104 +++- .../analyze_subqueries/clickhouse/query.sql | 9 + .../analyze_types/clickhouse/output.json | 244 +++++++- .../parse_basic/clickhouse/stdout.txt | 5 +- internal/engine/clickhouse/convert.go | 273 ++++++-- .../engine/clickhouse/dialect/dialect.json | 12 +- .../engine/clickhouse/dialect/functions.jsonl | 582 +++++++++++++++++- internal/sql/preprocess/dialect.go | 12 +- internal/sql/preprocess/preprocess.go | 8 +- internal/testcheck/README.md | 6 +- internal/testcheck/clickhouse/analyze.go | 9 +- internal/testcheck/clickhouse/install.go | 2 +- internal/testcheck/clickhouse/queries.go | 21 +- .../testcheck/{ => cmd/testcheck}/main.go | 4 +- 27 files changed, 1964 insertions(+), 202 deletions(-) create mode 100644 internal/core/typeexpr.go rename internal/testcheck/{ => cmd/testcheck}/main.go (94%) diff --git a/internal/cmd/analyze.go b/internal/cmd/analyze.go index 7d52b3fe2f..ba278498b6 100644 --- a/internal/cmd/analyze.go +++ b/internal/cmd/analyze.go @@ -11,6 +11,7 @@ import ( "github.com/sqlc-dev/sqlc/internal/compiler" "github.com/sqlc-dev/sqlc/internal/config" + "github.com/sqlc-dev/sqlc/internal/core" "github.com/sqlc-dev/sqlc/internal/multierr" "github.com/sqlc-dev/sqlc/internal/opts" "github.com/sqlc-dev/sqlc/internal/sql/ast" @@ -204,9 +205,9 @@ type analyzedQuery struct { } type analyzedColumn struct { - Name string `json:"name"` - Type *analyzedType `json:"type,omitempty"` - Table string `json:"table,omitempty"` + Name string `json:"name"` + Type *core.TypeExpr `json:"type,omitempty"` + Table string `json:"table,omitempty"` } type analyzedParam struct { @@ -214,26 +215,6 @@ type analyzedParam struct { Column analyzedColumn `json:"column"` } -// analyzedType writes a type as a call expression: a name applied to -// arguments that are other types, integers, booleans or strings, each with -// an optional label, and a nullable flag at whatever depth it applies. An -// array of text is array(text); a nullable column of it has nullable set on -// the array node. Names are recorded as the engine reports them and resolve -// against the catalog afterwards. -type analyzedType struct { - Name string `json:"name"` - Nullable bool `json:"nullable,omitempty"` - Args []analyzedArg `json:"args,omitempty"` -} - -type analyzedArg struct { - Label string `json:"label,omitempty"` - Type *analyzedType `json:"type,omitempty"` - Int *int64 `json:"int,omitempty"` - Bool *bool `json:"bool,omitempty"` - String *string `json:"string,omitempty"` -} - func newAnalyzedQuery(q *compiler.Query, includeAST bool) analyzedQuery { aq := analyzedQuery{ Name: q.Metadata.Name, @@ -270,20 +251,24 @@ func newAnalyzedColumn(col *compiler.Column) analyzedColumn { return ac } -// newAnalyzedType builds the type expression the compiler's flat column -// description amounts to: the data type wrapped in one array node per -// dimension, with the column's nullability on the outermost node. -func newAnalyzedType(col *compiler.Column) *analyzedType { +// newAnalyzedType is the column's type as an expression: the one the +// analysis core wrote when it did, otherwise the flat description the +// compiler holds, which is the data type wrapped in one array node per +// dimension with the column's nullability on the outermost node. +func newAnalyzedType(col *compiler.Column) *core.TypeExpr { + if col.TypeExpr != nil { + return col.TypeExpr + } if col.DataType == "" { return nil } - t := &analyzedType{Name: col.DataType} + t := core.ParseTypeExpr(col.DataType) dims := col.ArrayDims if col.IsArray && dims == 0 { dims = 1 } for i := 0; i < dims; i++ { - t = &analyzedType{Name: "array", Args: []analyzedArg{{Type: t}}} + t = &core.TypeExpr{Name: "array", Args: []core.TypeArg{{Type: t}}} } t.Nullable = !col.NotNull return t diff --git a/internal/compiler/parse_core.go b/internal/compiler/parse_core.go index 6cae7586c3..7e5685a962 100644 --- a/internal/compiler/parse_core.go +++ b/internal/compiler/parse_core.go @@ -105,6 +105,7 @@ func coreColumn(c core.Column) *Column { DataType: c.DataType, NotNull: c.NotNull, IsArray: c.IsArray, + TypeExpr: c.Type, } // The core reports arrays without dimensions, and codegen renders one // "[]" per dimension. @@ -129,6 +130,7 @@ func coreParamColumn(p core.Parameter, params *named.ParamSet) *Column { DataType: p.DataType, NotNull: p.NotNull, IsArray: p.IsArray, + TypeExpr: p.Type, } if p.IsArray { col.ArrayDims = 1 diff --git a/internal/compiler/query.go b/internal/compiler/query.go index b3cf9d6154..8753d5a7cc 100644 --- a/internal/compiler/query.go +++ b/internal/compiler/query.go @@ -1,6 +1,7 @@ package compiler import ( + "github.com/sqlc-dev/sqlc/internal/core" "github.com/sqlc-dev/sqlc/internal/metadata" "github.com/sqlc-dev/sqlc/internal/sql/ast" "github.com/sqlc-dev/sqlc/internal/sql/catalog" @@ -37,6 +38,11 @@ type Column struct { Type *ast.TypeName EmbedTable *ast.TableName + // TypeExpr is the type as the analysis core wrote it, with the + // arguments and nesting DataType and IsArray flatten away. It is unset + // on the legacy path. + TypeExpr *core.TypeExpr + IsSqlcSlice bool // is this sqlc.slice() skipTableRequiredCheck bool diff --git a/internal/core/analysis.go b/internal/core/analysis.go index 822c12c411..7542d12b1e 100644 --- a/internal/core/analysis.go +++ b/internal/core/analysis.go @@ -53,8 +53,11 @@ type ColumnSource struct { } type Column struct { - Name string `json:"name"` - DataType string `json:"data_type"` + Name string `json:"name"` + DataType string `json:"data_type"` + // Type is the column's type as an expression, carrying what DataType + // and IsArray flatten away: arguments, nesting and inner nullability. + Type *TypeExpr `json:"type,omitempty"` TypeOID int64 `json:"type_oid,omitempty"` NotNull bool `json:"not_null"` IsArray bool `json:"is_array,omitempty"` @@ -73,6 +76,7 @@ type Parameter struct { Number int `json:"number"` Name string `json:"name,omitempty"` DataType string `json:"data_type,omitempty"` + Type *TypeExpr `json:"type,omitempty"` TypeOID int64 `json:"type_oid,omitempty"` NotNull bool `json:"not_null"` IsArray bool `json:"is_array,omitempty"` diff --git a/internal/core/analyzer/analyzer.go b/internal/core/analyzer/analyzer.go index 4608ff1bdc..8bc92bf5b3 100644 --- a/internal/core/analyzer/analyzer.go +++ b/internal/core/analyzer/analyzer.go @@ -119,6 +119,20 @@ func derivedRel(alias string, cols []core.Column) scopeRel { } func (a *analyzer) result() core.PrepareResult { + // A placeholder nothing constrained takes the dialect's type for one, when + // the dialect has such a type. + if oid, ok := a.cat.UntypedTypeOID(); ok { + for n, p := range a.params { + if p.TypeOID == 0 && p.DataType == "" { + t := exprType{typeOID: oid, nullable: true} + p.TypeOID = oid + p.DataType, p.IsArray = a.typeNameOf(t) + p.NotNull = false + p.Type = a.typeExprOf(t, "") + a.params[n] = p + } + } + } res := core.PrepareResult{ Command: a.command, Columns: a.columns, @@ -213,9 +227,39 @@ func (a *analyzer) analyzeSelect(s *ast.SelectStmt) error { return err } } + for _, item := range listItems(s.SortClause) { + if sb, ok := item.(*ast.SortBy); ok { + if _, err := a.typeExpr(sb.Node); err != nil { + return fmt.Errorf("order by: %w", err) + } + } + } + for _, n := range []ast.Node{s.LimitCount, s.LimitOffset} { + if err := a.typeLimit(n); err != nil { + return fmt.Errorf("limit: %w", err) + } + } return nil } +// typeLimit types a LIMIT or OFFSET count. A bare placeholder there holds +// whatever the dialect counts rows in. +func (a *analyzer) typeLimit(n ast.Node) error { + if n == nil { + return nil + } + if pr, ok := n.(*ast.ParamRef); ok { + oid, err := a.cat.LimitTypeOID() + if err != nil { + return err + } + a.inferParam(pr.Number, exprType{typeOID: oid}) + return nil + } + _, err := a.typeExpr(n) + return err +} + func (a *analyzer) typeValuesLists(l *ast.List) error { for _, row := range listItems(l) { if _, err := a.typeExpr(row); err != nil { diff --git a/internal/core/analyzer/expr.go b/internal/core/analyzer/expr.go index a36cf25cbf..d034432662 100644 --- a/internal/core/analyzer/expr.go +++ b/internal/core/analyzer/expr.go @@ -134,7 +134,7 @@ func (a *analyzer) typeConst(c *ast.A_Const) (exprType, error) { } func (a *analyzer) boolType(nullable bool) (exprType, error) { - oid, err := a.cat.ConstTypeOID(core.ConstBool) + oid, err := a.cat.BoolTypeOID() if err != nil { return exprType{}, err } @@ -207,10 +207,12 @@ func (a *analyzer) inferParam(number int, t exprType) { if !ok { cur = core.Parameter{Number: number} } - if cur.TypeOID == 0 && cur.DataType == "" && (t.typeOID != 0 || t.typeName != "") { + typed := cur.TypeOID == 0 && cur.DataType == "" && (t.typeOID != 0 || t.typeName != "") + if typed { cur.TypeOID = t.typeOID cur.DataType, cur.IsArray = a.typeNameOf(t) cur.NotNull = !t.nullable + cur.Type = a.typeExprOf(t, "") } if cur.Source == nil && t.sourceAttributeOID != 0 { ad, err := a.cat.LookupAttribute(t.sourceAttributeOID) @@ -221,11 +223,29 @@ func (a *analyzer) inferParam(number int, t exprType) { TableAlias: t.sourceTableAlias, Column: ad.Column, } + if typed { + cur.Type = a.typeExprOf(t, ad.DeclType) + } } } a.params[number] = cur } +// nameParamAfter names a placeholder compared with a function call after +// the function, the way a placeholder compared with a column is named after +// the column. +func (a *analyzer) nameParamAfter(number int, other ast.Node) { + fc, ok := other.(*ast.FuncCall) + if !ok { + return + } + cur := a.params[number] + if cur.Name == "" && cur.Source == nil { + cur.Name = funcCallName(fc) + a.params[number] = cur + } +} + func (a *analyzer) typeAExpr(e *ast.A_Expr) (exprType, error) { // Not every engine classifies its operators, so a zero kind is a plain // operator application rather than an unset field. LIKE and its relatives @@ -272,10 +292,12 @@ func (a *analyzer) typeAExpr(e *ast.A_Expr) (exprType, error) { if pr, ok := e.Lexpr.(*ast.ParamRef); ok && rightT.typeOID != 0 { a.inferParam(pr.Number, rightT) + a.nameParamAfter(pr.Number, e.Rexpr) leftT = rightT } if pr, ok := e.Rexpr.(*ast.ParamRef); ok && leftT.typeOID != 0 { a.inferParam(pr.Number, leftT) + a.nameParamAfter(pr.Number, e.Lexpr) rightT = leftT } @@ -335,6 +357,19 @@ func (a *analyzer) typeIn(e *ast.In) (exprType, error) { return exprType{}, err } } + // "x IN (SELECT ...)" compares x against the subquery's column, and the + // subquery's own placeholders are reported with the rest. + if sel, ok := e.Sel.(*ast.SelectStmt); ok { + cols, err := a.subqueryColumns(sel) + if err != nil { + return exprType{}, err + } + if len(cols) > 0 { + if err := a.typeOperands(e.Expr, exprType{typeOID: cols[0].TypeOID, nullable: !cols[0].NotNull}); err != nil { + return exprType{}, err + } + } + } return a.boolType(false) } @@ -385,10 +420,26 @@ func (a *analyzer) typeCase(e *ast.CaseExpr) (exprType, error) { return t, nil } -// typeCoalesce types COALESCE, which is its first argument's type and is null -// only when every argument is. +// typeCoalesce types COALESCE, which is its first typed argument's type and +// is null only when every argument is. func (a *analyzer) typeCoalesce(e *ast.CoalesceExpr) (exprType, error) { - return a.typeFirstOf(listItems(e.Args), false) + var out exprType + found := false + nullable := true + for _, n := range listItems(e.Args) { + t, err := a.typeExpr(n) + if err != nil { + return exprType{}, err + } + if !found && t.typeOID != 0 { + // The result is an expression's, not the column's it came from. + out = exprType{typeOID: t.typeOID, typeName: t.typeName} + found = true + } + nullable = nullable && t.nullable + } + out.nullable = nullable + return out, nil } // typeFirstOf types a set of alternative results, taking the first one that has @@ -591,7 +642,7 @@ func (a *analyzer) resolveOperator(name string, leftOID, rightOID int64) (core.O // operator's name implies: a comparison yields a boolean and anything // else yields the type it was applied to. if a.cat.IsComparisonOperator(name) { - boolOID, err := a.cat.ConstTypeOID(core.ConstBool) + boolOID, err := a.cat.BoolTypeOID() if err != nil { return core.OperatorOverload{}, err } @@ -631,13 +682,17 @@ func (a *analyzer) typeFuncCall(f *ast.FuncCall) (exprType, error) { } args := listItems(f.Args) - argTypes := make([]int64, 0, len(args)) + argTypes := make([]exprType, 0, len(args)) + argOIDs := make([]int64, 0, len(args)) + anyNullable := false for _, arg := range args { t, err := a.typeExpr(arg) if err != nil { return exprType{}, err } - argTypes = append(argTypes, t.typeOID) + argTypes = append(argTypes, t) + argOIDs = append(argOIDs, t.typeOID) + anyNullable = anyNullable || t.nullable } overloads, err := a.cat.FindProcs(name, nil) @@ -650,30 +705,84 @@ func (a *analyzer) typeFuncCall(f *ast.FuncCall) (exprType, error) { // rather than failing the query. return exprType{nullable: true}, nil } - p := pickOverload(overloads, argTypes) - // An argument that is a bare placeholder takes the parameter's type. + p := a.pickOverload(overloads, argOIDs) + // An argument that is a bare placeholder takes the parameter's type, + // unless the parameter is polymorphic and says nothing. for i, arg := range args { if i >= len(p.ArgTypes) { break } + if a.isPolymorphicOID(p.ArgTypes[i]) { + continue + } if err := a.typeOperands(arg, exprType{typeOID: p.ArgTypes[i]}); err != nil { return exprType{}, err } } - return exprType{typeOID: a.returnType(p, argTypes), nullable: p.ReturnNullable}, nil + ret := a.returnType(p, argTypes) + ret.nullable = p.ReturnNullable + if !p.NeverNull && anyNullable && a.cat.PropagatesNullable() { + ret.nullable = true + } + return ret, nil } -// returnType resolves a polymorphic return type — max(anyelement) and its -// like — to the type the call was made with. -func (a *analyzer) returnType(p core.ProcOverload, argTypes []int64) int64 { - if p.ReturnTypeOID == 0 || len(argTypes) == 0 || argTypes[0] == 0 { - return p.ReturnTypeOID +// returnType resolves a polymorphic return type — max(anyelement), or a +// seed's "$2" for the type of the second argument — to the type the call was +// made with. +func (a *analyzer) returnType(p core.ProcOverload, argTypes []exprType) exprType { + if p.ReturnTypeOID == 0 || len(argTypes) == 0 { + return exprType{typeOID: p.ReturnTypeOID} } name, err := a.cat.TypeName(p.ReturnTypeOID) - if err != nil || !isPolymorphic(name) { - return p.ReturnTypeOID + if err != nil { + return exprType{typeOID: p.ReturnTypeOID} + } + if n, ok := argIndex(name); ok { + if n < len(argTypes) { + return exprType{typeOID: argTypes[n].typeOID, typeName: argTypes[n].typeName} + } + return exprType{} + } + if isPolymorphic(name) && argTypes[0].typeOID != 0 { + return exprType{typeOID: argTypes[0].typeOID} + } + return exprType{typeOID: p.ReturnTypeOID} +} + +// argIndex reads a seed's "$n" pseudo-type as the zero-based index of the +// argument whose type it stands for. +func argIndex(typeName string) (int, bool) { + rest, ok := strings.CutPrefix(typeName, "$") + if !ok { + return 0, false } - return argTypes[0] + n := 0 + for _, r := range rest { + if r < '0' || r > '9' { + return 0, false + } + n = n*10 + int(r-'0') + } + if n == 0 { + return 0, false + } + return n - 1, true +} + +// isPolymorphicOID reports whether a parameter type accepts any argument. +func (a *analyzer) isPolymorphicOID(oid int64) bool { + if oid == 0 { + return true + } + name, err := a.cat.TypeName(oid) + if err != nil { + return false + } + if _, ok := argIndex(name); ok { + return true + } + return isPolymorphic(name) } func isPolymorphic(typeName string) bool { @@ -687,30 +796,32 @@ func isPolymorphic(typeName string) bool { } // pickOverload chooses the overload whose parameters the call's arguments -// match, preferring an exact match on types over one on arity alone. -func pickOverload(overloads []core.ProcOverload, argTypes []int64) core.ProcOverload { - var byArity *core.ProcOverload +// match best: an exact type match on a parameter beats a polymorphic one, +// which beats a mismatch, and any overload of the right arity beats one of +// the wrong arity. +func (a *analyzer) pickOverload(overloads []core.ProcOverload, argTypes []int64) core.ProcOverload { + best := -1 + bestScore := -1 for i := range overloads { ov := &overloads[i] if len(ov.ArgTypes) != len(argTypes) { continue } - if byArity == nil { - byArity = ov - } - exact := true + score := 0 for j, oid := range argTypes { - if oid != ov.ArgTypes[j] { - exact = false - break + switch { + case oid != 0 && oid == ov.ArgTypes[j]: + score += 2 + case a.isPolymorphicOID(ov.ArgTypes[j]): + score += 1 } } - if exact { - return *ov + if score > bestScore { + best, bestScore = i, score } } - if byArity != nil { - return *byArity + if best >= 0 { + return overloads[best] } return overloads[0] } diff --git a/internal/core/analyzer/projection.go b/internal/core/analyzer/projection.go index 64c37f7251..f3293dcbc6 100644 --- a/internal/core/analyzer/projection.go +++ b/internal/core/analyzer/projection.go @@ -23,6 +23,21 @@ func (a *analyzer) projectTarget(rt *ast.ResTarget) error { if err != nil { return err } + // A placeholder selected directly is named by its alias and, when nothing + // constrains it, typed as the dialect types such a placeholder. + if pr, ok := rt.Val.(*ast.ParamRef); ok { + if rt.Name != nil && *rt.Name != "" { + if p := a.params[pr.Number]; p.Name == "" { + p.Name = *rt.Name + a.params[pr.Number] = p + } + } + if t.typeOID == 0 && t.typeName == "" { + if oid, ok := a.cat.UntypedTypeOID(); ok { + t = exprType{typeOID: oid, nullable: true} + } + } + } col := core.Column{ Name: targetName(rt, fields), TypeOID: t.typeOID, @@ -32,10 +47,65 @@ func (a *analyzer) projectTarget(rt *ast.ResTarget) error { } col.DataType, col.IsArray = a.typeNameOf(t) a.decorateSource(&col, t.sourceAttributeOID, t.sourceTableAlias) + col.Type = a.typeExprOf(t, col.DeclType) + if rt.Name == nil || *rt.Name == "" { + a.qualifyDuplicate(&col, t.sourceTableAlias) + } a.columns = append(a.columns, col) return nil } +// qualifyDuplicate names a column after its relation when an earlier result +// column from another relation already has its name, in a dialect that +// does so. +func (a *analyzer) qualifyDuplicate(col *core.Column, alias string) { + if alias == "" || !a.cat.QualifiesDuplicateColumns() { + return + } + for _, prev := range a.columns { + if prev.Name != col.Name { + continue + } + prevAlias := "" + if prev.Source != nil { + prevAlias = prev.Source.TableAlias + } + if prevAlias != alias { + col.Name = alias + "." + col.Name + return + } + } +} + +// typeExprOf writes a type as an expression. A source column's declared +// spelling carries what the catalog's flat name cannot, so it is parsed +// when there is one; otherwise the expression is the type's name, wrapped +// in an array when the type is one. Nullability comes from the spelling +// when the spelling says anything about it, and from the analysis +// otherwise. +func (a *analyzer) typeExprOf(t exprType, declType string) *core.TypeExpr { + name, isArray := a.typeNameOf(t) + if name == "" && declType == "" { + return nil + } + var expr *core.TypeExpr + if declType != "" { + expr = core.ParseTypeExpr(declType) + if isArray && expr.Name != "array" { + expr = &core.TypeExpr{Name: "array", Args: []core.TypeArg{{Type: expr}}} + } + } else { + expr = core.ParseTypeExpr(name) + if isArray { + expr = &core.TypeExpr{Name: "array", Args: []core.TypeArg{{Type: expr}}} + } + } + if !expr.HasNullable() { + expr.Nullable = t.nullable + } + return expr +} + func (a *analyzer) decorateSource(col *core.Column, attOID int64, tableAlias string) { if attOID == 0 { return @@ -109,6 +179,8 @@ func (a *analyzer) emitStar(rt *ast.ResTarget, fields []string) { } col.DataType, col.IsArray = a.typeNameOf(exprType{typeOID: c.TypeOID}) a.decorateSource(&col, c.AttOID, rel.alias) + col.Type = a.typeExprOf(exprType{typeOID: c.TypeOID, nullable: !c.NotNull}, col.DeclType) + a.qualifyDuplicate(&col, rel.alias) a.columns = append(a.columns, col) star.Columns = append(star.Columns, core.StarColumn{ Relation: rel.alias, diff --git a/internal/core/dialect.go b/internal/core/dialect.go index 5b3286e901..bad80dedcb 100644 --- a/internal/core/dialect.go +++ b/internal/core/dialect.go @@ -54,6 +54,29 @@ func (c *Catalog) DialectFlag(dialectOID int64, key string) (string, error) { // same ones the seeded types have. const FlagComparisonOperators = "operators.comparison" +// FlagBoolType holds the type a comparison or predicate yields, when it is +// not the type a boolean literal has: ClickHouse compares to UInt8 while +// writing true as Bool. +const FlagBoolType = "types.bool" + +// FlagLimitType holds the type a LIMIT or OFFSET count has, which is what a +// placeholder in one is typed as. +const FlagLimitType = "types.limit" + +// FlagUntypedType holds the type a placeholder takes when nothing in the +// query constrains it, for a dialect that gives such a placeholder one. +const FlagUntypedType = "types.untyped" + +// FlagPropagateNullable is set when a function's result is nullable whenever +// one of its arguments is, the way ClickHouse's ordinary functions behave. +const FlagPropagateNullable = "functions.propagate_nullable" + +// FlagQualifyDuplicateColumns is set for a dialect that names a result +// column after its relation when an earlier result column from another +// relation has the same name, as ClickHouse names the second id of a join +// e.id. +const FlagQualifyDuplicateColumns = "columns.qualify_duplicates" + // FlagCastCategories holds the categories whose types are all implicitly // castable to one another, as the dialect's seed declared them, so that a type // arriving after the seed — an extension's, say — can join its category. @@ -109,3 +132,61 @@ func (c *Catalog) ConstTypeOID(kind string) (int64, error) { } return c.TypeOID(name) } + +// BoolTypeOID returns the type a comparison or predicate yields. +func (c *Catalog) BoolTypeOID() (int64, error) { + if c.dialectOID != 0 { + if name, _ := c.DialectFlag(c.dialectOID, FlagBoolType); name != "" { + return c.TypeOID(name) + } + } + return c.ConstTypeOID(ConstBool) +} + +// LimitTypeOID returns the type a LIMIT or OFFSET count has, falling back to +// the type of an integer literal. +func (c *Catalog) LimitTypeOID() (int64, error) { + if c.dialectOID != 0 { + if name, _ := c.DialectFlag(c.dialectOID, FlagLimitType); name != "" { + return c.TypeOID(name) + } + } + return c.ConstTypeOID(ConstInteger) +} + +// UntypedTypeOID returns the type an unconstrained placeholder takes, and +// whether the dialect gives it one at all. +func (c *Catalog) UntypedTypeOID() (int64, bool) { + if c.dialectOID == 0 { + return 0, false + } + name, _ := c.DialectFlag(c.dialectOID, FlagUntypedType) + if name == "" { + return 0, false + } + oid, err := c.TypeOID(name) + if err != nil { + return 0, false + } + return oid, true +} + +// PropagatesNullable reports whether a function's result is nullable +// whenever one of its arguments is. +func (c *Catalog) PropagatesNullable() bool { + if c.dialectOID == 0 { + return false + } + v, _ := c.DialectFlag(c.dialectOID, FlagPropagateNullable) + return v == "true" +} + +// QualifiesDuplicateColumns reports whether a result column that repeats an +// earlier one's name from another relation is named after its relation. +func (c *Catalog) QualifiesDuplicateColumns() bool { + if c.dialectOID == 0 { + return false + } + v, _ := c.DialectFlag(c.dialectOID, FlagQualifyDuplicateColumns) + return v == "true" +} diff --git a/internal/core/proc.go b/internal/core/proc.go index baee74e132..2409a4c4af 100644 --- a/internal/core/proc.go +++ b/internal/core/proc.go @@ -17,11 +17,22 @@ type ProcSpec struct { ReturnTypeOID int64 ReturnSet bool ReturnNullable bool - Strict bool - VariadicKind string - Args []ProcArg + // NeverNull marks a function whose result is never NULL even when an + // argument is, in a dialect that otherwise propagates nullability. + NeverNull bool + Strict bool + VariadicKind string + Args []ProcArg } +// The proc table stores nullability as one integer: 0 leaves it to the +// dialect's rule, 1 is always nullable and 2 is never nullable. +const ( + nullableDefault int64 = 0 + nullableAlways int64 = 1 + nullableNever int64 = 2 +) + type ProcArg struct { Name string TypeOID int64 @@ -44,7 +55,7 @@ func (c *Catalog) CreateProc(p ProcSpec) (int64, error) { Kind: p.Kind, ReturnTypeOid: p.ReturnTypeOID, ReturnSet: boolToInt64(p.ReturnSet), - ReturnNullable: boolToInt64(p.ReturnNullable), + ReturnNullable: returnNullable(p), Strict: boolToInt64(p.Strict), VariadicKind: p.VariadicKind, }) @@ -71,12 +82,23 @@ func (c *Catalog) CreateProc(p ProcSpec) (int64, error) { return procOID, nil } +func returnNullable(p ProcSpec) int64 { + switch { + case p.NeverNull: + return nullableNever + case p.ReturnNullable: + return nullableAlways + } + return nullableDefault +} + type ProcOverload struct { OID int64 Name string Kind string ReturnTypeOID int64 ReturnNullable bool + NeverNull bool ArgTypes []int64 } @@ -99,7 +121,8 @@ func (c *Catalog) FindProcs(name string, namespaceOIDs []int64) ([]ProcOverload, Name: r.Name, Kind: r.Kind, ReturnTypeOID: r.ReturnTypeOid, - ReturnNullable: r.ReturnNullable != 0, + ReturnNullable: r.ReturnNullable == nullableAlways, + NeverNull: r.ReturnNullable == nullableNever, }) } } else { @@ -121,7 +144,8 @@ func (c *Catalog) FindProcs(name string, namespaceOIDs []int64) ([]ProcOverload, Name: r.Name, Kind: r.Kind, ReturnTypeOID: r.ReturnTypeOid, - ReturnNullable: r.ReturnNullable != 0, + ReturnNullable: r.ReturnNullable == nullableAlways, + NeverNull: r.ReturnNullable == nullableNever, }) } } diff --git a/internal/core/schema/schema.go b/internal/core/schema/schema.go index 19dc3cc827..470d6716b5 100644 --- a/internal/core/schema/schema.go +++ b/internal/core/schema/schema.go @@ -162,7 +162,7 @@ func applyCreateTable(cat *core.Catalog, stmt *ast.CreateTableStmt) error { Num: i + 1, NotNull: col.IsNotNull || col.PrimaryKey, IsPrimaryKey: col.PrimaryKey, - DeclType: col.TypeName.Name, + DeclType: declType(col.TypeName), Hidden: col.IsHidden, }); err != nil { return fmt.Errorf("attr %s.%s: %w", stmt.Name.Name, col.Colname, err) @@ -234,7 +234,7 @@ func applyAlterTable(cat *core.Catalog, stmt *ast.AlterTableStmt) error { Num: num, NotNull: cmd.Def.IsNotNull || cmd.Def.PrimaryKey, IsPrimaryKey: cmd.Def.PrimaryKey, - DeclType: cmd.Def.TypeName.Name, + DeclType: declType(cmd.Def.TypeName), }); err != nil { return err } @@ -427,3 +427,15 @@ func columnTypeOID(cat *core.Catalog, col *ast.ColumnDef) (int64, error) { } return cat.ResolveTypeName(name) } + +// declType is the type as the schema spelled it: an engine that folds or +// reduces the name for the catalog keeps the full spelling alongside. +func declType(tn *ast.TypeName) string { + if tn == nil { + return "" + } + if tn.Spelling != "" { + return tn.Spelling + } + return tn.Name +} diff --git a/internal/core/seed/seed.go b/internal/core/seed/seed.go index cc0c5a7977..4dc7525092 100644 --- a/internal/core/seed/seed.go +++ b/internal/core/seed/seed.go @@ -63,6 +63,24 @@ type Settings struct { // Bool names the type comparisons return. Bool string `json:"bool,omitempty"` + // Limit names the type a LIMIT or OFFSET count has, which is what a + // placeholder in one is typed as. Unset, it is the integer literal's. + Limit string `json:"limit,omitempty"` + + // Untyped names the type a placeholder takes when nothing in the query + // constrains it. Unset, such a placeholder stays untyped. + Untyped string `json:"untyped,omitempty"` + + // PropagateNullable makes a function's result nullable whenever one of + // its arguments is, the way ClickHouse's ordinary functions behave, + // unless the function is seeded as never null. + PropagateNullable bool `json:"propagate_nullable,omitempty"` + + // QualifyDuplicateColumns names a result column after its relation when + // an earlier result column from another relation has the same name, as + // ClickHouse names the second id of a join e.id. + QualifyDuplicateColumns bool `json:"qualify_duplicate_columns,omitempty"` + // Comparison operators are registered as (T, T) -> Bool for every type in // ComparisonCategories. Comparison []string `json:"comparison,omitempty"` @@ -108,11 +126,16 @@ type Cast struct { // Function is a function the dialect ships with. Kind is 'f'unction, // 'a'ggregate, 'w'indow or 'p'rocedure. type Function struct { - Name string `json:"name"` - Kind string `json:"kind,omitempty"` - Args []Arg `json:"args,omitempty"` + Name string `json:"name,omitempty"` + Kind string `json:"kind,omitempty"` + Args []Arg `json:"args,omitempty"` + // Returns names the result type, or "$1", "$2"... for the type of that + // argument. Returns string `json:"returns"` Nullable bool `json:"nullable,omitempty"` + // NeverNull marks a result that is never NULL even when an argument + // is, in a dialect that propagates nullability. + NeverNull bool `json:"never_null,omitempty"` } // Relation is a table or view the dialect ships with, such as one of @@ -450,6 +473,31 @@ func (b *builder) consts() error { return err } } + for key, name := range map[string]string{ + core.FlagBoolType: b.settings.Bool, + core.FlagLimitType: b.settings.Limit, + core.FlagUntypedType: b.settings.Untyped, + } { + if name == "" { + continue + } + if _, ok := b.oids[strings.ToLower(name)]; !ok { + return fmt.Errorf("seed %s: %s names unknown type %q", b.settings.Dialect, key, name) + } + if err := b.cat.SetDialectFlag(b.dialectOID, key, name); err != nil { + return err + } + } + if b.settings.PropagateNullable { + if err := b.cat.SetDialectFlag(b.dialectOID, core.FlagPropagateNullable, "true"); err != nil { + return err + } + } + if b.settings.QualifyDuplicateColumns { + if err := b.cat.SetDialectFlag(b.dialectOID, core.FlagQualifyDuplicateColumns, "true"); err != nil { + return err + } + } // A schema declares types the seed knows nothing about — enums, domains, // arrays, a SQLite column typed whatever the author felt like. Recording // the comparison operators lets the catalog give those types the same ones. @@ -602,6 +650,7 @@ func (b *builder) addFunction(fn Function) error { Kind: fn.Kind, ReturnTypeOID: returnOID, ReturnNullable: fn.Nullable, + NeverNull: fn.NeverNull, Args: args, }) if err != nil { diff --git a/internal/core/typeexpr.go b/internal/core/typeexpr.go new file mode 100644 index 0000000000..bebb7e83ce --- /dev/null +++ b/internal/core/typeexpr.go @@ -0,0 +1,198 @@ +package core + +import ( + "strconv" + "strings" +) + +// TypeExpr is a type written as a call expression: a lowercased name applied +// to arguments that are other types, integers, booleans or strings, each +// with an optional label, and a nullable flag at whatever depth it applies. +// Nothing about a nested type is special-cased, so an array of nullable +// strings is array(string nullable), a map is map(string, uint32) and a +// named tuple is tuple(lat: float64, lon: float64). The catalog resolves the +// names; the expression only records what was declared or inferred. +type TypeExpr struct { + Name string `json:"name"` + Nullable bool `json:"nullable,omitempty"` + Args []TypeArg `json:"args,omitempty"` +} + +// TypeArg is one argument of a TypeExpr: exactly one of Type, Int, Bool or +// String is set. +type TypeArg struct { + Label string `json:"label,omitempty"` + Type *TypeExpr `json:"type,omitempty"` + Int *int64 `json:"int,omitempty"` + Bool *bool `json:"bool,omitempty"` + String *string `json:"string,omitempty"` +} + +// ParseTypeExpr reads a type spelled the way every dialect spells one, as a +// name optionally applied to a parenthesised, comma-separated argument list +// whose entries may be labelled (`lat Float64` in a tuple, `'a' = 1` in an +// enum). A Nullable(T) wrapper becomes T with Nullable set, and a trailing +// [] becomes an array of the element. +func ParseTypeExpr(s string) *TypeExpr { + s = strings.TrimSpace(s) + if element, ok := strings.CutSuffix(s, ArraySuffix); ok { + return &TypeExpr{Name: "array", Args: []TypeArg{{Type: ParseTypeExpr(element)}}} + } + name, args := splitTypeArgs(s) + name = strings.ToLower(name) + if name == "nullable" && len(args) == 1 { + t := ParseTypeExpr(args[0]) + t.Nullable = true + return t + } + t := &TypeExpr{Name: name} + for _, a := range args { + t.Args = append(t.Args, parseTypeArg(a)) + } + return t +} + +func parseTypeArg(a string) TypeArg { + a = strings.TrimSpace(a) + if strings.HasPrefix(a, "'") { + end := quotedEnd(a) + lit := strings.ReplaceAll(strings.ReplaceAll(a[1:end-1], `\'`, `'`), `''`, `'`) + if rest := strings.TrimSpace(a[end:]); strings.HasPrefix(rest, "=") { + arg := parseTypeArg(rest[1:]) + arg.Label = lit + return arg + } + return TypeArg{String: &lit} + } + if n, err := strconv.ParseInt(a, 10, 64); err == nil { + return TypeArg{Int: &n} + } + switch strings.ToLower(a) { + case "true", "false": + b := strings.EqualFold(a, "true") + return TypeArg{Bool: &b} + } + // A label is a word before a space that comes before any parenthesis, + // as in `lat Float64` or `tags Array(String)`. + head := a + if p := strings.IndexByte(a, '('); p >= 0 { + head = a[:p] + } + if i := strings.IndexByte(head, ' '); i > 0 { + arg := parseTypeArg(a[i+1:]) + arg.Label = a[:i] + return arg + } + return TypeArg{Type: ParseTypeExpr(a)} +} + +// quotedEnd returns the index just past the single-quoted literal starting +// at the beginning of s, honouring backslash escapes and doubled quotes. +func quotedEnd(s string) int { + for i := 1; i < len(s); i++ { + switch { + case s[i] == '\\' && i+1 < len(s): + i++ + case s[i] == '\'' && i+1 < len(s) && s[i+1] == '\'': + i++ + case s[i] == '\'': + return i + 1 + } + } + return len(s) +} + +// splitTypeArgs splits `Base(arg, arg)` into its base name and top-level +// arguments, leaving nested parentheses and quoted strings intact. +func splitTypeArgs(t string) (string, []string) { + open := strings.IndexByte(t, '(') + if open < 0 || !strings.HasSuffix(t, ")") { + return t, nil + } + base := strings.TrimSpace(t[:open]) + inner := t[open+1 : len(t)-1] + var ( + args []string + depth int + quote byte + start int + ) + for i := 0; i < len(inner); i++ { + c := inner[i] + switch { + case quote != 0: + if c == '\\' { + i++ + } else if c == quote { + quote = 0 + } + case c == '\'' || c == '"' || c == '`': + quote = c + case c == '(': + depth++ + case c == ')': + depth-- + case c == ',' && depth == 0: + args = append(args, strings.TrimSpace(inner[start:i])) + start = i + 1 + } + } + if last := strings.TrimSpace(inner[start:]); last != "" || len(args) > 0 { + args = append(args, last) + } + return base, args +} + +// HasNullable reports whether the expression marks nullability anywhere, +// which tells whether the spelling it came from said so itself. +func (t *TypeExpr) HasNullable() bool { + if t == nil { + return false + } + if t.Nullable { + return true + } + for _, a := range t.Args { + if a.Type.HasNullable() { + return true + } + } + return false +} + +// String renders the expression in its canonical text form, with a trailing +// "nullable" marking a nullable type. +func (t *TypeExpr) String() string { + if t == nil { + return "" + } + var b strings.Builder + b.WriteString(t.Name) + if len(t.Args) > 0 { + b.WriteByte('(') + for i, a := range t.Args { + if i > 0 { + b.WriteString(", ") + } + if a.Label != "" { + b.WriteString(a.Label) + b.WriteString(": ") + } + switch { + case a.Type != nil: + b.WriteString(a.Type.String()) + case a.Int != nil: + b.WriteString(strconv.FormatInt(*a.Int, 10)) + case a.Bool != nil: + b.WriteString(strconv.FormatBool(*a.Bool)) + case a.String != nil: + b.WriteString("'" + strings.ReplaceAll(*a.String, "'", `\'`) + "'") + } + } + b.WriteByte(')') + } + if t.Nullable { + b.WriteString(" nullable") + } + return b.String() +} diff --git a/internal/endtoend/testdata/analyze_expressions/clickhouse/output.json b/internal/endtoend/testdata/analyze_expressions/clickhouse/output.json index 0553ffac14..e3e4716696 100644 --- a/internal/endtoend/testdata/analyze_expressions/clickhouse/output.json +++ b/internal/endtoend/testdata/analyze_expressions/clickhouse/output.json @@ -10,16 +10,29 @@ } }, { - "name": "top" + "name": "top", + "type": { + "name": "float64" + } }, { - "name": "sum" + "name": "sum(amount)", + "type": { + "name": "float64" + } }, { - "name": "first_tag" + "name": "first_tag", + "type": { + "name": "string", + "nullable": true + } }, { - "name": "uniq" + "name": "uniq(name)", + "type": { + "name": "uint64" + } } ], "params": [] @@ -35,25 +48,42 @@ } }, { - "name": "maybe" + "name": "maybe", + "type": { + "name": "string", + "nullable": true + } }, { - "name": "tag_or_none" + "name": "tag_or_none", + "type": { + "name": "string" + } }, { - "name": "lower" + "name": "lower(name)", + "type": { + "name": "string" + } }, { - "name": "today" + "name": "today", + "type": { + "name": "date" + } }, { "name": "big", "type": { - "name": "bool" + "name": "uint8" } }, { - "name": "word" + "name": "word", + "type": { + "name": "string", + "nullable": true + } } ], "params": [] @@ -158,7 +188,49 @@ "table": "events" } ], - "params": [] + "params": [ + { + "number": 1, + "column": { + "name": "name", + "type": { + "name": "string" + }, + "table": "events" + } + }, + { + "number": 2, + "column": { + "name": "tag", + "type": { + "name": "string", + "nullable": true + }, + "table": "events" + } + }, + { + "number": 3, + "column": { + "name": "amount", + "type": { + "name": "float64" + }, + "table": "events" + } + }, + { + "number": 4, + "column": { + "name": "amount", + "type": { + "name": "float64" + }, + "table": "events" + } + } + ] }, { "name": "Functions", @@ -176,7 +248,10 @@ { "number": 1, "column": { - "name": "" + "name": "lower", + "type": { + "name": "string" + } } }, { @@ -192,7 +267,10 @@ { "number": 3, "column": { - "name": "" + "name": "toDate", + "type": { + "name": "date" + } } } ] @@ -209,14 +287,37 @@ "table": "events" } ], - "params": [] + "params": [ + { + "number": 1, + "column": { + "name": "", + "type": { + "name": "uint64" + } + } + }, + { + "number": 2, + "column": { + "name": "", + "type": { + "name": "uint64" + } + } + } + ] }, { "name": "Projected", "cmd": ":one", "columns": [ { - "name": "echo" + "name": "echo", + "type": { + "name": "nothing", + "nullable": true + } }, { "name": "lit", @@ -229,7 +330,11 @@ { "number": 1, "column": { - "name": "" + "name": "echo", + "type": { + "name": "nothing", + "nullable": true + } } } ] diff --git a/internal/endtoend/testdata/analyze_subqueries/clickhouse/output.json b/internal/endtoend/testdata/analyze_subqueries/clickhouse/output.json index db310e24a1..bc4f0ec8e9 100644 --- a/internal/endtoend/testdata/analyze_subqueries/clickhouse/output.json +++ b/internal/endtoend/testdata/analyze_subqueries/clickhouse/output.json @@ -11,10 +11,16 @@ "table": "events" }, { - "name": "total" + "name": "total", + "type": { + "name": "float64" + } }, { - "name": "email" + "name": "email", + "type": { + "name": "string" + } } ], "params": [] @@ -52,11 +58,101 @@ "table": "events" }, { - "name": "?column?", + "name": "user_count", "type": { - "name": "bool" + "name": "uint64", + "nullable": true + } + } + ], + "params": [ + { + "number": 1, + "column": { + "name": "email", + "type": { + "name": "string" + }, + "table": "users" } } + ] + }, + { + "name": "Cte", + "cmd": ":many", + "columns": [ + { + "name": "id", + "type": { + "name": "uint64" + }, + "table": "events" + }, + { + "name": "tag", + "type": { + "name": "string", + "nullable": true + }, + "table": "events" + }, + { + "name": "cnt", + "type": { + "name": "uint64" + } + } + ], + "params": [] + }, + { + "name": "Star", + "cmd": ":many", + "columns": [ + { + "name": "id", + "type": { + "name": "uint64" + }, + "table": "users" + }, + { + "name": "email", + "type": { + "name": "string" + }, + "table": "users" + }, + { + "name": "e.id", + "type": { + "name": "uint64" + }, + "table": "events" + }, + { + "name": "name", + "type": { + "name": "string" + }, + "table": "events" + }, + { + "name": "tag", + "type": { + "name": "string", + "nullable": true + }, + "table": "events" + }, + { + "name": "amount", + "type": { + "name": "float64" + }, + "table": "events" + } ], "params": [] } diff --git a/internal/endtoend/testdata/analyze_subqueries/clickhouse/query.sql b/internal/endtoend/testdata/analyze_subqueries/clickhouse/query.sql index 8633227102..601ba8e146 100644 --- a/internal/endtoend/testdata/analyze_subqueries/clickhouse/query.sql +++ b/internal/endtoend/testdata/analyze_subqueries/clickhouse/query.sql @@ -16,3 +16,12 @@ SELECT id, (SELECT count() FROM users) AS user_count FROM events WHERE id IN (SELECT id FROM users WHERE email = ?); + +-- name: Cte :many +WITH t AS (SELECT id, tag FROM events) +SELECT t.id, t.tag, s.cnt +FROM t +JOIN (SELECT id, count() AS cnt FROM events GROUP BY id) s ON s.id = t.id; + +-- name: Star :many +SELECT * FROM users u JOIN events e ON e.id = u.id; diff --git a/internal/endtoend/testdata/analyze_types/clickhouse/output.json b/internal/endtoend/testdata/analyze_types/clickhouse/output.json index d9a4d7934c..5ef32d6b93 100644 --- a/internal/endtoend/testdata/analyze_types/clickhouse/output.json +++ b/internal/endtoend/testdata/analyze_types/clickhouse/output.json @@ -50,11 +50,11 @@ "name": "labels", "type": { "name": "array", - "nullable": true, "args": [ { "type": { - "name": "string" + "name": "string", + "nullable": true } } ] @@ -68,7 +68,14 @@ "args": [ { "type": { - "name": "uint8" + "name": "array", + "args": [ + { + "type": { + "name": "uint8" + } + } + ] } } ] @@ -78,8 +85,15 @@ { "name": "kind", "type": { - "name": "string", - "nullable": true + "name": "lowcardinality", + "args": [ + { + "type": { + "name": "string", + "nullable": true + } + } + ] }, "table": "things" }, @@ -93,49 +107,126 @@ { "name": "updated", "type": { - "name": "datetime64" + "name": "datetime64", + "args": [ + { + "int": 3 + }, + { + "string": "UTC" + } + ] }, "table": "things" }, { "name": "price", "type": { - "name": "decimal" + "name": "decimal", + "args": [ + { + "int": 10 + }, + { + "int": 2 + } + ] }, "table": "things" }, { "name": "status", "type": { - "name": "enum8" + "name": "enum8", + "args": [ + { + "label": "active", + "int": 1 + }, + { + "label": "deleted", + "int": 2 + } + ] }, "table": "things" }, { "name": "attrs", "type": { - "name": "map" + "name": "map", + "args": [ + { + "type": { + "name": "string" + } + }, + { + "type": { + "name": "uint32" + } + } + ] }, "table": "things" }, { "name": "pos", "type": { - "name": "tuple" + "name": "tuple", + "args": [ + { + "type": { + "name": "float64" + } + }, + { + "type": { + "name": "float64" + } + } + ] }, "table": "things" }, { "name": "geo", "type": { - "name": "tuple" + "name": "tuple", + "args": [ + { + "label": "lat", + "type": { + "name": "float64" + } + }, + { + "label": "lon", + "type": { + "name": "float64" + } + } + ] }, "table": "things" }, { "name": "scores", "type": { - "name": "map" + "name": "map", + "args": [ + { + "type": { + "name": "string" + } + }, + { + "type": { + "name": "uint8", + "nullable": true + } + } + ] }, "table": "things" }, @@ -156,7 +247,12 @@ { "name": "fixed", "type": { - "name": "fixedstring" + "name": "fixedstring", + "args": [ + { + "int": 4 + } + ] }, "table": "things" }, @@ -221,11 +317,11 @@ "name": "labels", "type": { "name": "array", - "nullable": true, "args": [ { "type": { - "name": "string" + "name": "string", + "nullable": true } } ] @@ -239,7 +335,14 @@ "args": [ { "type": { - "name": "uint8" + "name": "array", + "args": [ + { + "type": { + "name": "uint8" + } + } + ] } } ] @@ -249,8 +352,15 @@ { "name": "kind", "type": { - "name": "string", - "nullable": true + "name": "lowcardinality", + "args": [ + { + "type": { + "name": "string", + "nullable": true + } + } + ] }, "table": "things" }, @@ -264,49 +374,126 @@ { "name": "updated", "type": { - "name": "datetime64" + "name": "datetime64", + "args": [ + { + "int": 3 + }, + { + "string": "UTC" + } + ] }, "table": "things" }, { "name": "price", "type": { - "name": "decimal" + "name": "decimal", + "args": [ + { + "int": 10 + }, + { + "int": 2 + } + ] }, "table": "things" }, { "name": "status", "type": { - "name": "enum8" + "name": "enum8", + "args": [ + { + "label": "active", + "int": 1 + }, + { + "label": "deleted", + "int": 2 + } + ] }, "table": "things" }, { "name": "attrs", "type": { - "name": "map" + "name": "map", + "args": [ + { + "type": { + "name": "string" + } + }, + { + "type": { + "name": "uint32" + } + } + ] }, "table": "things" }, { "name": "pos", "type": { - "name": "tuple" + "name": "tuple", + "args": [ + { + "type": { + "name": "float64" + } + }, + { + "type": { + "name": "float64" + } + } + ] }, "table": "things" }, { "name": "geo", "type": { - "name": "tuple" + "name": "tuple", + "args": [ + { + "label": "lat", + "type": { + "name": "float64" + } + }, + { + "label": "lon", + "type": { + "name": "float64" + } + } + ] }, "table": "things" }, { "name": "scores", "type": { - "name": "map" + "name": "map", + "args": [ + { + "type": { + "name": "string" + } + }, + { + "type": { + "name": "uint8", + "nullable": true + } + } + ] }, "table": "things" }, @@ -327,7 +514,12 @@ { "name": "fixed", "type": { - "name": "fixedstring" + "name": "fixedstring", + "args": [ + { + "int": 4 + } + ] }, "table": "things" }, diff --git a/internal/endtoend/testdata/parse_basic/clickhouse/stdout.txt b/internal/endtoend/testdata/parse_basic/clickhouse/stdout.txt index 82c35f9819..0344534966 100644 --- a/internal/endtoend/testdata/parse_basic/clickhouse/stdout.txt +++ b/internal/endtoend/testdata/parse_basic/clickhouse/stdout.txt @@ -11,15 +11,16 @@ "items": [ { "tag": "ResTarget", + "name": "1", "val": { "tag": "A_Const", "val": { "tag": "Integer", "ival": 1 }, - "location": 31 + "location": 30 }, - "location": 31 + "location": 30 } ] }, diff --git a/internal/engine/clickhouse/convert.go b/internal/engine/clickhouse/convert.go index ea4f342e2c..de2a1419bf 100644 --- a/internal/engine/clickhouse/convert.go +++ b/internal/engine/clickhouse/convert.go @@ -6,6 +6,7 @@ import ( "strings" chast "github.com/sqlc-dev/doubleclick/ast" + "github.com/sqlc-dev/doubleclick/token" "github.com/sqlc-dev/sqlc/internal/sql/ast" ) @@ -14,6 +15,18 @@ type cc struct { paramCount int } +// pos is a node's byte offset in the query. doubleclick counts positions +// from one; the rest of sqlc counts from zero. +func pos(n interface{ Pos() token.Position }) int { + if n == nil { + return 0 + } + if off := n.Pos().Offset; off > 0 { + return off - 1 + } + return 0 +} + func (c *cc) convert(node chast.Node) ast.Node { switch n := node.(type) { case *chast.SelectWithUnionQuery: @@ -153,19 +166,24 @@ func (c *cc) convertSelectQuery(n *chast.SelectQuery) *ast.SelectStmt { Ctes: &ast.List{}, } for _, cte := range n.With { - if aliased, ok := cte.(*chast.AliasedExpr); ok { - cteNode := &ast.CommonTableExpr{ - Ctename: &aliased.Alias, - } - // CTE expression may be a Subquery containing the actual SELECT - if subq, ok := aliased.Expr.(*chast.Subquery); ok { - cteNode.Ctequery = c.convert(subq.Query) - } else { - // Fallback: treat the expression itself as the query - cteNode.Ctequery = c.convertExpr(aliased.Expr) - } - stmt.WithClause.Ctes.Items = append(stmt.WithClause.Ctes.Items, cteNode) + var name string + var query chast.Expression + switch w := cte.(type) { + case *chast.WithElement: + // "name AS (SELECT ...)" or a scalar "(expr) AS name". + name, query = w.Name, w.Query + case *chast.AliasedExpr: + name, query = w.Alias, w.Expr + default: + continue + } + cteNode := &ast.CommonTableExpr{Ctename: &name} + if subq, ok := query.(*chast.Subquery); ok { + cteNode.Ctequery = c.convert(subq.Query) + } else { + cteNode.Ctequery = c.convertExpr(query) } + stmt.WithClause.Ctes.Items = append(stmt.WithClause.Ctes.Items, cteNode) } } @@ -174,7 +192,7 @@ func (c *cc) convertSelectQuery(n *chast.SelectQuery) *ast.SelectStmt { func (c *cc) convertToResTarget(expr chast.Expression) *ast.ResTarget { res := &ast.ResTarget{ - Location: expr.Pos().Offset, + Location: pos(expr), } switch e := expr.(type) { @@ -197,24 +215,160 @@ func (c *cc) convertToResTarget(expr chast.Expression) *ast.ResTarget { }, } } + return res case *chast.AliasedExpr: res.Name = &e.Alias res.Val = c.convertExpr(e.Expr) + return res case *chast.Identifier: if e.Alias != "" { res.Name = &e.Alias } res.Val = c.convertIdentifier(e) + return res + } + + res.Val = c.convertExpr(expr) + if alias := exprAlias(expr); alias != "" { + res.Name = &alias + } else if name := columnName(expr); name != "" { + // ClickHouse names an unaliased expression column after the + // expression itself, written in its canonical function form. + res.Name = &name + } + return res +} + +// exprAlias returns the alias the parser attached to an expression, for +// the node kinds that carry one directly. +func exprAlias(expr chast.Expression) string { + switch e := expr.(type) { + case *chast.AliasedExpr: + return e.Alias + case *chast.Identifier: + return e.Alias case *chast.FunctionCall: - if e.Alias != "" { - res.Name = &e.Alias + return e.Alias + case *chast.CaseExpr: + return e.Alias + case *chast.CastExpr: + return e.Alias + case *chast.LikeExpr: + return e.Alias + case *chast.ExtractExpr: + return e.Alias + case *chast.Subquery: + return e.Alias + } + return "" +} + +// binaryFunctions are the function names ClickHouse gives its operators when +// it names a column after an expression. +var binaryFunctions = map[string]string{ + "+": "plus", "-": "minus", "*": "multiply", "/": "divide", "%": "modulo", + "=": "equals", "==": "equals", "!=": "notEquals", "<>": "notEquals", + "<": "less", ">": "greater", "<=": "lessOrEquals", ">=": "greaterOrEquals", + "AND": "and", "OR": "or", "||": "concat", +} + +// columnName writes an expression the way ClickHouse names a result column +// that has no alias: a function call with its arguments, an operator as the +// function it stands for, an identifier or literal as written. It returns "" +// for an expression it cannot write, which then keeps sqlc's own name. +func columnName(expr chast.Expression) string { + switch e := expr.(type) { + case *chast.Identifier: + return e.Name() + case *chast.Literal: + return literalText(e) + case *chast.FunctionCall: + if e.Over != nil || e.Filter != nil || e.Distinct { + return "" } - res.Val = c.convertFunctionCall(e) - default: - res.Val = c.convertExpr(expr) + var params, args []string + for _, p := range e.Parameters { + s := columnName(p) + if s == "" { + return "" + } + params = append(params, s) + } + for _, a := range e.Arguments { + s := columnName(a) + if s == "" { + return "" + } + args = append(args, s) + } + name := e.Name + if len(e.Parameters) > 0 { + name += "(" + strings.Join(params, ", ") + ")" + } + return name + "(" + strings.Join(args, ", ") + ")" + case *chast.BinaryExpr: + fn, ok := binaryFunctions[strings.ToUpper(e.Op)] + if !ok { + return "" + } + left, right := columnName(e.Left), columnName(e.Right) + if left == "" || right == "" { + return "" + } + return fn + "(" + left + ", " + right + ")" + case *chast.UnaryExpr: + operand := columnName(e.Operand) + if operand == "" { + return "" + } + switch strings.ToUpper(e.Op) { + case "-": + return "negate(" + operand + ")" + case "NOT": + return "not(" + operand + ")" + } + return "" + case *chast.IsNullExpr: + arg := columnName(e.Expr) + if arg == "" { + return "" + } + if e.Not { + return "isNotNull(" + arg + ")" + } + return "isNull(" + arg + ")" + case *chast.TernaryExpr: + cond, then, els := columnName(e.Condition), columnName(e.Then), columnName(e.Else) + if cond == "" || then == "" || els == "" { + return "" + } + return "if(" + cond + ", " + then + ", " + els + ")" } + return "" +} - return res +// literalText writes a literal as ClickHouse prints it in a column name. +func literalText(l *chast.Literal) string { + switch l.Type { + case chast.LiteralString: + return quoteString(fmt.Sprint(l.Value)) + case chast.LiteralNull: + return "NULL" + case chast.LiteralBoolean: + return fmt.Sprint(l.Value) + case chast.LiteralInteger, chast.LiteralFloat: + if l.Source != "" { + return l.Source + } + return fmt.Sprint(l.Value) + } + return "" +} + +func quoteString(s string) string { + s = strings.ReplaceAll(s, `\`, `\\`) + s = strings.ReplaceAll(s, "'", `\'`) + return "'" + s + "'" } func (c *cc) convertTablesInSelectQuery(n *chast.TablesInSelectQuery) *ast.List { @@ -341,7 +495,23 @@ func (c *cc) convertExpr(expr chast.Expression) ast.Node { case *chast.BinaryExpr: return c.convertBinaryExpr(e) case *chast.FunctionCall: + switch strings.ToLower(e.Name) { + case "coalesce", "ifnull": + // COALESCE is null only when every argument is, which a seeded + // function signature cannot say. + args := &ast.List{} + for _, arg := range e.Arguments { + args.Items = append(args.Items, c.convertExpr(arg)) + } + return &ast.CoalesceExpr{Args: args, Location: pos(e)} + } return c.convertFunctionCall(e) + case *chast.ExistsExpr: + return &ast.SubLink{ + SubLinkType: ast.EXISTS_SUBLINK, + Subselect: c.convert(e.Query), + Location: pos(e), + } case *chast.AliasedExpr: return c.convertExpr(e.Expr) case *chast.Parameter: @@ -381,7 +551,7 @@ func (c *cc) convertIdentifier(n *chast.Identifier) *ast.ColumnRef { } return &ast.ColumnRef{ Fields: fields, - Location: n.Pos().Offset, + Location: pos(n), } } @@ -391,7 +561,7 @@ func (c *cc) convertLiteral(n *chast.Literal) *ast.A_Const { str := n.Value.(string) return &ast.A_Const{ Val: &ast.String{Str: str}, - Location: n.Pos().Offset, + Location: pos(n), } case chast.LiteralInteger: var ival int64 @@ -407,7 +577,7 @@ func (c *cc) convertLiteral(n *chast.Literal) *ast.A_Const { } return &ast.A_Const{ Val: &ast.Integer{Ival: ival}, - Location: n.Pos().Offset, + Location: pos(n), } case chast.LiteralFloat: var fval float64 @@ -420,7 +590,7 @@ func (c *cc) convertLiteral(n *chast.Literal) *ast.A_Const { str := strconv.FormatFloat(fval, 'f', -1, 64) return &ast.A_Const{ Val: &ast.Float{Str: str}, - Location: n.Pos().Offset, + Location: pos(n), } case chast.LiteralBoolean: // ClickHouse booleans are typically 0/1 @@ -428,21 +598,21 @@ func (c *cc) convertLiteral(n *chast.Literal) *ast.A_Const { if bval { return &ast.A_Const{ Val: &ast.Integer{Ival: 1}, - Location: n.Pos().Offset, + Location: pos(n), } } return &ast.A_Const{ Val: &ast.Integer{Ival: 0}, - Location: n.Pos().Offset, + Location: pos(n), } case chast.LiteralNull: return &ast.A_Const{ Val: &ast.Null{}, - Location: n.Pos().Offset, + Location: pos(n), } default: return &ast.A_Const{ - Location: n.Pos().Offset, + Location: pos(n), } } } @@ -466,7 +636,7 @@ func (c *cc) convertBinaryExpr(n *chast.BinaryExpr) ast.Node { c.convertExpr(n.Right), }, }, - Location: n.Pos().Offset, + Location: pos(n), } } @@ -478,7 +648,7 @@ func (c *cc) convertBinaryExpr(n *chast.BinaryExpr) ast.Node { }, Lexpr: c.convertExpr(n.Left), Rexpr: c.convertExpr(n.Right), - Location: n.Pos().Offset, + Location: pos(n), } } @@ -487,7 +657,7 @@ func (c *cc) convertFunctionCall(n *chast.FunctionCall) *ast.FuncCall { Funcname: &ast.List{ Items: []ast.Node{&ast.String{Str: n.Name}}, }, - Location: n.Pos().Offset, + Location: pos(n), AggDistinct: n.Distinct, } @@ -528,7 +698,7 @@ func (c *cc) convertParameter(n *chast.Parameter) ast.Node { } return &ast.ParamRef{ Number: c.paramCount, - Location: n.Pos().Offset, + Location: pos(n), } } @@ -540,13 +710,13 @@ func (c *cc) convertAsterisk(n *chast.Asterisk) *ast.ColumnRef { fields.Items = append(fields.Items, &ast.A_Star{}) return &ast.ColumnRef{ Fields: fields, - Location: n.Pos().Offset, + Location: pos(n), } } func (c *cc) convertCaseExpr(n *chast.CaseExpr) *ast.CaseExpr { ce := &ast.CaseExpr{ - Location: n.Pos().Offset, + Location: pos(n), } // Convert test expression (CASE expr WHEN ...) @@ -577,7 +747,7 @@ func (c *cc) convertCaseExpr(n *chast.CaseExpr) *ast.CaseExpr { func (c *cc) convertCastExpr(n *chast.CastExpr) *ast.TypeCast { tc := &ast.TypeCast{ Arg: c.convertExpr(n.Expr), - Location: n.Pos().Offset, + Location: pos(n), } if n.Type != nil { @@ -595,7 +765,7 @@ func (c *cc) convertBetweenExpr(n *chast.BetweenExpr) *ast.BetweenExpr { Left: c.convertExpr(n.Low), Right: c.convertExpr(n.High), Not: n.Not, - Location: n.Pos().Offset, + Location: pos(n), } } @@ -603,7 +773,7 @@ func (c *cc) convertInExpr(n *chast.InExpr) *ast.In { in := &ast.In{ Expr: c.convertExpr(n.Expr), Not: n.Not, - Location: n.Pos().Offset, + Location: pos(n), } // Convert the list @@ -625,7 +795,7 @@ func (c *cc) convertInExpr(n *chast.InExpr) *ast.In { func (c *cc) convertIsNullExpr(n *chast.IsNullExpr) *ast.NullTest { nullTest := &ast.NullTest{ Arg: c.convertExpr(n.Expr), - Location: n.Pos().Offset, + Location: pos(n), } if n.Not { nullTest.Nulltesttype = ast.NullTestTypeIsNotNull @@ -655,14 +825,17 @@ func (c *cc) convertLikeExpr(n *chast.LikeExpr) *ast.A_Expr { }, Lexpr: c.convertExpr(n.Expr), Rexpr: c.convertExpr(n.Pattern), - Location: n.Pos().Offset, + Location: pos(n), } } +// convertSubquery converts a subquery used as a value: a scalar subquery. +// EXISTS is its own node. func (c *cc) convertSubquery(n *chast.Subquery) *ast.SubLink { return &ast.SubLink{ - SubLinkType: ast.EXISTS_SUBLINK, + SubLinkType: ast.EXPR_SUBLINK, Subselect: c.convert(n.Query), + Location: pos(n), } } @@ -688,7 +861,7 @@ func (c *cc) convertUnaryExpr(n *chast.UnaryExpr) ast.Node { Args: &ast.List{ Items: []ast.Node{c.convertExpr(n.Operand)}, }, - Location: n.Pos().Offset, + Location: pos(n), } } @@ -698,14 +871,14 @@ func (c *cc) convertUnaryExpr(n *chast.UnaryExpr) ast.Node { Items: []ast.Node{&ast.String{Str: n.Op}}, }, Rexpr: c.convertExpr(n.Operand), - Location: n.Pos().Offset, + Location: pos(n), } } func (c *cc) convertOrderByElement(n *chast.OrderByElement) *ast.SortBy { sortBy := &ast.SortBy{ Node: c.convertExpr(n.Expression), - Location: n.Expression.Pos().Offset, + Location: pos(n.Expression), } if n.Descending { @@ -828,8 +1001,11 @@ func (c *cc) convertColumnDeclaration(n *chast.ColumnDeclaration) *ast.ColumnDef } if n.Type != nil { - base, isArray, nullable := unwrapTypeString(renderDataType(n.Type)) - colDef.TypeName = &ast.TypeName{Name: base} + spelling := renderDataType(n.Type) + base, isArray, nullable := unwrapTypeString(spelling) + // The catalog resolves the base type; the full spelling, with its + // arguments and nesting, is kept for the analysis to report. + colDef.TypeName = &ast.TypeName{Name: base, Spelling: spelling} colDef.IsArray = isArray if nullable { colDef.IsNotNull = false @@ -873,12 +1049,21 @@ func renderTypeParam(e chast.Expression) string { case *chast.DataType: return renderDataType(v) case *chast.Literal: + if v.Type == chast.LiteralString { + return quoteString(fmt.Sprint(v.Value)) + } if v.Source != "" { return v.Source } return fmt.Sprintf("%v", v.Value) case *chast.Identifier: return strings.Join(v.Parts, ".") + case *chast.NameTypePair: + // A named tuple or nested element: `lat Float64`. + return v.Name + " " + renderDataType(v.Type) + case *chast.BinaryExpr: + // An enum member: `'active' = 1`. + return renderTypeParam(v.Left) + " " + v.Op + " " + renderTypeParam(v.Right) default: return "" } diff --git a/internal/engine/clickhouse/dialect/dialect.json b/internal/engine/clickhouse/dialect/dialect.json index 3be4b1c4d2..fa7f2c5c73 100644 --- a/internal/engine/clickhouse/dialect/dialect.json +++ b/internal/engine/clickhouse/dialect/dialect.json @@ -6,9 +6,13 @@ "string": "String", "bool": "Bool" }, - "bool": "Bool", - "comparison": ["=", "<>", "!=", "<", "<=", ">", ">="], - "comparison_categories": "NBSD", - "arithmetic": ["+", "-", "*", "/"], + "bool": "UInt8", + "limit": "UInt64", + "untyped": "Nothing", + "propagate_nullable": true, + "qualify_duplicate_columns": true, + "comparison": ["=", "==", "<>", "!=", "<", "<=", ">", ">="], + "comparison_categories": "NBSDU", + "arithmetic": ["+", "-", "*", "/", "%"], "arithmetic_categories": "N" } diff --git a/internal/engine/clickhouse/dialect/functions.jsonl b/internal/engine/clickhouse/dialect/functions.jsonl index 3d0f936072..20898b8c9d 100644 --- a/internal/engine/clickhouse/dialect/functions.jsonl +++ b/internal/engine/clickhouse/dialect/functions.jsonl @@ -1 +1,581 @@ -{"name": "count", "kind": "a", "returns": "UInt64"} +{"name": "count", "kind": "a", "returns": "UInt64", "never_null": true} +{"name": "count", "kind": "a", "args": [{"type": "any"}], "returns": "UInt64", "never_null": true} +{"name": "countIf", "kind": "a", "args": [{"type": "any"}, {"type": "any"}], "returns": "UInt64", "never_null": true} +{"name": "sum", "kind": "a", "args": [{"type": "UInt8"}], "returns": "UInt64"} +{"name": "sum", "kind": "a", "args": [{"type": "UInt16"}], "returns": "UInt64"} +{"name": "sum", "kind": "a", "args": [{"type": "UInt32"}], "returns": "UInt64"} +{"name": "sum", "kind": "a", "args": [{"type": "Int8"}], "returns": "Int64"} +{"name": "sum", "kind": "a", "args": [{"type": "Int16"}], "returns": "Int64"} +{"name": "sum", "kind": "a", "args": [{"type": "Int32"}], "returns": "Int64"} +{"name": "sum", "kind": "a", "args": [{"type": "Float32"}], "returns": "Float64"} +{"name": "sum", "kind": "a", "args": [{"type": "any"}], "returns": "$1"} +{"name": "sumIf", "kind": "a", "args": [{"type": "any"}, {"type": "any"}], "returns": "$1"} +{"name": "avg", "kind": "a", "args": [{"type": "any"}], "returns": "Float64"} +{"name": "avgIf", "kind": "a", "args": [{"type": "any"}, {"type": "any"}], "returns": "Float64"} +{"name": "min", "kind": "a", "args": [{"type": "any"}], "returns": "$1"} +{"name": "max", "kind": "a", "args": [{"type": "any"}], "returns": "$1"} +{"name": "any", "kind": "a", "args": [{"type": "any"}], "returns": "$1"} +{"name": "anyLast", "kind": "a", "args": [{"type": "any"}], "returns": "$1"} +{"name": "minIf", "kind": "a", "args": [{"type": "any"}, {"type": "any"}], "returns": "$1"} +{"name": "maxIf", "kind": "a", "args": [{"type": "any"}, {"type": "any"}], "returns": "$1"} +{"name": "anyIf", "kind": "a", "args": [{"type": "any"}, {"type": "any"}], "returns": "$1"} +{"name": "uniq", "kind": "a", "args": [{"type": "any"}], "returns": "UInt64", "never_null": true} +{"name": "uniqExact", "kind": "a", "args": [{"type": "any"}], "returns": "UInt64", "never_null": true} +{"name": "uniqCombined", "kind": "a", "args": [{"type": "any"}], "returns": "UInt64", "never_null": true} +{"name": "uniqHLL12", "kind": "a", "args": [{"type": "any"}], "returns": "UInt64", "never_null": true} +{"name": "uniqIf", "kind": "a", "args": [{"type": "any"}, {"type": "any"}], "returns": "UInt64", "never_null": true} +{"name": "argMin", "kind": "a", "args": [{"type": "any"}, {"type": "any"}], "returns": "$1"} +{"name": "argMax", "kind": "a", "args": [{"type": "any"}, {"type": "any"}], "returns": "$1"} +{"name": "stddevPop", "kind": "a", "args": [{"type": "any"}], "returns": "Float64"} +{"name": "stddevSamp", "kind": "a", "args": [{"type": "any"}], "returns": "Float64"} +{"name": "varPop", "kind": "a", "args": [{"type": "any"}], "returns": "Float64"} +{"name": "varSamp", "kind": "a", "args": [{"type": "any"}], "returns": "Float64"} +{"name": "median", "kind": "a", "args": [{"type": "any"}], "returns": "Float64"} +{"name": "quantile", "kind": "a", "args": [{"type": "any"}], "returns": "Float64"} +{"name": "quantile", "kind": "a", "args": [{"type": "any"}, {"type": "any"}], "returns": "Float64"} +{"name": "row_number", "kind": "w", "returns": "UInt64", "never_null": true} +{"name": "rank", "kind": "w", "returns": "UInt64", "never_null": true} +{"name": "dense_rank", "kind": "w", "returns": "UInt64", "never_null": true} +{"name": "plus", "args": [{"type": "any"}, {"type": "any"}], "returns": "$1"} +{"name": "minus", "args": [{"type": "any"}, {"type": "any"}], "returns": "$1"} +{"name": "multiply", "args": [{"type": "any"}, {"type": "any"}], "returns": "$1"} +{"name": "intDiv", "args": [{"type": "any"}, {"type": "any"}], "returns": "$1"} +{"name": "intDivOrZero", "args": [{"type": "any"}, {"type": "any"}], "returns": "$1"} +{"name": "modulo", "args": [{"type": "any"}, {"type": "any"}], "returns": "$1"} +{"name": "moduloOrZero", "args": [{"type": "any"}, {"type": "any"}], "returns": "$1"} +{"name": "gcd", "args": [{"type": "any"}, {"type": "any"}], "returns": "$1"} +{"name": "lcm", "args": [{"type": "any"}, {"type": "any"}], "returns": "$1"} +{"name": "bitAnd", "args": [{"type": "any"}, {"type": "any"}], "returns": "$1"} +{"name": "bitOr", "args": [{"type": "any"}, {"type": "any"}], "returns": "$1"} +{"name": "bitXor", "args": [{"type": "any"}, {"type": "any"}], "returns": "$1"} +{"name": "bitShiftLeft", "args": [{"type": "any"}, {"type": "any"}], "returns": "$1"} +{"name": "bitShiftRight", "args": [{"type": "any"}, {"type": "any"}], "returns": "$1"} +{"name": "greatest", "args": [{"type": "any"}, {"type": "any"}], "returns": "$1"} +{"name": "least", "args": [{"type": "any"}, {"type": "any"}], "returns": "$1"} +{"name": "divide", "args": [{"type": "any"}, {"type": "any"}], "returns": "Float64"} +{"name": "negate", "args": [{"type": "any"}], "returns": "$1"} +{"name": "abs", "args": [{"type": "any"}], "returns": "$1"} +{"name": "round", "args": [{"type": "any"}], "returns": "$1"} +{"name": "floor", "args": [{"type": "any"}], "returns": "$1"} +{"name": "ceil", "args": [{"type": "any"}], "returns": "$1"} +{"name": "ceiling", "args": [{"type": "any"}], "returns": "$1"} +{"name": "trunc", "args": [{"type": "any"}], "returns": "$1"} +{"name": "truncate", "args": [{"type": "any"}], "returns": "$1"} +{"name": "identity", "args": [{"type": "any"}], "returns": "$1"} +{"name": "bitNot", "args": [{"type": "any"}], "returns": "$1"} +{"name": "round", "args": [{"type": "any"}, {"type": "any"}], "returns": "$1"} +{"name": "floor", "args": [{"type": "any"}, {"type": "any"}], "returns": "$1"} +{"name": "ceil", "args": [{"type": "any"}, {"type": "any"}], "returns": "$1"} +{"name": "trunc", "args": [{"type": "any"}, {"type": "any"}], "returns": "$1"} +{"name": "exp", "args": [{"type": "any"}], "returns": "Float64"} +{"name": "log", "args": [{"type": "any"}], "returns": "Float64"} +{"name": "ln", "args": [{"type": "any"}], "returns": "Float64"} +{"name": "exp2", "args": [{"type": "any"}], "returns": "Float64"} +{"name": "log2", "args": [{"type": "any"}], "returns": "Float64"} +{"name": "exp10", "args": [{"type": "any"}], "returns": "Float64"} +{"name": "log10", "args": [{"type": "any"}], "returns": "Float64"} +{"name": "sqrt", "args": [{"type": "any"}], "returns": "Float64"} +{"name": "cbrt", "args": [{"type": "any"}], "returns": "Float64"} +{"name": "sin", "args": [{"type": "any"}], "returns": "Float64"} +{"name": "cos", "args": [{"type": "any"}], "returns": "Float64"} +{"name": "tan", "args": [{"type": "any"}], "returns": "Float64"} +{"name": "asin", "args": [{"type": "any"}], "returns": "Float64"} +{"name": "acos", "args": [{"type": "any"}], "returns": "Float64"} +{"name": "atan", "args": [{"type": "any"}], "returns": "Float64"} +{"name": "sigmoid", "args": [{"type": "any"}], "returns": "Float64"} +{"name": "erf", "args": [{"type": "any"}], "returns": "Float64"} +{"name": "lgamma", "args": [{"type": "any"}], "returns": "Float64"} +{"name": "tgamma", "args": [{"type": "any"}], "returns": "Float64"} +{"name": "pow", "args": [{"type": "any"}, {"type": "any"}], "returns": "Float64"} +{"name": "power", "args": [{"type": "any"}, {"type": "any"}], "returns": "Float64"} +{"name": "atan2", "args": [{"type": "any"}, {"type": "any"}], "returns": "Float64"} +{"name": "hypot", "args": [{"type": "any"}, {"type": "any"}], "returns": "Float64"} +{"name": "e", "returns": "Float64", "never_null": true} +{"name": "pi", "returns": "Float64", "never_null": true} +{"name": "sign", "args": [{"type": "any"}], "returns": "Int8"} +{"name": "isFinite", "args": [{"type": "any"}], "returns": "UInt8"} +{"name": "isInfinite", "args": [{"type": "any"}], "returns": "UInt8"} +{"name": "isNaN", "args": [{"type": "any"}], "returns": "UInt8"} +{"name": "rand", "returns": "UInt32", "never_null": true} +{"name": "rand32", "returns": "UInt32", "never_null": true} +{"name": "rand64", "returns": "UInt64", "never_null": true} +{"name": "randCanonical", "returns": "Float64", "never_null": true} +{"name": "equals", "args": [{"type": "any"}, {"type": "any"}], "returns": "UInt8"} +{"name": "notEquals", "args": [{"type": "any"}, {"type": "any"}], "returns": "UInt8"} +{"name": "less", "args": [{"type": "any"}, {"type": "any"}], "returns": "UInt8"} +{"name": "greater", "args": [{"type": "any"}, {"type": "any"}], "returns": "UInt8"} +{"name": "lessOrEquals", "args": [{"type": "any"}, {"type": "any"}], "returns": "UInt8"} +{"name": "greaterOrEquals", "args": [{"type": "any"}, {"type": "any"}], "returns": "UInt8"} +{"name": "and", "args": [{"type": "any"}, {"type": "any"}], "returns": "UInt8"} +{"name": "or", "args": [{"type": "any"}, {"type": "any"}], "returns": "UInt8"} +{"name": "xor", "args": [{"type": "any"}, {"type": "any"}], "returns": "UInt8"} +{"name": "like", "args": [{"type": "any"}, {"type": "any"}], "returns": "UInt8"} +{"name": "notLike", "args": [{"type": "any"}, {"type": "any"}], "returns": "UInt8"} +{"name": "ilike", "args": [{"type": "any"}, {"type": "any"}], "returns": "UInt8"} +{"name": "notILike", "args": [{"type": "any"}, {"type": "any"}], "returns": "UInt8"} +{"name": "match", "args": [{"type": "any"}, {"type": "any"}], "returns": "UInt8"} +{"name": "startsWith", "args": [{"type": "any"}, {"type": "any"}], "returns": "UInt8"} +{"name": "endsWith", "args": [{"type": "any"}, {"type": "any"}], "returns": "UInt8"} +{"name": "has", "args": [{"type": "any"}, {"type": "any"}], "returns": "UInt8"} +{"name": "hasAll", "args": [{"type": "any"}, {"type": "any"}], "returns": "UInt8"} +{"name": "hasAny", "args": [{"type": "any"}, {"type": "any"}], "returns": "UInt8"} +{"name": "mapContains", "args": [{"type": "any"}, {"type": "any"}], "returns": "UInt8"} +{"name": "multiSearchAny", "args": [{"type": "any"}, {"type": "any"}], "returns": "UInt8"} +{"name": "in", "args": [{"type": "any"}, {"type": "any"}], "returns": "UInt8"} +{"name": "notIn", "args": [{"type": "any"}, {"type": "any"}], "returns": "UInt8"} +{"name": "not", "args": [{"type": "any"}], "returns": "UInt8"} +{"name": "isNull", "args": [{"type": "any"}], "returns": "UInt8", "never_null": true} +{"name": "isNotNull", "args": [{"type": "any"}], "returns": "UInt8", "never_null": true} +{"name": "isZeroOrNull", "args": [{"type": "any"}], "returns": "UInt8", "never_null": true} +{"name": "assumeNotNull", "args": [{"type": "any"}], "returns": "$1", "never_null": true} +{"name": "toNullable", "args": [{"type": "any"}], "returns": "$1", "nullable": true} +{"name": "nullIf", "args": [{"type": "any"}, {"type": "any"}], "returns": "$1", "nullable": true} +{"name": "if", "args": [{"type": "any"}, {"type": "any"}, {"type": "any"}], "returns": "$2"} +{"name": "empty", "args": [{"type": "any"}], "returns": "UInt8"} +{"name": "notEmpty", "args": [{"type": "any"}], "returns": "UInt8"} +{"name": "length", "args": [{"type": "any"}], "returns": "UInt64"} +{"name": "lengthUTF8", "args": [{"type": "any"}], "returns": "UInt64"} +{"name": "char_length", "args": [{"type": "any"}], "returns": "UInt64"} +{"name": "character_length", "args": [{"type": "any"}], "returns": "UInt64"} +{"name": "position", "args": [{"type": "any"}, {"type": "any"}], "returns": "UInt64"} +{"name": "positionUTF8", "args": [{"type": "any"}, {"type": "any"}], "returns": "UInt64"} +{"name": "positionCaseInsensitive", "args": [{"type": "any"}, {"type": "any"}], "returns": "UInt64"} +{"name": "countSubstrings", "args": [{"type": "any"}, {"type": "any"}], "returns": "UInt64"} +{"name": "indexOf", "args": [{"type": "any"}, {"type": "any"}], "returns": "UInt64"} +{"name": "countEqual", "args": [{"type": "any"}, {"type": "any"}], "returns": "UInt64"} +{"name": "ngramSearch", "args": [{"type": "any"}, {"type": "any"}], "returns": "UInt64"} +{"name": "lower", "args": [{"type": "any"}], "returns": "String"} +{"name": "upper", "args": [{"type": "any"}], "returns": "String"} +{"name": "lowerUTF8", "args": [{"type": "any"}], "returns": "String"} +{"name": "upperUTF8", "args": [{"type": "any"}], "returns": "String"} +{"name": "toString", "args": [{"type": "any"}], "returns": "String"} +{"name": "toValidUTF8", "args": [{"type": "any"}], "returns": "String"} +{"name": "trim", "args": [{"type": "any"}], "returns": "String"} +{"name": "trimLeft", "args": [{"type": "any"}], "returns": "String"} +{"name": "trimRight", "args": [{"type": "any"}], "returns": "String"} +{"name": "trimBoth", "args": [{"type": "any"}], "returns": "String"} +{"name": "ltrim", "args": [{"type": "any"}], "returns": "String"} +{"name": "rtrim", "args": [{"type": "any"}], "returns": "String"} +{"name": "hex", "args": [{"type": "any"}], "returns": "String"} +{"name": "unhex", "args": [{"type": "any"}], "returns": "String"} +{"name": "bin", "args": [{"type": "any"}], "returns": "String"} +{"name": "base64Encode", "args": [{"type": "any"}], "returns": "String"} +{"name": "base64Decode", "args": [{"type": "any"}], "returns": "String"} +{"name": "tryBase64Decode", "args": [{"type": "any"}], "returns": "String"} +{"name": "reverseUTF8", "args": [{"type": "any"}], "returns": "String"} +{"name": "initcap", "args": [{"type": "any"}], "returns": "String"} +{"name": "normalizeQuery", "args": [{"type": "any"}], "returns": "String"} +{"name": "typeName", "args": [{"type": "any"}], "returns": "String"} +{"name": "dumpColumnStructure", "args": [{"type": "any"}], "returns": "String"} +{"name": "urlDecode", "args": [{"type": "any"}], "returns": "String"} +{"name": "encodeURLComponent", "args": [{"type": "any"}], "returns": "String"} +{"name": "decodeURLComponent", "args": [{"type": "any"}], "returns": "String"} +{"name": "domain", "args": [{"type": "any"}], "returns": "String"} +{"name": "topLevelDomain", "args": [{"type": "any"}], "returns": "String"} +{"name": "path", "args": [{"type": "any"}], "returns": "String"} +{"name": "pathFull", "args": [{"type": "any"}], "returns": "String"} +{"name": "protocol", "args": [{"type": "any"}], "returns": "String"} +{"name": "queryString", "args": [{"type": "any"}], "returns": "String"} +{"name": "fragment", "args": [{"type": "any"}], "returns": "String"} +{"name": "cutQueryString", "args": [{"type": "any"}], "returns": "String"} +{"name": "cutFragment", "args": [{"type": "any"}], "returns": "String"} +{"name": "cutWWW", "args": [{"type": "any"}], "returns": "String"} +{"name": "firstSignificantSubdomain", "args": [{"type": "any"}], "returns": "String"} +{"name": "IPv4NumToString", "args": [{"type": "any"}], "returns": "String"} +{"name": "IPv6NumToString", "args": [{"type": "any"}], "returns": "String"} +{"name": "formatReadableSize", "args": [{"type": "any"}], "returns": "String"} +{"name": "formatReadableQuantity", "args": [{"type": "any"}], "returns": "String"} +{"name": "formatReadableDecimalSize", "args": [{"type": "any"}], "returns": "String"} +{"name": "formatReadableTimeDelta", "args": [{"type": "any"}], "returns": "String"} +{"name": "toDecimalString", "args": [{"type": "any"}], "returns": "String"} +{"name": "monthName", "args": [{"type": "any"}], "returns": "String"} +{"name": "toJSONString", "args": [{"type": "any"}], "returns": "String"} +{"name": "toTypeName", "args": [{"type": "any"}], "returns": "String", "never_null": true} +{"name": "hostName", "returns": "String", "never_null": true} +{"name": "version", "returns": "String", "never_null": true} +{"name": "currentDatabase", "returns": "String", "never_null": true} +{"name": "currentUser", "returns": "String", "never_null": true} +{"name": "FQDN", "returns": "String", "never_null": true} +{"name": "queryID", "returns": "String", "never_null": true} +{"name": "generateULID", "returns": "String", "never_null": true} +{"name": "concat", "args": [{"type": "any"}, {"type": "any"}], "returns": "String"} +{"name": "substring", "args": [{"type": "any"}, {"type": "any"}], "returns": "String"} +{"name": "substr", "args": [{"type": "any"}, {"type": "any"}], "returns": "String"} +{"name": "mid", "args": [{"type": "any"}, {"type": "any"}], "returns": "String"} +{"name": "substringUTF8", "args": [{"type": "any"}, {"type": "any"}], "returns": "String"} +{"name": "repeat", "args": [{"type": "any"}, {"type": "any"}], "returns": "String"} +{"name": "leftPad", "args": [{"type": "any"}, {"type": "any"}], "returns": "String"} +{"name": "rightPad", "args": [{"type": "any"}, {"type": "any"}], "returns": "String"} +{"name": "left", "args": [{"type": "any"}, {"type": "any"}], "returns": "String"} +{"name": "right", "args": [{"type": "any"}, {"type": "any"}], "returns": "String"} +{"name": "extract", "args": [{"type": "any"}, {"type": "any"}], "returns": "String"} +{"name": "formatDateTime", "args": [{"type": "any"}, {"type": "any"}], "returns": "String"} +{"name": "formatDateTimeInJodaSyntax", "args": [{"type": "any"}, {"type": "any"}], "returns": "String"} +{"name": "dateName", "args": [{"type": "any"}, {"type": "any"}], "returns": "String"} +{"name": "JSONExtractString", "args": [{"type": "any"}, {"type": "any"}], "returns": "String"} +{"name": "JSONExtractRaw", "args": [{"type": "any"}, {"type": "any"}], "returns": "String"} +{"name": "visitParamExtractString", "args": [{"type": "any"}, {"type": "any"}], "returns": "String"} +{"name": "simpleJSONExtractString", "args": [{"type": "any"}, {"type": "any"}], "returns": "String"} +{"name": "arrayStringConcat", "args": [{"type": "any"}, {"type": "any"}], "returns": "String"} +{"name": "regexpExtract", "args": [{"type": "any"}, {"type": "any"}], "returns": "String"} +{"name": "translate", "args": [{"type": "any"}, {"type": "any"}], "returns": "String"} +{"name": "format", "args": [{"type": "any"}, {"type": "any"}], "returns": "String"} +{"name": "toDecimalString", "args": [{"type": "any"}, {"type": "any"}], "returns": "String"} +{"name": "concat", "args": [{"type": "any"}, {"type": "any"}, {"type": "any"}], "returns": "String"} +{"name": "substring", "args": [{"type": "any"}, {"type": "any"}, {"type": "any"}], "returns": "String"} +{"name": "substr", "args": [{"type": "any"}, {"type": "any"}, {"type": "any"}], "returns": "String"} +{"name": "mid", "args": [{"type": "any"}, {"type": "any"}, {"type": "any"}], "returns": "String"} +{"name": "substringUTF8", "args": [{"type": "any"}, {"type": "any"}, {"type": "any"}], "returns": "String"} +{"name": "leftPad", "args": [{"type": "any"}, {"type": "any"}, {"type": "any"}], "returns": "String"} +{"name": "rightPad", "args": [{"type": "any"}, {"type": "any"}, {"type": "any"}], "returns": "String"} +{"name": "replaceOne", "args": [{"type": "any"}, {"type": "any"}, {"type": "any"}], "returns": "String"} +{"name": "replaceAll", "args": [{"type": "any"}, {"type": "any"}, {"type": "any"}], "returns": "String"} +{"name": "replaceRegexpOne", "args": [{"type": "any"}, {"type": "any"}, {"type": "any"}], "returns": "String"} +{"name": "replaceRegexpAll", "args": [{"type": "any"}, {"type": "any"}, {"type": "any"}], "returns": "String"} +{"name": "translate", "args": [{"type": "any"}, {"type": "any"}, {"type": "any"}], "returns": "String"} +{"name": "format", "args": [{"type": "any"}, {"type": "any"}, {"type": "any"}], "returns": "String"} +{"name": "concatWithSeparator", "args": [{"type": "any"}, {"type": "any"}, {"type": "any"}], "returns": "String"} +{"name": "regexpExtract", "args": [{"type": "any"}, {"type": "any"}, {"type": "any"}], "returns": "String"} +{"name": "formatDateTime", "args": [{"type": "any"}, {"type": "any"}, {"type": "any"}], "returns": "String"} +{"name": "JSONExtractString", "args": [{"type": "any"}, {"type": "any"}, {"type": "any"}], "returns": "String"} +{"name": "JSONExtractRaw", "args": [{"type": "any"}, {"type": "any"}, {"type": "any"}], "returns": "String"} +{"name": "concat", "args": [{"type": "any"}, {"type": "any"}, {"type": "any"}, {"type": "any"}], "returns": "String"} +{"name": "concat", "args": [{"type": "any"}, {"type": "any"}, {"type": "any"}, {"type": "any"}, {"type": "any"}], "returns": "String"} +{"name": "concat", "args": [{"type": "any"}, {"type": "any"}, {"type": "any"}, {"type": "any"}, {"type": "any"}, {"type": "any"}], "returns": "String"} +{"name": "arrayStringConcat", "args": [{"type": "any"}], "returns": "String"} +{"name": "splitByChar", "args": [{"type": "any"}, {"type": "any"}], "returns": "Array(String)"} +{"name": "splitByString", "args": [{"type": "any"}, {"type": "any"}], "returns": "Array(String)"} +{"name": "splitByRegexp", "args": [{"type": "any"}, {"type": "any"}], "returns": "Array(String)"} +{"name": "splitByWhitespace", "args": [{"type": "any"}, {"type": "any"}], "returns": "Array(String)"} +{"name": "alphaTokens", "args": [{"type": "any"}, {"type": "any"}], "returns": "Array(String)"} +{"name": "extractAll", "args": [{"type": "any"}, {"type": "any"}], "returns": "Array(String)"} +{"name": "tokens", "args": [{"type": "any"}, {"type": "any"}], "returns": "Array(String)"} +{"name": "splitByWhitespace", "args": [{"type": "any"}], "returns": "Array(String)"} +{"name": "alphaTokens", "args": [{"type": "any"}], "returns": "Array(String)"} +{"name": "tokens", "args": [{"type": "any"}], "returns": "Array(String)"} +{"name": "reverse", "args": [{"type": "any"}], "returns": "$1"} +{"name": "md5", "args": [{"type": "any"}], "returns": "FixedString(16)"} +{"name": "MD5", "args": [{"type": "any"}], "returns": "FixedString(16)"} +{"name": "sipHash128", "args": [{"type": "any"}], "returns": "FixedString(16)"} +{"name": "murmurHash3_128", "args": [{"type": "any"}], "returns": "FixedString(16)"} +{"name": "cityHash128", "args": [{"type": "any"}], "returns": "FixedString(16)"} +{"name": "SHA1", "args": [{"type": "any"}], "returns": "FixedString(20)"} +{"name": "SHA224", "args": [{"type": "any"}], "returns": "FixedString(28)"} +{"name": "SHA256", "args": [{"type": "any"}], "returns": "FixedString(32)"} +{"name": "SHA512", "args": [{"type": "any"}], "returns": "FixedString(64)"} +{"name": "sipHash64", "args": [{"type": "any"}], "returns": "UInt64", "never_null": true} +{"name": "cityHash64", "args": [{"type": "any"}], "returns": "UInt64", "never_null": true} +{"name": "xxHash64", "args": [{"type": "any"}], "returns": "UInt64", "never_null": true} +{"name": "halfMD5", "args": [{"type": "any"}], "returns": "UInt64", "never_null": true} +{"name": "farmHash64", "args": [{"type": "any"}], "returns": "UInt64", "never_null": true} +{"name": "javaHash", "args": [{"type": "any"}], "returns": "UInt64", "never_null": true} +{"name": "intHash64", "args": [{"type": "any"}], "returns": "UInt64", "never_null": true} +{"name": "murmurHash2_64", "args": [{"type": "any"}], "returns": "UInt64", "never_null": true} +{"name": "murmurHash3_64", "args": [{"type": "any"}], "returns": "UInt64", "never_null": true} +{"name": "metroHash64", "args": [{"type": "any"}], "returns": "UInt64", "never_null": true} +{"name": "xxh3", "args": [{"type": "any"}], "returns": "UInt64", "never_null": true} +{"name": "wyHash64", "args": [{"type": "any"}], "returns": "UInt64", "never_null": true} +{"name": "xxHash32", "args": [{"type": "any"}], "returns": "UInt32", "never_null": true} +{"name": "intHash32", "args": [{"type": "any"}], "returns": "UInt32", "never_null": true} +{"name": "murmurHash2_32", "args": [{"type": "any"}], "returns": "UInt32", "never_null": true} +{"name": "murmurHash3_32", "args": [{"type": "any"}], "returns": "UInt32", "never_null": true} +{"name": "crc32", "args": [{"type": "any"}], "returns": "UInt32", "never_null": true} +{"name": "CRC32", "args": [{"type": "any"}], "returns": "UInt32", "never_null": true} +{"name": "javaHash", "args": [{"type": "any"}], "returns": "UInt32", "never_null": true} +{"name": "levenshteinDistance", "args": [{"type": "any"}, {"type": "any"}], "returns": "UInt64"} +{"name": "editDistance", "args": [{"type": "any"}, {"type": "any"}], "returns": "UInt64"} +{"name": "ngramDistance", "args": [{"type": "any"}, {"type": "any"}], "returns": "Float32"} +{"name": "ngramSearch", "args": [{"type": "any"}, {"type": "any"}], "returns": "Float32"} +{"name": "jaroSimilarity", "args": [{"type": "any"}, {"type": "any"}], "returns": "Float32"} +{"name": "jaroWinklerSimilarity", "args": [{"type": "any"}, {"type": "any"}], "returns": "Float32"} +{"name": "toInt8", "args": [{"type": "any"}], "returns": "Int8"} +{"name": "toInt8OrZero", "args": [{"type": "any"}], "returns": "Int8"} +{"name": "toInt8OrNull", "args": [{"type": "any"}], "returns": "Int8", "nullable": true} +{"name": "toInt16", "args": [{"type": "any"}], "returns": "Int16"} +{"name": "toInt16OrZero", "args": [{"type": "any"}], "returns": "Int16"} +{"name": "toInt16OrNull", "args": [{"type": "any"}], "returns": "Int16", "nullable": true} +{"name": "toInt32", "args": [{"type": "any"}], "returns": "Int32"} +{"name": "toInt32OrZero", "args": [{"type": "any"}], "returns": "Int32"} +{"name": "toInt32OrNull", "args": [{"type": "any"}], "returns": "Int32", "nullable": true} +{"name": "toInt64", "args": [{"type": "any"}], "returns": "Int64"} +{"name": "toInt64OrZero", "args": [{"type": "any"}], "returns": "Int64"} +{"name": "toInt64OrNull", "args": [{"type": "any"}], "returns": "Int64", "nullable": true} +{"name": "toInt128", "args": [{"type": "any"}], "returns": "Int128"} +{"name": "toInt128OrZero", "args": [{"type": "any"}], "returns": "Int128"} +{"name": "toInt128OrNull", "args": [{"type": "any"}], "returns": "Int128", "nullable": true} +{"name": "toInt256", "args": [{"type": "any"}], "returns": "Int256"} +{"name": "toInt256OrZero", "args": [{"type": "any"}], "returns": "Int256"} +{"name": "toInt256OrNull", "args": [{"type": "any"}], "returns": "Int256", "nullable": true} +{"name": "toUInt8", "args": [{"type": "any"}], "returns": "UInt8"} +{"name": "toUInt8OrZero", "args": [{"type": "any"}], "returns": "UInt8"} +{"name": "toUInt8OrNull", "args": [{"type": "any"}], "returns": "UInt8", "nullable": true} +{"name": "toUInt16", "args": [{"type": "any"}], "returns": "UInt16"} +{"name": "toUInt16OrZero", "args": [{"type": "any"}], "returns": "UInt16"} +{"name": "toUInt16OrNull", "args": [{"type": "any"}], "returns": "UInt16", "nullable": true} +{"name": "toUInt32", "args": [{"type": "any"}], "returns": "UInt32"} +{"name": "toUInt32OrZero", "args": [{"type": "any"}], "returns": "UInt32"} +{"name": "toUInt32OrNull", "args": [{"type": "any"}], "returns": "UInt32", "nullable": true} +{"name": "toUInt64", "args": [{"type": "any"}], "returns": "UInt64"} +{"name": "toUInt64OrZero", "args": [{"type": "any"}], "returns": "UInt64"} +{"name": "toUInt64OrNull", "args": [{"type": "any"}], "returns": "UInt64", "nullable": true} +{"name": "toUInt128", "args": [{"type": "any"}], "returns": "UInt128"} +{"name": "toUInt128OrZero", "args": [{"type": "any"}], "returns": "UInt128"} +{"name": "toUInt128OrNull", "args": [{"type": "any"}], "returns": "UInt128", "nullable": true} +{"name": "toUInt256", "args": [{"type": "any"}], "returns": "UInt256"} +{"name": "toUInt256OrZero", "args": [{"type": "any"}], "returns": "UInt256"} +{"name": "toUInt256OrNull", "args": [{"type": "any"}], "returns": "UInt256", "nullable": true} +{"name": "toFloat32", "args": [{"type": "any"}], "returns": "Float32"} +{"name": "toFloat32OrZero", "args": [{"type": "any"}], "returns": "Float32"} +{"name": "toFloat32OrNull", "args": [{"type": "any"}], "returns": "Float32", "nullable": true} +{"name": "toFloat64", "args": [{"type": "any"}], "returns": "Float64"} +{"name": "toFloat64OrZero", "args": [{"type": "any"}], "returns": "Float64"} +{"name": "toFloat64OrNull", "args": [{"type": "any"}], "returns": "Float64", "nullable": true} +{"name": "toDate", "args": [{"type": "any"}], "returns": "Date"} +{"name": "toDateOrZero", "args": [{"type": "any"}], "returns": "Date"} +{"name": "toDateOrNull", "args": [{"type": "any"}], "returns": "Date", "nullable": true} +{"name": "toDate32", "args": [{"type": "any"}], "returns": "Date32"} +{"name": "toDate32OrZero", "args": [{"type": "any"}], "returns": "Date32"} +{"name": "toDate32OrNull", "args": [{"type": "any"}], "returns": "Date32", "nullable": true} +{"name": "toDateTime", "args": [{"type": "any"}], "returns": "DateTime"} +{"name": "toDateTimeOrZero", "args": [{"type": "any"}], "returns": "DateTime"} +{"name": "toDateTimeOrNull", "args": [{"type": "any"}], "returns": "DateTime", "nullable": true} +{"name": "toUUID", "args": [{"type": "any"}], "returns": "UUID"} +{"name": "toUUIDOrZero", "args": [{"type": "any"}], "returns": "UUID"} +{"name": "toUUIDOrNull", "args": [{"type": "any"}], "returns": "UUID", "nullable": true} +{"name": "toIPv4", "args": [{"type": "any"}], "returns": "IPv4"} +{"name": "toIPv4OrZero", "args": [{"type": "any"}], "returns": "IPv4"} +{"name": "toIPv4OrNull", "args": [{"type": "any"}], "returns": "IPv4", "nullable": true} +{"name": "toIPv6", "args": [{"type": "any"}], "returns": "IPv6"} +{"name": "toIPv6OrZero", "args": [{"type": "any"}], "returns": "IPv6"} +{"name": "toIPv6OrNull", "args": [{"type": "any"}], "returns": "IPv6", "nullable": true} +{"name": "toBool", "args": [{"type": "any"}], "returns": "Bool"} +{"name": "toString", "args": [{"type": "any"}], "returns": "String"} +{"name": "toDateTime", "args": [{"type": "any"}, {"type": "any"}], "returns": "DateTime"} +{"name": "toDate", "args": [{"type": "any"}, {"type": "any"}], "returns": "Date"} +{"name": "toDateTime64", "args": [{"type": "any"}, {"type": "any"}], "returns": "DateTime64"} +{"name": "toDateTime64", "args": [{"type": "any"}, {"type": "any"}, {"type": "any"}], "returns": "DateTime64"} +{"name": "toDecimal32", "args": [{"type": "any"}, {"type": "any"}], "returns": "Decimal32"} +{"name": "toDecimal64", "args": [{"type": "any"}, {"type": "any"}], "returns": "Decimal64"} +{"name": "toDecimal128", "args": [{"type": "any"}, {"type": "any"}], "returns": "Decimal128"} +{"name": "toDecimal256", "args": [{"type": "any"}, {"type": "any"}], "returns": "Decimal256"} +{"name": "toFixedString", "args": [{"type": "any"}, {"type": "any"}], "returns": "FixedString"} +{"name": "toStringCutToZero", "args": [{"type": "any"}], "returns": "String"} +{"name": "reinterpretAsString", "args": [{"type": "any"}], "returns": "String"} +{"name": "reinterpretAsUInt64", "args": [{"type": "any"}], "returns": "UInt64"} +{"name": "toUnixTimestamp", "args": [{"type": "any"}], "returns": "UInt32"} +{"name": "toUnixTimestamp", "args": [{"type": "any"}, {"type": "any"}], "returns": "UInt32"} +{"name": "fromUnixTimestamp", "args": [{"type": "any"}], "returns": "DateTime"} +{"name": "toUUIDOrNull", "args": [{"type": "any"}], "returns": "UUID", "nullable": true} +{"name": "toUUIDOrZero", "args": [{"type": "any"}], "returns": "UUID"} +{"name": "generateUUIDv4", "returns": "UUID", "never_null": true} +{"name": "generateUUIDv7", "returns": "UUID", "never_null": true} +{"name": "toIntervalSecond", "args": [{"type": "any"}], "returns": "IntervalSecond"} +{"name": "toIntervalMinute", "args": [{"type": "any"}], "returns": "IntervalMinute"} +{"name": "toIntervalHour", "args": [{"type": "any"}], "returns": "IntervalHour"} +{"name": "toIntervalDay", "args": [{"type": "any"}], "returns": "IntervalDay"} +{"name": "toIntervalWeek", "args": [{"type": "any"}], "returns": "IntervalWeek"} +{"name": "toIntervalMonth", "args": [{"type": "any"}], "returns": "IntervalMonth"} +{"name": "toIntervalYear", "args": [{"type": "any"}], "returns": "IntervalYear"} +{"name": "now", "returns": "DateTime", "never_null": true} +{"name": "now", "args": [{"type": "any"}], "returns": "DateTime", "never_null": true} +{"name": "now64", "returns": "DateTime64", "never_null": true} +{"name": "now64", "args": [{"type": "any"}], "returns": "DateTime64", "never_null": true} +{"name": "today", "returns": "Date", "never_null": true} +{"name": "yesterday", "returns": "Date", "never_null": true} +{"name": "timeZone", "returns": "String", "never_null": true} +{"name": "serverTimezone", "returns": "String", "never_null": true} +{"name": "toYear", "args": [{"type": "any"}], "returns": "UInt16"} +{"name": "toISOYear", "args": [{"type": "any"}], "returns": "UInt16"} +{"name": "toDayOfYear", "args": [{"type": "any"}], "returns": "UInt16"} +{"name": "toRelativeYearNum", "args": [{"type": "any"}], "returns": "UInt16"} +{"name": "toWeek", "args": [{"type": "any"}], "returns": "UInt8"} +{"name": "toISOWeek", "args": [{"type": "any"}], "returns": "UInt8"} +{"name": "toYearWeek", "args": [{"type": "any"}], "returns": "UInt32"} +{"name": "toQuarter", "args": [{"type": "any"}], "returns": "UInt8"} +{"name": "toMonth", "args": [{"type": "any"}], "returns": "UInt8"} +{"name": "toDayOfMonth", "args": [{"type": "any"}], "returns": "UInt8"} +{"name": "toDayOfWeek", "args": [{"type": "any"}], "returns": "UInt8"} +{"name": "toHour", "args": [{"type": "any"}], "returns": "UInt8"} +{"name": "toMinute", "args": [{"type": "any"}], "returns": "UInt8"} +{"name": "toSecond", "args": [{"type": "any"}], "returns": "UInt8"} +{"name": "toMillisecond", "args": [{"type": "any"}], "returns": "UInt8"} +{"name": "toYYYYMM", "args": [{"type": "any"}], "returns": "UInt32"} +{"name": "toYYYYMMDD", "args": [{"type": "any"}], "returns": "UInt32"} +{"name": "toYYYYMMDDhhmmss", "args": [{"type": "any"}], "returns": "UInt64"} +{"name": "toStartOfDay", "args": [{"type": "any"}], "returns": "DateTime"} +{"name": "toStartOfHour", "args": [{"type": "any"}], "returns": "DateTime"} +{"name": "toStartOfMinute", "args": [{"type": "any"}], "returns": "DateTime"} +{"name": "toStartOfFiveMinutes", "args": [{"type": "any"}], "returns": "DateTime"} +{"name": "toStartOfFifteenMinutes", "args": [{"type": "any"}], "returns": "DateTime"} +{"name": "toStartOfTenMinutes", "args": [{"type": "any"}], "returns": "DateTime"} +{"name": "toTime", "args": [{"type": "any"}], "returns": "DateTime"} +{"name": "timeSlot", "args": [{"type": "any"}], "returns": "DateTime"} +{"name": "toLastDayOfMonth", "args": [{"type": "any"}], "returns": "DateTime"} +{"name": "toLastDayOfMonth", "args": [{"type": "any"}], "returns": "Date"} +{"name": "toStartOfMonth", "args": [{"type": "any"}], "returns": "Date"} +{"name": "toStartOfQuarter", "args": [{"type": "any"}], "returns": "Date"} +{"name": "toStartOfYear", "args": [{"type": "any"}], "returns": "Date"} +{"name": "toStartOfWeek", "args": [{"type": "any"}], "returns": "Date"} +{"name": "toMonday", "args": [{"type": "any"}], "returns": "Date"} +{"name": "toStartOfISOYear", "args": [{"type": "any"}], "returns": "Date"} +{"name": "toStartOfWeek", "args": [{"type": "any"}, {"type": "any"}], "returns": "Date"} +{"name": "toStartOfInterval", "args": [{"type": "any"}, {"type": "any"}], "returns": "$1"} +{"name": "addYears", "args": [{"type": "any"}, {"type": "any"}], "returns": "$1"} +{"name": "addQuarters", "args": [{"type": "any"}, {"type": "any"}], "returns": "$1"} +{"name": "addMonths", "args": [{"type": "any"}, {"type": "any"}], "returns": "$1"} +{"name": "addWeeks", "args": [{"type": "any"}, {"type": "any"}], "returns": "$1"} +{"name": "addDays", "args": [{"type": "any"}, {"type": "any"}], "returns": "$1"} +{"name": "addHours", "args": [{"type": "any"}, {"type": "any"}], "returns": "$1"} +{"name": "addMinutes", "args": [{"type": "any"}, {"type": "any"}], "returns": "$1"} +{"name": "addSeconds", "args": [{"type": "any"}, {"type": "any"}], "returns": "$1"} +{"name": "addMilliseconds", "args": [{"type": "any"}, {"type": "any"}], "returns": "$1"} +{"name": "subtractYears", "args": [{"type": "any"}, {"type": "any"}], "returns": "$1"} +{"name": "subtractQuarters", "args": [{"type": "any"}, {"type": "any"}], "returns": "$1"} +{"name": "subtractMonths", "args": [{"type": "any"}, {"type": "any"}], "returns": "$1"} +{"name": "subtractWeeks", "args": [{"type": "any"}, {"type": "any"}], "returns": "$1"} +{"name": "subtractDays", "args": [{"type": "any"}, {"type": "any"}], "returns": "$1"} +{"name": "subtractHours", "args": [{"type": "any"}, {"type": "any"}], "returns": "$1"} +{"name": "subtractMinutes", "args": [{"type": "any"}, {"type": "any"}], "returns": "$1"} +{"name": "subtractSeconds", "args": [{"type": "any"}, {"type": "any"}], "returns": "$1"} +{"name": "addInterval", "args": [{"type": "any"}, {"type": "any"}], "returns": "$1"} +{"name": "subtractInterval", "args": [{"type": "any"}, {"type": "any"}], "returns": "$1"} +{"name": "dateDiff", "args": [{"type": "any"}, {"type": "any"}, {"type": "any"}], "returns": "Int64"} +{"name": "date_diff", "args": [{"type": "any"}, {"type": "any"}, {"type": "any"}], "returns": "Int64"} +{"name": "timestampDiff", "args": [{"type": "any"}, {"type": "any"}, {"type": "any"}], "returns": "Int64"} +{"name": "timestamp_diff", "args": [{"type": "any"}, {"type": "any"}, {"type": "any"}], "returns": "Int64"} +{"name": "age", "args": [{"type": "any"}, {"type": "any"}, {"type": "any"}], "returns": "Int64"} +{"name": "dateAdd", "args": [{"type": "any"}, {"type": "any"}, {"type": "any"}], "returns": "$3"} +{"name": "date_add", "args": [{"type": "any"}, {"type": "any"}, {"type": "any"}], "returns": "$3"} +{"name": "dateSub", "args": [{"type": "any"}, {"type": "any"}, {"type": "any"}], "returns": "$3"} +{"name": "date_sub", "args": [{"type": "any"}, {"type": "any"}, {"type": "any"}], "returns": "$3"} +{"name": "timestampAdd", "args": [{"type": "any"}, {"type": "any"}, {"type": "any"}], "returns": "$3"} +{"name": "timestampSub", "args": [{"type": "any"}, {"type": "any"}, {"type": "any"}], "returns": "$3"} +{"name": "dateTrunc", "args": [{"type": "any"}, {"type": "any"}, {"type": "any"}], "returns": "$3"} +{"name": "date_trunc", "args": [{"type": "any"}, {"type": "any"}, {"type": "any"}], "returns": "$3"} +{"name": "dateTrunc", "args": [{"type": "any"}, {"type": "any"}], "returns": "$2"} +{"name": "date_trunc", "args": [{"type": "any"}, {"type": "any"}], "returns": "$2"} +{"name": "toRelativeDayNum", "args": [{"type": "any"}], "returns": "UInt32"} +{"name": "toRelativeHourNum", "args": [{"type": "any"}], "returns": "UInt32"} +{"name": "toRelativeMinuteNum", "args": [{"type": "any"}], "returns": "UInt32"} +{"name": "toRelativeSecondNum", "args": [{"type": "any"}], "returns": "UInt32"} +{"name": "toRelativeMonthNum", "args": [{"type": "any"}], "returns": "UInt32"} +{"name": "toRelativeWeekNum", "args": [{"type": "any"}], "returns": "UInt32"} +{"name": "toTimeZone", "args": [{"type": "any"}, {"type": "any"}], "returns": "$1"} +{"name": "toTimezone", "args": [{"type": "any"}, {"type": "any"}], "returns": "$1"} +{"name": "parseDateTimeBestEffort", "args": [{"type": "any"}], "returns": "DateTime"} +{"name": "parseDateTimeBestEffortOrNull", "args": [{"type": "any"}], "returns": "DateTime", "nullable": true} +{"name": "parseDateTimeBestEffortOrZero", "args": [{"type": "any"}], "returns": "DateTime"} +{"name": "parseDateTime", "args": [{"type": "any"}, {"type": "any"}], "returns": "DateTime"} +{"name": "parseDateTime64BestEffort", "args": [{"type": "any"}], "returns": "DateTime64"} +{"name": "makeDate", "args": [{"type": "any"}, {"type": "any"}, {"type": "any"}], "returns": "Date"} +{"name": "makeDateTime", "args": [{"type": "any"}, {"type": "any"}, {"type": "any"}, {"type": "any"}, {"type": "any"}, {"type": "any"}], "returns": "DateTime"} +{"name": "arrayConcat", "args": [{"type": "any"}], "returns": "$1"} +{"name": "arrayConcat", "args": [{"type": "any"}, {"type": "any"}], "returns": "$1"} +{"name": "arrayConcat", "args": [{"type": "any"}, {"type": "any"}, {"type": "any"}], "returns": "$1"} +{"name": "arrayDistinct", "args": [{"type": "any"}], "returns": "$1"} +{"name": "arrayDistinct", "args": [{"type": "any"}, {"type": "any"}], "returns": "$1"} +{"name": "arrayDistinct", "args": [{"type": "any"}, {"type": "any"}, {"type": "any"}], "returns": "$1"} +{"name": "arraySort", "args": [{"type": "any"}], "returns": "$1"} +{"name": "arraySort", "args": [{"type": "any"}, {"type": "any"}], "returns": "$1"} +{"name": "arraySort", "args": [{"type": "any"}, {"type": "any"}, {"type": "any"}], "returns": "$1"} +{"name": "arrayReverseSort", "args": [{"type": "any"}], "returns": "$1"} +{"name": "arrayReverseSort", "args": [{"type": "any"}, {"type": "any"}], "returns": "$1"} +{"name": "arrayReverseSort", "args": [{"type": "any"}, {"type": "any"}, {"type": "any"}], "returns": "$1"} +{"name": "arrayReverse", "args": [{"type": "any"}], "returns": "$1"} +{"name": "arrayReverse", "args": [{"type": "any"}, {"type": "any"}], "returns": "$1"} +{"name": "arrayReverse", "args": [{"type": "any"}, {"type": "any"}, {"type": "any"}], "returns": "$1"} +{"name": "arraySlice", "args": [{"type": "any"}], "returns": "$1"} +{"name": "arraySlice", "args": [{"type": "any"}, {"type": "any"}], "returns": "$1"} +{"name": "arraySlice", "args": [{"type": "any"}, {"type": "any"}, {"type": "any"}], "returns": "$1"} +{"name": "arrayResize", "args": [{"type": "any"}], "returns": "$1"} +{"name": "arrayResize", "args": [{"type": "any"}, {"type": "any"}], "returns": "$1"} +{"name": "arrayResize", "args": [{"type": "any"}, {"type": "any"}, {"type": "any"}], "returns": "$1"} +{"name": "arrayPushBack", "args": [{"type": "any"}], "returns": "$1"} +{"name": "arrayPushBack", "args": [{"type": "any"}, {"type": "any"}], "returns": "$1"} +{"name": "arrayPushBack", "args": [{"type": "any"}, {"type": "any"}, {"type": "any"}], "returns": "$1"} +{"name": "arrayPushFront", "args": [{"type": "any"}], "returns": "$1"} +{"name": "arrayPushFront", "args": [{"type": "any"}, {"type": "any"}], "returns": "$1"} +{"name": "arrayPushFront", "args": [{"type": "any"}, {"type": "any"}, {"type": "any"}], "returns": "$1"} +{"name": "arrayPopBack", "args": [{"type": "any"}], "returns": "$1"} +{"name": "arrayPopBack", "args": [{"type": "any"}, {"type": "any"}], "returns": "$1"} +{"name": "arrayPopBack", "args": [{"type": "any"}, {"type": "any"}, {"type": "any"}], "returns": "$1"} +{"name": "arrayPopFront", "args": [{"type": "any"}], "returns": "$1"} +{"name": "arrayPopFront", "args": [{"type": "any"}, {"type": "any"}], "returns": "$1"} +{"name": "arrayPopFront", "args": [{"type": "any"}, {"type": "any"}, {"type": "any"}], "returns": "$1"} +{"name": "arrayFlatten", "args": [{"type": "any"}], "returns": "$1"} +{"name": "arrayFlatten", "args": [{"type": "any"}, {"type": "any"}], "returns": "$1"} +{"name": "arrayFlatten", "args": [{"type": "any"}, {"type": "any"}, {"type": "any"}], "returns": "$1"} +{"name": "arrayCompact", "args": [{"type": "any"}], "returns": "$1"} +{"name": "arrayCompact", "args": [{"type": "any"}, {"type": "any"}], "returns": "$1"} +{"name": "arrayCompact", "args": [{"type": "any"}, {"type": "any"}, {"type": "any"}], "returns": "$1"} +{"name": "arrayIntersect", "args": [{"type": "any"}], "returns": "$1"} +{"name": "arrayIntersect", "args": [{"type": "any"}, {"type": "any"}], "returns": "$1"} +{"name": "arrayIntersect", "args": [{"type": "any"}, {"type": "any"}, {"type": "any"}], "returns": "$1"} +{"name": "arrayUnion", "args": [{"type": "any"}], "returns": "$1"} +{"name": "arrayUnion", "args": [{"type": "any"}, {"type": "any"}], "returns": "$1"} +{"name": "arrayUnion", "args": [{"type": "any"}, {"type": "any"}, {"type": "any"}], "returns": "$1"} +{"name": "arrayShuffle", "args": [{"type": "any"}], "returns": "$1"} +{"name": "arrayShuffle", "args": [{"type": "any"}, {"type": "any"}], "returns": "$1"} +{"name": "arrayShuffle", "args": [{"type": "any"}, {"type": "any"}, {"type": "any"}], "returns": "$1"} +{"name": "arrayCount", "args": [{"type": "any"}], "returns": "UInt32"} +{"name": "arrayCount", "args": [{"type": "any"}, {"type": "any"}], "returns": "UInt32"} +{"name": "arrayUniq", "args": [{"type": "any"}], "returns": "UInt32"} +{"name": "arrayExists", "args": [{"type": "any"}, {"type": "any"}], "returns": "UInt8"} +{"name": "arrayAll", "args": [{"type": "any"}, {"type": "any"}], "returns": "UInt8"} +{"name": "hasSubstr", "args": [{"type": "any"}, {"type": "any"}], "returns": "UInt8"} +{"name": "arrayEnumerate", "args": [{"type": "any"}], "returns": "Array(UInt32)"} +{"name": "arrayEnumerateUniq", "args": [{"type": "any"}], "returns": "Array(UInt32)"} +{"name": "range", "args": [{"type": "any"}], "returns": "Array(UInt64)"} +{"name": "range", "args": [{"type": "any"}, {"type": "any"}], "returns": "Array(UInt64)"} +{"name": "range", "args": [{"type": "any"}, {"type": "any"}, {"type": "any"}], "returns": "Array(UInt64)"} +{"name": "mapKeys", "args": [{"type": "any"}], "returns": "$1"} +{"name": "mapValues", "args": [{"type": "any"}], "returns": "$1"} +{"name": "length", "args": [{"type": "any"}], "returns": "UInt64"} +{"name": "JSONExtractInt", "args": [{"type": "any"}, {"type": "any"}], "returns": "Int64"} +{"name": "JSONExtractInt64", "args": [{"type": "any"}, {"type": "any"}], "returns": "Int64"} +{"name": "JSONExtractUInt", "args": [{"type": "any"}, {"type": "any"}], "returns": "UInt64"} +{"name": "JSONExtractFloat", "args": [{"type": "any"}, {"type": "any"}], "returns": "Float64"} +{"name": "JSONExtractBool", "args": [{"type": "any"}, {"type": "any"}], "returns": "UInt8"} +{"name": "JSONHas", "args": [{"type": "any"}, {"type": "any"}], "returns": "UInt8"} +{"name": "JSONLength", "args": [{"type": "any"}, {"type": "any"}], "returns": "UInt64"} +{"name": "JSONType", "args": [{"type": "any"}, {"type": "any"}], "returns": "Enum8"} +{"name": "isValidJSON", "args": [{"type": "any"}], "returns": "UInt8"} +{"name": "JSONArrayLength", "args": [{"type": "any"}], "returns": "UInt64", "nullable": true} +{"name": "simpleJSONExtractInt", "args": [{"type": "any"}, {"type": "any"}], "returns": "Int64"} +{"name": "simpleJSONExtractUInt", "args": [{"type": "any"}, {"type": "any"}], "returns": "UInt64"} +{"name": "simpleJSONExtractFloat", "args": [{"type": "any"}, {"type": "any"}], "returns": "Float64"} +{"name": "simpleJSONExtractBool", "args": [{"type": "any"}, {"type": "any"}], "returns": "UInt8"} +{"name": "simpleJSONHas", "args": [{"type": "any"}, {"type": "any"}], "returns": "UInt8"} +{"name": "visitParamExtractInt", "args": [{"type": "any"}, {"type": "any"}], "returns": "Int64"} +{"name": "visitParamExtractUInt", "args": [{"type": "any"}, {"type": "any"}], "returns": "UInt64"} +{"name": "visitParamExtractFloat", "args": [{"type": "any"}, {"type": "any"}], "returns": "Float64"} +{"name": "visitParamExtractBool", "args": [{"type": "any"}, {"type": "any"}], "returns": "UInt8"} +{"name": "visitParamHas", "args": [{"type": "any"}, {"type": "any"}], "returns": "UInt8"} +{"name": "IPv4StringToNum", "args": [{"type": "any"}], "returns": "UInt32"} +{"name": "IPv4StringToNumOrNull", "args": [{"type": "any"}], "returns": "UInt32", "nullable": true} +{"name": "IPv6StringToNum", "args": [{"type": "any"}], "returns": "FixedString(16)"} +{"name": "isIPv4String", "args": [{"type": "any"}], "returns": "UInt8"} +{"name": "isIPv6String", "args": [{"type": "any"}], "returns": "UInt8"} +{"name": "sleep", "args": [{"type": "any"}], "returns": "UInt8"} +{"name": "blockSize", "returns": "UInt64", "never_null": true} +{"name": "rowNumberInAllBlocks", "returns": "UInt64", "never_null": true} +{"name": "materialize", "args": [{"type": "any"}], "returns": "$1"} +{"name": "ignore", "args": [{"type": "any"}], "returns": "UInt8", "never_null": true} +{"name": "ifNotFinite", "args": [{"type": "any"}, {"type": "any"}], "returns": "$1"} +{"name": "throwIf", "args": [{"type": "any"}], "returns": "UInt8"} +{"name": "transform", "args": [{"type": "any"}, {"type": "any"}, {"type": "any"}, {"type": "any"}], "returns": "$4"} +{"name": "bar", "args": [{"type": "any"}, {"type": "any"}, {"type": "any"}], "returns": "String"} +{"name": "bar", "args": [{"type": "any"}, {"type": "any"}, {"type": "any"}, {"type": "any"}], "returns": "String"} +{"name": "dictGetString", "args": [{"type": "any"}, {"type": "any"}, {"type": "any"}], "returns": "String"} +{"name": "dictGetUInt64", "args": [{"type": "any"}, {"type": "any"}, {"type": "any"}], "returns": "UInt64"} +{"name": "dictGetInt64", "args": [{"type": "any"}, {"type": "any"}, {"type": "any"}], "returns": "Int64"} +{"name": "dictGetFloat64", "args": [{"type": "any"}, {"type": "any"}, {"type": "any"}], "returns": "Float64"} +{"name": "dictGetDate", "args": [{"type": "any"}, {"type": "any"}, {"type": "any"}], "returns": "Date"} +{"name": "dictGetDateTime", "args": [{"type": "any"}, {"type": "any"}, {"type": "any"}], "returns": "DateTime"} +{"name": "dictHas", "args": [{"type": "any"}, {"type": "any"}], "returns": "UInt8"} +{"name": "tuple", "args": [{"type": "any"}], "returns": "Tuple"} +{"name": "tupleElement", "args": [{"type": "any"}, {"type": "any"}], "returns": "any"} +{"name": "map", "args": [{"type": "any"}, {"type": "any"}], "returns": "Map"} +{"name": "multiIf", "args": [{"type": "any"}, {"type": "any"}, {"type": "any"}], "returns": "$2"} +{"name": "multiIf", "args": [{"type": "any"}, {"type": "any"}, {"type": "any"}, {"type": "any"}, {"type": "any"}], "returns": "$2"} +{"name": "multiIf", "args": [{"type": "any"}, {"type": "any"}, {"type": "any"}, {"type": "any"}, {"type": "any"}, {"type": "any"}, {"type": "any"}], "returns": "$2"} diff --git a/internal/sql/preprocess/dialect.go b/internal/sql/preprocess/dialect.go index 29340bb1f4..4d8d4ae85b 100644 --- a/internal/sql/preprocess/dialect.go +++ b/internal/sql/preprocess/dialect.go @@ -88,11 +88,19 @@ var dialects = map[config.Engine]Dialect{ Backtick: true, Backslash: true, }, + // ClickHouse binds with an unnumbered ?, like MySQL, and its identifiers + // keep their case. + config.EngineClickHouse: { + Style: StyleQuestion, + Question: true, + Backtick: true, + Backslash: true, + }, } // DialectFor returns the lexical rules for an engine, and whether the engine is -// preprocessed at all. GoogleSQL and ClickHouse are not: they handle their own -// parameter syntax, so their queries reach the parser unchanged. +// preprocessed at all. GoogleSQL is not: it handles its own parameter syntax, +// so its queries reach the parser unchanged. func DialectFor(engine config.Engine) (Dialect, bool) { d, ok := dialects[engine] return d, ok diff --git a/internal/sql/preprocess/preprocess.go b/internal/sql/preprocess/preprocess.go index 76501cb60a..ad25bc9b2f 100644 --- a/internal/sql/preprocess/preprocess.go +++ b/internal/sql/preprocess/preprocess.go @@ -166,10 +166,10 @@ type occurrence struct { // File rewrites every sqlc construct in src to native SQL for the given engine. // -// Engines that are not preprocessed — GoogleSQL and ClickHouse, which handle -// their own parameter syntax — get the source back unchanged, with an empty -// side table. sqlc.arg() and friends are not rewritten for them, so they reach -// the parser as the function calls they look like. +// An engine that is not preprocessed — GoogleSQL, which handles its own +// parameter syntax — gets the source back unchanged, with an empty side +// table. sqlc.arg() and friends are not rewritten for it, so they reach the +// parser as the function calls they look like. func File(engine config.Engine, src string) *Result { d, ok := DialectFor(engine) if !ok { diff --git a/internal/testcheck/README.md b/internal/testcheck/README.md index b56553bd55..77d707aa8f 100644 --- a/internal/testcheck/README.md +++ b/internal/testcheck/README.md @@ -12,9 +12,9 @@ so it never shares code with the analysis it checks. Run it from this directory: ```bash -go run . install clickhouse # download the pinned clickhouse binary once -go run . check # check every engine whose database is available -go run . check clickhouse # check one engine +go run ./cmd/testcheck install clickhouse # download the pinned clickhouse binary once +go run ./cmd/testcheck check # check every engine whose database is available +go run ./cmd/testcheck check clickhouse # check one engine go test ./... # the same checks as tests; engines without a database skip ``` diff --git a/internal/testcheck/clickhouse/analyze.go b/internal/testcheck/clickhouse/analyze.go index 36b08a0a97..69795aacf8 100644 --- a/internal/testcheck/clickhouse/analyze.go +++ b/internal/testcheck/clickhouse/analyze.go @@ -227,7 +227,14 @@ func (t *queryTree) describe(n *treeNode) analyzedColumn { case "FUNCTION": return column(n.attrs["function_name"], n.attrs["result_type"]) case "CONSTANT": - return column("", n.attrs["constant_value_type"]) + // A constant folded from a function call, such as toDate(now()), + // still names the function, which is what a placeholder compared + // with it is named after. + name := "" + if fn := firstNode(n.section("EXPRESSION").childrenOrNil()); fn != nil && fn.kind == "FUNCTION" { + name = fn.attrs["function_name"] + } + return column(name, n.attrs["constant_value_type"]) } return analyzedColumn{} } diff --git a/internal/testcheck/clickhouse/install.go b/internal/testcheck/clickhouse/install.go index 0af8e2e436..d76773c9e9 100644 --- a/internal/testcheck/clickhouse/install.go +++ b/internal/testcheck/clickhouse/install.go @@ -114,7 +114,7 @@ func Locate() (string, error) { return "", err } if _, err := os.Stat(path); err != nil { - return "", fmt.Errorf("clickhouse %s is not installed: run `go run . install clickhouse` in internal/testcheck, or set CLICKHOUSE to a clickhouse binary", DefaultVersion) + return "", fmt.Errorf("clickhouse %s is not installed: run `go run ./cmd/testcheck install clickhouse` in internal/testcheck, or set CLICKHOUSE to a clickhouse binary", DefaultVersion) } return path, nil } diff --git a/internal/testcheck/clickhouse/queries.go b/internal/testcheck/clickhouse/queries.go index 40f2ecdc36..69e3b1ac65 100644 --- a/internal/testcheck/clickhouse/queries.go +++ b/internal/testcheck/clickhouse/queries.go @@ -83,30 +83,17 @@ var namedArgRe = regexp.MustCompile(`^sqlc\.(n?arg)\(\s*'?([A-Za-z_][A-Za-z0-9_] // bindPlaceholders rewrites sqlc's parameter syntax (?, sqlc.arg(name), // sqlc.narg(name)) into constants ClickHouse can analyze, skipping string -// literals, quoted identifiers and comments. Placeholders are numbered the -// way sqlc numbers them: positionally, with a repeated name sharing a number. +// literals, quoted identifiers and comments. ClickHouse binds every ? +// positionally, so each placeholder is its own parameter even when a name +// repeats, which is how sqlc numbers them too. func bindPlaceholders(sql string) (string, []placeholder) { var ( out strings.Builder phs []placeholder - numbers = map[string]int{} lastWord string i = 0 ) - number := func(name string) int { - if name != "" { - if n, ok := numbers[name]; ok { - return n - } - } - n := len(numbers) + 1 - if name == "" { - // Positional placeholders never repeat; give them a private key. - name = fmt.Sprintf("?%d", n) - } - numbers[name] = n - return n - } + number := func(string) int { return len(phs) + 1 } for i < len(sql) { c := sql[i] switch { diff --git a/internal/testcheck/main.go b/internal/testcheck/cmd/testcheck/main.go similarity index 94% rename from internal/testcheck/main.go rename to internal/testcheck/cmd/testcheck/main.go index 3d77d4f508..264ae35a41 100644 --- a/internal/testcheck/main.go +++ b/internal/testcheck/cmd/testcheck/main.go @@ -5,8 +5,8 @@ // // Usage, from this directory: // -// go run . install clickhouse # download the pinned clickhouse binary -// go run . check [engine] # check every case, or one engine's +// go run ./cmd/testcheck install clickhouse # download the pinned clickhouse binary +// go run ./cmd/testcheck check [engine] # check every case, or one engine's // // `go test ./...` runs the same checks as tests, skipping engines whose // database is not available. From 25999661ebbddf02e4d7185c884d6fb640a0e2f4 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 3 Sep 2026 17:56:47 +0000 Subject: [PATCH 10/10] Move the analyze checks into goldeneye goldeneye already generates and checks the dialect seeds and reserved a place for the analysis checks, so the testcheck module folds into it: the case loader becomes goldeneye/endtoend, reusing goldeneye's diff, the ClickHouse analysis joins goldeneye's clickhouse package next to the dialect generator it shares a binary with, and `check` verifies both the committed dialect and the analyze cases. `go test ./...` in internal/goldeneye runs the same checks. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01Y6XkyWnx7iJFEb8q3AnYps --- CLAUDE.md | 16 +- internal/goldeneye/README.md | 27 ++- .../clickhouse/analyze.go | 0 .../clickhouse/check.go | 15 +- internal/goldeneye/clickhouse/clickhouse.go | 9 + .../goldeneye/clickhouse/clickhouse_test.go | 29 +++ .../clickhouse/queries.go | 0 .../clickhouse/tree.go | 0 .../clickhouse/types.go | 0 internal/goldeneye/cmd/goldeneye/main.go | 52 ++++- .../endtoend/endtoend.go | 52 +---- internal/testcheck/README.md | 40 ---- .../testcheck/clickhouse/clickhouse_test.go | 35 --- internal/testcheck/clickhouse/install.go | 221 ------------------ internal/testcheck/clickhouse/local.go | 82 ------- internal/testcheck/cmd/testcheck/main.go | 116 --------- internal/testcheck/go.mod | 3 - 17 files changed, 122 insertions(+), 575 deletions(-) rename internal/{testcheck => goldeneye}/clickhouse/analyze.go (100%) rename internal/{testcheck => goldeneye}/clickhouse/check.go (62%) rename internal/{testcheck => goldeneye}/clickhouse/queries.go (100%) rename internal/{testcheck => goldeneye}/clickhouse/tree.go (100%) rename internal/{testcheck => goldeneye}/clickhouse/types.go (100%) rename internal/{testcheck => goldeneye}/endtoend/endtoend.go (75%) delete mode 100644 internal/testcheck/README.md delete mode 100644 internal/testcheck/clickhouse/clickhouse_test.go delete mode 100644 internal/testcheck/clickhouse/install.go delete mode 100644 internal/testcheck/clickhouse/local.go delete mode 100644 internal/testcheck/cmd/testcheck/main.go delete mode 100644 internal/testcheck/go.mod diff --git a/CLAUDE.md b/CLAUDE.md index cbfa9892a2..f21fe118e5 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -147,8 +147,11 @@ the run early. Run a subset to get past one (`-run 'TestReplay/core/^select'`). The dialect seeds under `/internal/engine//dialect/` are generated from a live database by `/internal/goldeneye`, a nested module, and its tests -verify the committed files against one byte for byte. Engines whose database -is not available skip. +verify the committed files against one byte for byte. The same module checks +the `analyze_*` cases under `/internal/endtoend/testdata/` against what the +database itself reports for them, so a `fixture.sql` next to a case's schema +gives the queries rows to run against. Engines whose database is not +available skip. ```bash cd internal/goldeneye @@ -239,16 +242,15 @@ MYSQL_SERVER_URI="root:mysecretpassword@tcp(127.0.0.1:3306)/mysql?multiStatement JSONL read by `/internal/core/seed`; the generated parts come from `/internal/goldeneye` - `/internal/goldeneye/` - Nested module that generates the dialect seeds - under `/internal/engine//dialect/` from a live database and checks - the committed ones against it, one package per engine; see its README + under `/internal/engine//dialect/` from a live database, checks + the committed ones against it, and checks the analyze cases under + `/internal/endtoend/testdata/` against what the database reports, one + package per engine; see its README - `/internal/core/` - The analysis core: catalog, analyzer and dialect seeds - `/internal/compiler/` - Query compilation logic - `/internal/codegen/` - Code generation for different languages - `/internal/config/` - Configuration file parsing - `/internal/endtoend/` - End-to-end tests -- `/internal/testcheck/` - Nested module that verifies the analyze cases under - `/internal/endtoend/testdata/` against a real database, one package per - engine; see its README - `/internal/sqltest/` - Test database setup (Docker, native, local detection) - `/examples/` - Example projects for testing diff --git a/internal/goldeneye/README.md b/internal/goldeneye/README.md index 3bd0a35787..16e4a2c0f7 100644 --- a/internal/goldeneye/README.md +++ b/internal/goldeneye/README.md @@ -60,10 +60,29 @@ the hand-written files alone, and the checks do not look at them. - `dialect/` — the record types the files are made of, mirrored from `internal/core/seed`, and the helpers that write a generated set of files into an engine directory or diff it against what is committed. +- `endtoend/` — finds the analyze cases and compares an engine's answer with + a case's committed output. - `postgresql/`, `duckdb/`, `clickhouse/` — one package per engine, each - exposing `Locate`, `Version` and `Generate`, and a test that runs the check. + exposing `Locate`, `Version` and `Generate`, `Analyze` where the engine has + an analysis check, and tests that run the checks. - `cmd/goldeneye/` — the command. -The analysis checks — verifying the `analyze_*` cases under -`internal/endtoend/testdata` against what each database itself reports — are -meant to live here too, alongside the dialect checks. +## Analysis checks + +`check` also verifies the `analyze_*` cases under `internal/endtoend/testdata` +against what the database itself reports. A case is an +`analyze_/` directory whose `exec.json` runs the analyze +command; `endtoend/` finds them. The engine package loads the case's +`schema.sql` and optional `fixture.sql` into the database, runs `query.sql` +there, prints what the database reports in the JSON shape `sqlc analyze` +prints, and compares it with the committed `output.json` byte for byte. A +difference means sqlc's analysis disagrees with the database. A case that +asks for `--ast` is skipped, since only sqlc can print that. + +- **`clickhouse`** runs each case in an ephemeral `clickhouse local` process. + Column types come from the executed query's result header, provenance from + `EXPLAIN QUERY TREE`, and parameters from sentinel constants substituted for + `?`, `sqlc.arg()` and `sqlc.narg()`, since ClickHouse itself never sees a + placeholder; `INSERT ... VALUES` parameters map onto `DESCRIBE TABLE`. + +The other engines have no analysis check yet. diff --git a/internal/testcheck/clickhouse/analyze.go b/internal/goldeneye/clickhouse/analyze.go similarity index 100% rename from internal/testcheck/clickhouse/analyze.go rename to internal/goldeneye/clickhouse/analyze.go diff --git a/internal/testcheck/clickhouse/check.go b/internal/goldeneye/clickhouse/check.go similarity index 62% rename from internal/testcheck/clickhouse/check.go rename to internal/goldeneye/clickhouse/check.go index 983bd9bc8f..1ad8d9cb93 100644 --- a/internal/testcheck/clickhouse/check.go +++ b/internal/goldeneye/clickhouse/check.go @@ -1,13 +1,3 @@ -// Package clickhouse verifies the ClickHouse analyze cases under -// internal/endtoend/testdata against what ClickHouse itself reports. -// -// Each case's schema and fixture are loaded into an ephemeral -// `clickhouse local` process and its queries are run there. Result column -// types come from the executed query's result header, provenance from -// EXPLAIN QUERY TREE, and parameters from sentinel constants substituted -// for the placeholders, since ClickHouse itself never sees a ?. The answer -// is printed in the JSON shape sqlc analyze prints and compared with the -// case's committed output.json byte for byte. package clickhouse import ( @@ -17,12 +7,9 @@ import ( "fmt" "os" - "github.com/sqlc-dev/sqlc/internal/testcheck/endtoend" + "github.com/sqlc-dev/sqlc/internal/goldeneye/endtoend" ) -// Engine is the name of this engine's directory under each analyze case. -const Engine = "clickhouse" - // Analyze runs a case's queries through the clickhouse binary and returns // the analysis in the JSON shape sqlc analyze prints. func Analyze(ctx context.Context, binary string, c endtoend.Case) ([]byte, error) { diff --git a/internal/goldeneye/clickhouse/clickhouse.go b/internal/goldeneye/clickhouse/clickhouse.go index 5eb10d282c..d65539eec0 100644 --- a/internal/goldeneye/clickhouse/clickhouse.go +++ b/internal/goldeneye/clickhouse/clickhouse.go @@ -9,6 +9,15 @@ // system.functions carries no signatures — so functions.jsonl is written by // hand and is not this package's business. // +// The package also verifies the ClickHouse analyze cases under +// internal/endtoend/testdata against the same binary: each case's schema and +// fixture are loaded into a `clickhouse local` process and its queries run +// there. Result column types come from the executed query's result header, +// provenance from EXPLAIN QUERY TREE, and parameters from sentinel constants +// substituted for the placeholders, since ClickHouse itself never sees a ?. +// The answer is printed in the JSON shape sqlc analyze prints and compared +// with the case's committed output.json byte for byte. +// // The binary is downloaded once per pinned version by Install, or supplied // through the CLICKHOUSE environment variable. package clickhouse diff --git a/internal/goldeneye/clickhouse/clickhouse_test.go b/internal/goldeneye/clickhouse/clickhouse_test.go index 0e074eaf21..f4d58b8346 100644 --- a/internal/goldeneye/clickhouse/clickhouse_test.go +++ b/internal/goldeneye/clickhouse/clickhouse_test.go @@ -5,6 +5,7 @@ import ( "testing" "github.com/sqlc-dev/sqlc/internal/goldeneye/dialect" + "github.com/sqlc-dev/sqlc/internal/goldeneye/endtoend" ) // TestDialect verifies the committed ClickHouse dialect against what the @@ -35,3 +36,31 @@ func TestDialect(t *testing.T) { t.Errorf("%s does not match what %s reports:\n%s", dir, version, report) } } + +// TestAnalyzeCases verifies every ClickHouse analyze case under +// internal/endtoend/testdata against what ClickHouse reports. It skips +// unless the binary is installed. +func TestAnalyzeCases(t *testing.T) { + binary, err := Locate() + if err != nil { + t.Skip(err) + } + cases, err := endtoend.Cases(Engine) + if err != nil { + t.Fatal(err) + } + if len(cases) == 0 { + t.Fatal("no clickhouse analyze cases found") + } + for _, c := range cases { + t.Run(c.Name, func(t *testing.T) { + diff, err := Check(context.Background(), binary, c) + if err != nil { + t.Fatal(err) + } + if diff != "" { + t.Errorf("%s does not match what ClickHouse reports (-committed +clickhouse):\n%s", c.Output, diff) + } + }) + } +} diff --git a/internal/testcheck/clickhouse/queries.go b/internal/goldeneye/clickhouse/queries.go similarity index 100% rename from internal/testcheck/clickhouse/queries.go rename to internal/goldeneye/clickhouse/queries.go diff --git a/internal/testcheck/clickhouse/tree.go b/internal/goldeneye/clickhouse/tree.go similarity index 100% rename from internal/testcheck/clickhouse/tree.go rename to internal/goldeneye/clickhouse/tree.go diff --git a/internal/testcheck/clickhouse/types.go b/internal/goldeneye/clickhouse/types.go similarity index 100% rename from internal/testcheck/clickhouse/types.go rename to internal/goldeneye/clickhouse/types.go diff --git a/internal/goldeneye/cmd/goldeneye/main.go b/internal/goldeneye/cmd/goldeneye/main.go index c3af78df38..bda6e14b76 100644 --- a/internal/goldeneye/cmd/goldeneye/main.go +++ b/internal/goldeneye/cmd/goldeneye/main.go @@ -1,12 +1,13 @@ // Command goldeneye generates the dialect seeds under -// internal/engine//dialect from a live database, and checks the -// committed ones against it. +// internal/engine//dialect from a live database, checks the +// committed ones against it, and checks the analyze cases under +// internal/endtoend/testdata against what the database itself reports. // // Usage, from internal/goldeneye: // // go run ./cmd/goldeneye install clickhouse # download the pinned clickhouse binary // go run ./cmd/goldeneye generate [engine] # rewrite the generated files from the database -// go run ./cmd/goldeneye check [engine] # compare the committed files with the database +// go run ./cmd/goldeneye check [engine] # compare the committed files and analyze cases with the database // // Without an engine, generate and check cover every engine whose database // is available and say which ones they skipped. `go test ./...` runs the @@ -21,10 +22,12 @@ import ( "io" "os" "runtime" + "strings" "github.com/sqlc-dev/sqlc/internal/goldeneye/clickhouse" "github.com/sqlc-dev/sqlc/internal/goldeneye/dialect" "github.com/sqlc-dev/sqlc/internal/goldeneye/duckdb" + "github.com/sqlc-dev/sqlc/internal/goldeneye/endtoend" "github.com/sqlc-dev/sqlc/internal/goldeneye/postgresql" ) @@ -41,7 +44,7 @@ const usage = `usage: goldeneye generate [engine] rewrite the generated dialect files from the database, for every available engine or one goldeneye check [engine] - compare the committed dialect files with the database, for every available engine or one + compare the committed dialect files and analyze cases with the database, for every available engine or one engines: clickhouse, duckdb, postgresql` @@ -55,12 +58,15 @@ type engine struct { version func(context.Context, string) (string, error) // generate reads the dialect from the database. generate func(context.Context, string) (dialect.Files, error) + // analyze asks the database what it reports for an analyze case, in + // the shape sqlc analyze prints. Nil for an engine without one yet. + analyze func(context.Context, string, endtoend.Case) ([]byte, error) } var engines = []engine{ - {clickhouse.Engine, clickhouse.Locate, clickhouse.Version, clickhouse.Generate}, - {duckdb.Engine, duckdb.Locate, duckdb.Version, duckdb.Generate}, - {postgresql.Engine, postgresql.Locate, postgresql.Version, postgresql.Generate}, + {clickhouse.Engine, clickhouse.Locate, clickhouse.Version, clickhouse.Generate, clickhouse.Analyze}, + {duckdb.Engine, duckdb.Locate, duckdb.Version, duckdb.Generate, nil}, + {postgresql.Engine, postgresql.Locate, postgresql.Version, postgresql.Generate, nil}, } func run(ctx context.Context, args []string, stdout, stderr io.Writer) error { @@ -184,5 +190,37 @@ func check(ctx context.Context, e engine, handle string, stderr io.Writer) error return fmt.Errorf("%s does not match the database\n%s", dir, report) } fmt.Fprintf(stderr, "%s: ok, %d file(s) match\n", e.name, len(files)) + return checkAnalyzeCases(ctx, e, handle, stderr) +} + +// checkAnalyzeCases compares what the database reports for each of the +// engine's analyze cases with the case's committed output. +func checkAnalyzeCases(ctx context.Context, e engine, handle string, stderr io.Writer) error { + if e.analyze == nil { + return nil + } + cases, err := endtoend.Cases(e.name) + if err != nil { + return err + } + var report strings.Builder + for _, c := range cases { + got, err := e.analyze(ctx, handle, c) + if err != nil { + fmt.Fprintf(&report, "%s: %v\n", c.Name, err) + continue + } + diff, err := c.Compare(got) + if err != nil { + return err + } + if diff != "" { + fmt.Fprintf(&report, "%s (-committed +database)\n%s", c.Name, diff) + } + } + if report.Len() > 0 { + return fmt.Errorf("analyze cases do not match the database\n%s", report.String()) + } + fmt.Fprintf(stderr, "%s: ok, %d analyze case(s) match\n", e.name, len(cases)) return nil } diff --git a/internal/testcheck/endtoend/endtoend.go b/internal/goldeneye/endtoend/endtoend.go similarity index 75% rename from internal/testcheck/endtoend/endtoend.go rename to internal/goldeneye/endtoend/endtoend.go index 51d033e981..e6506f0cc0 100644 --- a/internal/testcheck/endtoend/endtoend.go +++ b/internal/goldeneye/endtoend/endtoend.go @@ -1,5 +1,7 @@ // Package endtoend finds the analyze cases under internal/endtoend/testdata -// and compares an engine's own answer with the output a case committed. +// and compares an engine's own answer with the output a case committed, +// the way the dialect package compares a generated dialect with the +// committed one. package endtoend import ( @@ -11,6 +13,8 @@ import ( "path/filepath" "runtime" "strings" + + "github.com/sqlc-dev/sqlc/internal/goldeneye/dialect" ) // Case is one analyze case: the files sqlc analyze ran with, the fixture @@ -132,49 +136,5 @@ func (c Case) Compare(got []byte) (string, error) { if bytes.Equal(want, got) { return "", nil } - return Diff(string(want), string(got)), nil -} - -// Diff is a line diff of two texts, marking lines only in want with "-" and -// lines only in got with "+". Outputs are small, so a plain longest common -// subsequence is fine. -func Diff(want, got string) string { - a := strings.Split(strings.TrimSuffix(want, "\n"), "\n") - b := strings.Split(strings.TrimSuffix(got, "\n"), "\n") - lcs := make([][]int, len(a)+1) - for i := range lcs { - lcs[i] = make([]int, len(b)+1) - } - for i := len(a) - 1; i >= 0; i-- { - for j := len(b) - 1; j >= 0; j-- { - if a[i] == b[j] { - lcs[i][j] = lcs[i+1][j+1] + 1 - } else { - lcs[i][j] = max(lcs[i+1][j], lcs[i][j+1]) - } - } - } - var out strings.Builder - i, j := 0, 0 - for i < len(a) && j < len(b) { - switch { - case a[i] == b[j]: - fmt.Fprintf(&out, " %s\n", a[i]) - i++ - j++ - case lcs[i+1][j] >= lcs[i][j+1]: - fmt.Fprintf(&out, "- %s\n", a[i]) - i++ - default: - fmt.Fprintf(&out, "+ %s\n", b[j]) - j++ - } - } - for ; i < len(a); i++ { - fmt.Fprintf(&out, "- %s\n", a[i]) - } - for ; j < len(b); j++ { - fmt.Fprintf(&out, "+ %s\n", b[j]) - } - return out.String() + return dialect.Diff(string(want), string(got)), nil } diff --git a/internal/testcheck/README.md b/internal/testcheck/README.md deleted file mode 100644 index 77d707aa8f..0000000000 --- a/internal/testcheck/README.md +++ /dev/null @@ -1,40 +0,0 @@ -# testcheck - -`testcheck` verifies the analyze cases under `internal/endtoend/testdata` -against a real database. It generates nothing. Each engine package reads a -case's `schema.sql`, `fixture.sql` and `query.sql`, asks the database what it -makes of the queries, prints the answer in the JSON shape `sqlc analyze` -prints, and compares it with the `output.json` the case committed, byte for -byte. A difference means sqlc's analysis disagrees with the database. - -It is a nested Go module with no dependencies beyond the standard library, -so it never shares code with the analysis it checks. Run it from this -directory: - -```bash -go run ./cmd/testcheck install clickhouse # download the pinned clickhouse binary once -go run ./cmd/testcheck check # check every engine whose database is available -go run ./cmd/testcheck check clickhouse # check one engine -go test ./... # the same checks as tests; engines without a database skip -``` - -## Cases - -A case is an `analyze_/` directory whose `exec.json` runs the -analyze command. `fixture.sql` is optional and is loaded after the schema, so -the queries run against real rows. A case that asks for `--ast` is skipped, -since only sqlc can print that. - -## Engines - -Each engine is its own package. - -- **`clickhouse`** needs no server. Each case runs in an ephemeral - `clickhouse local` process, downloaded once per pinned version by - `install` into the user cache directory, or supplied through the - `CLICKHOUSE` environment variable. The pinned version and the SHA-512 of - each platform's download live in `clickhouse/install.go`. Column types - come from the executed query's result header, provenance from - `EXPLAIN QUERY TREE`, and parameters from sentinel constants substituted - for `?`, `sqlc.arg()` and `sqlc.narg()`, since ClickHouse itself never - sees a placeholder. diff --git a/internal/testcheck/clickhouse/clickhouse_test.go b/internal/testcheck/clickhouse/clickhouse_test.go deleted file mode 100644 index f9181ed9dc..0000000000 --- a/internal/testcheck/clickhouse/clickhouse_test.go +++ /dev/null @@ -1,35 +0,0 @@ -package clickhouse - -import ( - "context" - "testing" - - "github.com/sqlc-dev/sqlc/internal/testcheck/endtoend" -) - -// TestEndToEnd verifies every ClickHouse analyze case against ClickHouse. -// It skips unless the clickhouse binary is installed. -func TestEndToEnd(t *testing.T) { - binary, err := Locate() - if err != nil { - t.Skip(err) - } - cases, err := endtoend.Cases(Engine) - if err != nil { - t.Fatal(err) - } - if len(cases) == 0 { - t.Fatal("no clickhouse analyze cases found") - } - for _, c := range cases { - t.Run(c.Name, func(t *testing.T) { - diff, err := Check(context.Background(), binary, c) - if err != nil { - t.Fatal(err) - } - if diff != "" { - t.Errorf("%s does not match what ClickHouse reports (-committed +clickhouse):\n%s", c.Output, diff) - } - }) - } -} diff --git a/internal/testcheck/clickhouse/install.go b/internal/testcheck/clickhouse/install.go deleted file mode 100644 index d76773c9e9..0000000000 --- a/internal/testcheck/clickhouse/install.go +++ /dev/null @@ -1,221 +0,0 @@ -package clickhouse - -import ( - "archive/tar" - "compress/gzip" - "context" - "crypto/sha512" - "encoding/hex" - "errors" - "fmt" - "io" - "net/http" - "os" - "path/filepath" - "strconv" - "strings" -) - -// DefaultVersion is the ClickHouse release the goldens are generated with. -// Bumping it is a deliberate change: the query tree format and type -// inference can shift between releases, so regenerate and review the goldens -// after changing it, and add the new release's assets to the table below. -const DefaultVersion = "25.8.2.29" - -// asset is one downloadable build of ClickHouse. Linux builds are published -// as clickhouse-common-static tarballs holding the binary at -// usr/bin/clickhouse; macOS builds are published as bare binaries. -type asset struct { - Version string - OS string - Arch string - Name string - SHA512 string -} - -// assets lists every build Install knows how to fetch, with the SHA-512 of -// the download. A version that is not in this table cannot be installed: -// verifying the download is the point of the table. -// -// The tarball checksums are the ones in the .sha512 files ClickHouse -// publishes next to them. ClickHouse publishes no checksum for the macOS -// binaries, so those were computed from the downloads. -var assets = []asset{ - {"25.8.2.29", "linux", "amd64", "clickhouse-common-static-25.8.2.29-amd64.tgz", "6ff0aa1ffac6e564970174422ecde0d645cdb96812247a6e544d39cad6d78a514265f90a2bc7b4bad49903cea96eddd16a415a45b2aeaf9164461be76331bdee"}, - {"25.8.2.29", "linux", "arm64", "clickhouse-common-static-25.8.2.29-arm64.tgz", "68204ca4d4e472790f808ee376251fae82e58066a31f35a40d15d442ce5988d697f18a1208d28b8bb8e2dfad4b20b7fcb5107e2178472abcd97251b8de7f058e"}, - {"25.8.2.29", "darwin", "amd64", "clickhouse-macos", "2805805ad2506e37a3e71b4ae9e797bdc010a9368dc28e99bcaaa2c70a72cfdd031c0fce8fc304248fad73211d28d645f75c6432cfae1e1e54d72d04e8626cd4"}, - {"25.8.2.29", "darwin", "arm64", "clickhouse-macos-aarch64", "4c9237e85c8d4e1aced2b339b32e086b4f23fa14f99754b056a7e33dda88c4fb5d52e09e29d20181ec5b580caadb00f36613802e71755ff34927535eb8babf79"}, -} - -// releaseTag returns the GitHub release tag for a version. ClickHouse tags -// its March and August releases as LTS and everything else as stable. -func releaseTag(version string) (string, error) { - parts := strings.Split(version, ".") - if len(parts) != 4 { - return "", fmt.Errorf("invalid ClickHouse version %q: want MAJOR.MINOR.PATCH.BUILD", version) - } - minor, err := strconv.Atoi(parts[1]) - if err != nil { - return "", fmt.Errorf("invalid ClickHouse version %q: %w", version, err) - } - suffix := "-stable" - if minor == 3 || minor == 8 { - suffix = "-lts" - } - return "v" + version + suffix, nil -} - -// releaseAsset finds the build for a platform in the table. -func releaseAsset(version, goos, goarch string) (asset, error) { - for _, a := range assets { - if a.Version == version && a.OS == goos && a.Arch == goarch { - return a, nil - } - } - for _, a := range assets { - if a.Version == version { - return asset{}, fmt.Errorf("no ClickHouse %s build is listed for %s/%s", version, goos, goarch) - } - } - return asset{}, fmt.Errorf("ClickHouse %s is not in the asset table; add its downloads and checksums to install.go", version) -} - -// url is the asset's download address on GitHub. -func (a asset) url() (string, error) { - tag, err := releaseTag(a.Version) - if err != nil { - return "", err - } - return "https://github.com/ClickHouse/ClickHouse/releases/download/" + tag + "/" + a.Name, nil -} - -// tarball reports whether the download is an archive rather than the binary. -func (a asset) tarball() bool { - return strings.HasSuffix(a.Name, ".tgz") -} - -// cachedBinary is where Install puts the binary for a version. -func cachedBinary(version string) (string, error) { - dir, err := os.UserCacheDir() - if err != nil { - return "", err - } - return filepath.Join(dir, "sqlc-clickhouse", version, "clickhouse"), nil -} - -// Locate finds a clickhouse binary: the CLICKHOUSE environment variable wins, -// then the cached copy of DefaultVersion. -func Locate() (string, error) { - if path := os.Getenv("CLICKHOUSE"); path != "" { - return path, nil - } - path, err := cachedBinary(DefaultVersion) - if err != nil { - return "", err - } - if _, err := os.Stat(path); err != nil { - return "", fmt.Errorf("clickhouse %s is not installed: run `go run ./cmd/testcheck install clickhouse` in internal/testcheck, or set CLICKHOUSE to a clickhouse binary", DefaultVersion) - } - return path, nil -} - -// Install downloads the clickhouse binary for a version into the cache and -// returns its path. It is a no-op when the version is already cached. The -// download is checked against the table's SHA-512 before it is installed. -func Install(ctx context.Context, version, goos, goarch string, progress io.Writer) (string, error) { - dest, err := cachedBinary(version) - if err != nil { - return "", err - } - if _, err := os.Stat(dest); err == nil { - return dest, nil - } - a, err := releaseAsset(version, goos, goarch) - if err != nil { - return "", err - } - url, err := a.url() - if err != nil { - return "", err - } - if err := os.MkdirAll(filepath.Dir(dest), 0o755); err != nil { - return "", err - } - - fmt.Fprintf(progress, "downloading %s\n", url) - req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) - if err != nil { - return "", err - } - resp, err := http.DefaultClient.Do(req) - if err != nil { - return "", err - } - defer resp.Body.Close() - if resp.StatusCode != http.StatusOK { - return "", fmt.Errorf("downloading %s: %s", url, resp.Status) - } - - // Write next to the destination and rename so a partial or corrupt - // download never masquerades as an installed binary. - tmp, err := os.CreateTemp(filepath.Dir(dest), "clickhouse-*.partial") - if err != nil { - return "", err - } - defer os.Remove(tmp.Name()) - - // Hash every byte that comes off the wire, including the parts of a - // tarball after the binary, which extraction would otherwise not read. - sum := sha512.New() - body := io.TeeReader(resp.Body, sum) - var src io.Reader = body - if a.tarball() { - src, err = binaryInTarball(body) - if err != nil { - return "", fmt.Errorf("downloading %s: %w", url, err) - } - } - if _, err := io.Copy(tmp, src); err != nil { - tmp.Close() - return "", err - } - if _, err := io.Copy(io.Discard, body); err != nil { - tmp.Close() - return "", err - } - if err := tmp.Close(); err != nil { - return "", err - } - if got := hex.EncodeToString(sum.Sum(nil)); got != a.SHA512 { - return "", fmt.Errorf("downloading %s: SHA-512 mismatch: got %s, want %s", url, got, a.SHA512) - } - if err := os.Chmod(tmp.Name(), 0o755); err != nil { - return "", err - } - if err := os.Rename(tmp.Name(), dest); err != nil { - return "", err - } - return dest, nil -} - -// binaryInTarball positions a reader at the clickhouse binary inside a -// clickhouse-common-static tarball. -func binaryInTarball(r io.Reader) (io.Reader, error) { - gz, err := gzip.NewReader(r) - if err != nil { - return nil, err - } - tr := tar.NewReader(gz) - for { - hdr, err := tr.Next() - if errors.Is(err, io.EOF) { - return nil, errors.New("tarball does not contain usr/bin/clickhouse") - } - if err != nil { - return nil, err - } - if hdr.Typeflag == tar.TypeReg && strings.HasSuffix(hdr.Name, "/usr/bin/clickhouse") { - return tr, nil - } - } -} diff --git a/internal/testcheck/clickhouse/local.go b/internal/testcheck/clickhouse/local.go deleted file mode 100644 index 5539fc79ce..0000000000 --- a/internal/testcheck/clickhouse/local.go +++ /dev/null @@ -1,82 +0,0 @@ -package clickhouse - -import ( - "bytes" - "context" - "encoding/json" - "errors" - "fmt" - "io" - "os" - "os/exec" - "path/filepath" - "strings" -) - -// local runs SQL through an ephemeral `clickhouse local` process. -type local struct { - binary string -} - -// resultSet is one JSON-format result printed by clickhouse local. Only -// statements that return rows print one; DDL and INSERT print nothing. -type resultSet struct { - Meta []resultColumn `json:"meta"` - Data []map[string]json.RawMessage `json:"data"` - Rows int `json:"rows"` -} - -type resultColumn struct { - Name string `json:"name"` - Type string `json:"type"` -} - -// run executes a multi-statement script in a fresh process with fresh -// storage and returns the result sets in statement order. Any statement -// failing fails the whole run with ClickHouse's own error message. -func (l local) run(ctx context.Context, script string) ([]resultSet, error) { - dir, err := os.MkdirTemp("", "sqlc-clickhouse-testgen-*") - if err != nil { - return nil, err - } - defer os.RemoveAll(dir) - - queries := filepath.Join(dir, "queries.sql") - if err := os.WriteFile(queries, []byte(script), 0o600); err != nil { - return nil, err - } - - // stdin must not be inherited: clickhouse local reads it as table data - // and blocks until it is closed. - cmd := exec.CommandContext(ctx, l.binary, "local", - "--multiquery", - "--queries-file", queries, - "--output-format", "JSON", - "--path", filepath.Join(dir, "data"), - ) - cmd.Stdin = nil - var stdout, stderr bytes.Buffer - cmd.Stdout = &stdout - cmd.Stderr = &stderr - if err := cmd.Run(); err != nil { - msg := strings.TrimSpace(stderr.String()) - if msg == "" { - msg = err.Error() - } - return nil, errors.New(msg) - } - - var results []resultSet - dec := json.NewDecoder(&stdout) - for { - var rs resultSet - err := dec.Decode(&rs) - if errors.Is(err, io.EOF) { - return results, nil - } - if err != nil { - return nil, fmt.Errorf("decoding clickhouse local output: %w", err) - } - results = append(results, rs) - } -} diff --git a/internal/testcheck/cmd/testcheck/main.go b/internal/testcheck/cmd/testcheck/main.go deleted file mode 100644 index 264ae35a41..0000000000 --- a/internal/testcheck/cmd/testcheck/main.go +++ /dev/null @@ -1,116 +0,0 @@ -// Command testcheck verifies the analyze cases under internal/endtoend/testdata -// against a real database. It generates nothing: each engine package reads a -// case's schema, fixture and queries, asks the database what it makes of -// them, and compares the answer with the output.json the case committed. -// -// Usage, from this directory: -// -// go run ./cmd/testcheck install clickhouse # download the pinned clickhouse binary -// go run ./cmd/testcheck check [engine] # check every case, or one engine's -// -// `go test ./...` runs the same checks as tests, skipping engines whose -// database is not available. -package main - -import ( - "context" - "errors" - "flag" - "fmt" - "io" - "os" - "runtime" - - "github.com/sqlc-dev/sqlc/internal/testcheck/clickhouse" - "github.com/sqlc-dev/sqlc/internal/testcheck/endtoend" -) - -func main() { - if err := run(context.Background(), os.Args[1:], os.Stdout, os.Stderr); err != nil { - fmt.Fprintln(os.Stderr, "testcheck:", err) - os.Exit(1) - } -} - -const usage = `usage: - testcheck install clickhouse [-version V] - download the pinned clickhouse binary into the user cache directory - testcheck check [engine] - verify the analyze cases against the database, for every engine or one` - -func run(ctx context.Context, args []string, stdout, stderr io.Writer) error { - if len(args) == 0 { - fmt.Fprintln(stderr, usage) - return errors.New("a command is required") - } - switch args[0] { - case "install": - return install(ctx, args[1:], stdout, stderr) - case "check": - return check(ctx, args[1:], stdout, stderr) - case "help", "-h", "--help": - fmt.Fprintln(stdout, usage) - return nil - } - fmt.Fprintln(stderr, usage) - return fmt.Errorf("unknown command %q", args[0]) -} - -func install(ctx context.Context, args []string, stdout, stderr io.Writer) error { - if len(args) == 0 || args[0] != clickhouse.Engine { - return errors.New("install takes the engine to install: clickhouse") - } - fs := flag.NewFlagSet("install", flag.ContinueOnError) - fs.SetOutput(stderr) - version := fs.String("version", clickhouse.DefaultVersion, "ClickHouse release to install") - if err := fs.Parse(args[1:]); err != nil { - return err - } - path, err := clickhouse.Install(ctx, *version, runtime.GOOS, runtime.GOARCH, stderr) - if err != nil { - return err - } - fmt.Fprintln(stdout, path) - return nil -} - -func check(ctx context.Context, args []string, stdout, stderr io.Writer) error { - engine := "" - if len(args) > 0 { - engine = args[0] - } - failed := 0 - if engine == "" || engine == clickhouse.Engine { - binary, err := clickhouse.Locate() - if err != nil { - if engine != "" { - return err - } - fmt.Fprintf(stderr, "skipping clickhouse: %v\n", err) - } else { - cases, err := endtoend.Cases(clickhouse.Engine) - if err != nil { - return err - } - for _, c := range cases { - diff, err := clickhouse.Check(ctx, binary, c) - switch { - case err != nil: - failed++ - fmt.Fprintf(stdout, "ERROR %s: %v\n", c.Name, err) - case diff != "": - failed++ - fmt.Fprintf(stdout, "FAIL %s (-committed +clickhouse)\n%s\n", c.Name, diff) - default: - fmt.Fprintf(stdout, "ok %s\n", c.Name) - } - } - } - } else { - return fmt.Errorf("unknown engine %q", engine) - } - if failed > 0 { - return fmt.Errorf("%d case(s) did not match", failed) - } - return nil -} diff --git a/internal/testcheck/go.mod b/internal/testcheck/go.mod deleted file mode 100644 index 154e10028e..0000000000 --- a/internal/testcheck/go.mod +++ /dev/null @@ -1,3 +0,0 @@ -module github.com/sqlc-dev/sqlc/internal/testcheck - -go 1.24.0