From d95f3da848514c6e94e677d5d4c928859ca386c1 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 7 Sep 2026 16:56:10 +0000 Subject: [PATCH 1/5] goldeneye: check the SQLite analyze cases against sqlite3 itself The analyze cases under internal/endtoend/testdata were only checked against a real database for ClickHouse. Now the sqlite ones are checked too, against one more sqlite3 shell that `install sqlite` builds, with SQLITE_ENABLE_COLUMN_METADATA and every extension option: `.stats stmt` says which table column each result column is read from, the query's own rows over the fixture and over no rows type the expressions and find the NULLs, and the parameters are followed through the bytecode EXPLAIN prints to the column each is compared with, stored into or seeks by. The query-file parsing, placeholder rewriting and the JSON shape of an engine's answer move from the clickhouse package into endtoend, where both engines share them. An analyze_expressions case with a fixture covers aggregates, IN lists, LIMIT, an outer join and RETURNING for SQLite; the five sqlite cases match what SQLite reports. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01PdKaVjSmdkmXPeUkxetyDY --- CLAUDE.md | 4 +- .../analyze_expressions/sqlite/exec.json | 5 + .../analyze_expressions/sqlite/fixture.sql | 4 + .../analyze_expressions/sqlite/query.sql | 23 + .../analyze_expressions/sqlite/schema.sql | 15 + .../analyze_expressions/sqlite/stdout.json | 238 +++++++++ internal/goldeneye/README.md | 35 +- internal/goldeneye/clickhouse/analyze.go | 73 ++- internal/goldeneye/clickhouse/check.go | 17 +- internal/goldeneye/clickhouse/queries.go | 128 +---- internal/goldeneye/clickhouse/types.go | 30 +- internal/goldeneye/cmd/goldeneye/main.go | 2 +- internal/goldeneye/endtoend/output.go | 62 +++ internal/goldeneye/endtoend/query.go | 163 +++++++ internal/goldeneye/sqlite/analyze.go | 268 ++++++++++ internal/goldeneye/sqlite/bytecode.go | 456 ++++++++++++++++++ internal/goldeneye/sqlite/catalog.go | 294 +++++++++++ internal/goldeneye/sqlite/install.go | 37 +- internal/goldeneye/sqlite/shell.go | 254 ++++++++++ internal/goldeneye/sqlite/sqlite.go | 26 +- internal/goldeneye/sqlite/sqlite_test.go | 29 ++ 21 files changed, 1946 insertions(+), 217 deletions(-) create mode 100644 internal/endtoend/testdata/analyze_expressions/sqlite/exec.json create mode 100644 internal/endtoend/testdata/analyze_expressions/sqlite/fixture.sql create mode 100644 internal/endtoend/testdata/analyze_expressions/sqlite/query.sql create mode 100644 internal/endtoend/testdata/analyze_expressions/sqlite/schema.sql create mode 100644 internal/endtoend/testdata/analyze_expressions/sqlite/stdout.json create mode 100644 internal/goldeneye/endtoend/output.go create mode 100644 internal/goldeneye/endtoend/query.go create mode 100644 internal/goldeneye/sqlite/analyze.go create mode 100644 internal/goldeneye/sqlite/bytecode.go create mode 100644 internal/goldeneye/sqlite/catalog.go create mode 100644 internal/goldeneye/sqlite/shell.go diff --git a/CLAUDE.md b/CLAUDE.md index 3ae280206d..53ea4e0cd2 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -150,8 +150,8 @@ from a live database by `/internal/goldeneye`, a nested module, and its tests verify the committed files against one byte for byte. The same module checks the `analyze_*` cases under `/internal/endtoend/testdata/` against what the database itself reports for them, so a `fixture.sql` next to a case's schema -gives the queries rows to run against. Engines whose database is not -available skip. +gives the queries rows to run against. ClickHouse and SQLite have the check +today; engines whose database is not available skip. ```bash cd internal/goldeneye diff --git a/internal/endtoend/testdata/analyze_expressions/sqlite/exec.json b/internal/endtoend/testdata/analyze_expressions/sqlite/exec.json new file mode 100644 index 0000000000..aa77909cb2 --- /dev/null +++ b/internal/endtoend/testdata/analyze_expressions/sqlite/exec.json @@ -0,0 +1,5 @@ +{ + "command": "analyze", + "args": ["--dialect", "sqlite", "--schema", "schema.sql", "query.sql"], + "contexts": ["base"] +} diff --git a/internal/endtoend/testdata/analyze_expressions/sqlite/fixture.sql b/internal/endtoend/testdata/analyze_expressions/sqlite/fixture.sql new file mode 100644 index 0000000000..cda60b0180 --- /dev/null +++ b/internal/endtoend/testdata/analyze_expressions/sqlite/fixture.sql @@ -0,0 +1,4 @@ +INSERT INTO users (id, name, bio, score) VALUES (1, 'ann', NULL, 1.5); +INSERT INTO users (id, name, bio, score) VALUES (2, 'bob', 'hello', 2.5); +INSERT INTO posts (id, user_id, title, created) VALUES (1, 1, 'first', '2024-01-01'); +INSERT INTO posts (id, user_id, title, created) VALUES (2, 1, NULL, '2024-01-02'); diff --git a/internal/endtoend/testdata/analyze_expressions/sqlite/query.sql b/internal/endtoend/testdata/analyze_expressions/sqlite/query.sql new file mode 100644 index 0000000000..8913215427 --- /dev/null +++ b/internal/endtoend/testdata/analyze_expressions/sqlite/query.sql @@ -0,0 +1,23 @@ +-- name: PostStats :one +SELECT count(*) AS total, max(id) AS latest, min(created) AS first +FROM posts WHERE user_id = ?; + +-- name: UserScores :one +SELECT avg(score) AS mean, sum(score) AS sum, group_concat(name) AS names +FROM users; + +-- name: ListUsers :many +SELECT id, lower(name) AS lname, id + 1 AS next +FROM users WHERE id IN (?, ?); + +-- name: ListPosts :many +SELECT id, title FROM posts ORDER BY created LIMIT ? OFFSET ?; + +-- name: UserPosts :many +SELECT u.name, p.title, p.created +FROM users u LEFT JOIN posts p ON p.user_id = u.id +WHERE u.name LIKE ? || '%'; + +-- name: CreatePost :one +INSERT INTO posts (user_id, title, created) VALUES (?, ?, datetime('now')) +RETURNING id, created; diff --git a/internal/endtoend/testdata/analyze_expressions/sqlite/schema.sql b/internal/endtoend/testdata/analyze_expressions/sqlite/schema.sql new file mode 100644 index 0000000000..72c6a57080 --- /dev/null +++ b/internal/endtoend/testdata/analyze_expressions/sqlite/schema.sql @@ -0,0 +1,15 @@ +CREATE TABLE users ( + id INTEGER PRIMARY KEY, + name TEXT NOT NULL, + bio TEXT, + score REAL NOT NULL DEFAULT 0 +); + +CREATE TABLE posts ( + id INTEGER PRIMARY KEY, + user_id INTEGER NOT NULL REFERENCES users(id), + title TEXT, + created TEXT NOT NULL +); + +CREATE INDEX posts_user ON posts(user_id); diff --git a/internal/endtoend/testdata/analyze_expressions/sqlite/stdout.json b/internal/endtoend/testdata/analyze_expressions/sqlite/stdout.json new file mode 100644 index 0000000000..3c6eab97a0 --- /dev/null +++ b/internal/endtoend/testdata/analyze_expressions/sqlite/stdout.json @@ -0,0 +1,238 @@ +[ + { + "name": "PostStats", + "cmd": ":one", + "columns": [ + { + "name": "total", + "type": { + "name": "integer" + } + }, + { + "name": "latest", + "type": { + "name": "integer", + "nullable": true + } + }, + { + "name": "first", + "type": { + "name": "text", + "nullable": true + } + } + ], + "params": [ + { + "number": 1, + "column": { + "name": "user_id", + "type": { + "name": "integer" + }, + "table": "posts" + } + } + ] + }, + { + "name": "UserScores", + "cmd": ":one", + "columns": [ + { + "name": "mean", + "type": { + "name": "real", + "nullable": true + } + }, + { + "name": "sum", + "type": { + "name": "real", + "nullable": true + } + }, + { + "name": "names", + "type": { + "name": "text", + "nullable": true + } + } + ], + "params": [] + }, + { + "name": "ListUsers", + "cmd": ":many", + "columns": [ + { + "name": "id", + "type": { + "name": "integer" + }, + "table": "users" + }, + { + "name": "lname", + "type": { + "name": "text" + } + }, + { + "name": "next", + "type": { + "name": "integer" + } + } + ], + "params": [ + { + "number": 1, + "column": { + "name": "id", + "type": { + "name": "integer" + }, + "table": "users" + } + }, + { + "number": 2, + "column": { + "name": "id", + "type": { + "name": "integer" + }, + "table": "users" + } + } + ] + }, + { + "name": "ListPosts", + "cmd": ":many", + "columns": [ + { + "name": "id", + "type": { + "name": "integer" + }, + "table": "posts" + }, + { + "name": "title", + "type": { + "name": "text", + "nullable": true + }, + "table": "posts" + } + ], + "params": [ + { + "number": 1, + "column": { + "name": "", + "type": { + "name": "integer" + } + } + }, + { + "number": 2, + "column": { + "name": "", + "type": { + "name": "integer" + } + } + } + ] + }, + { + "name": "UserPosts", + "cmd": ":many", + "columns": [ + { + "name": "name", + "type": { + "name": "text" + }, + "table": "users" + }, + { + "name": "title", + "type": { + "name": "text", + "nullable": true + }, + "table": "posts" + }, + { + "name": "created", + "type": { + "name": "text" + }, + "table": "posts" + } + ], + "params": [ + { + "number": 1, + "column": { + "name": "", + "type": { + "name": "text" + } + } + } + ] + }, + { + "name": "CreatePost", + "cmd": ":one", + "columns": [ + { + "name": "id", + "type": { + "name": "integer" + }, + "table": "posts" + }, + { + "name": "created", + "type": { + "name": "text" + }, + "table": "posts" + } + ], + "params": [ + { + "number": 1, + "column": { + "name": "user_id", + "type": { + "name": "integer" + }, + "table": "posts" + } + }, + { + "number": 2, + "column": { + "name": "title", + "type": { + "name": "text", + "nullable": true + }, + "table": "posts" + } + } + ] + } +] diff --git a/internal/goldeneye/README.md b/internal/goldeneye/README.md index 76807265ee..a9ddf3bb23 100644 --- a/internal/goldeneye/README.md +++ b/internal/goldeneye/README.md @@ -80,15 +80,17 @@ the hand-written files alone, and the checks do not look at them. `sqlite/signatures.go`, since a SQLite function returns NULL as often by setting no result as by saying so. The pinned release is the one the main module's driver embeds. SQLite has no catalog of types or operators, so - `types.jsonl` and `operators.jsonl` are hand-written. + `types.jsonl` and `operators.jsonl` are hand-written. `install` builds one + more shell, for the analysis check below; nothing is generated from it. ## Layout - `dialect/` — the record types the files are made of, mirrored from `internal/core/seed`, and the helpers that write a generated set of files into an engine directory or diff it against what is committed. -- `endtoend/` — finds the analyze cases and compares an engine's answer with - a case's committed output. +- `endtoend/` — finds the analyze cases, splits their query files, and + holds the shape of an engine's answer, which it compares with a case's + committed output. - `postgresql/`, `duckdb/`, `clickhouse/`, `sqlite/` — one package per engine, each exposing `Locate`, `Version` and `Generate`, `Analyze` where the engine has an analysis check, and tests that run the checks. @@ -111,5 +113,32 @@ asks for `--ast` is skipped, since only sqlc can print that. `EXPLAIN QUERY TREE`, and parameters from sentinel constants substituted for `?`, `sqlc.arg()` and `sqlc.narg()`, since ClickHouse itself never sees a placeholder; `INSERT ... VALUES` parameters map onto `DESCRIBE TABLE`. +- **`sqlite`** runs each case through one more shell `install` builds, under + `analysis/`: every extension option at once, so that any case's schema + loads, and `SQLITE_ENABLE_COLUMN_METADATA`, which lets the shell's `.stats + stmt` say which table column each result column of a statement is read + from. That column's declared type is the result column's, and its NOT + NULL decides nullability, the rowid counting as NOT NULL. SQLite types + values rather than expressions, so a column the library has no origin for + — an aggregate, an arithmetic result — is typed by the storage class of + the value it returns, which is why a case wants a `fixture.sql`: the query + is run over the fixture, with each parameter bound to a value of the + column it stands in for, and again over no rows, and a column is nullable + when either run returns a NULL for it — an aggregate over nothing, the far + side of an outer join. The library reports nothing about a parameter but + its number, so parameters are found in the bytecode `EXPLAIN` prints, the + way ClickHouse's are found in its query tree: each is followed from the + register its `Variable` loads, through copies and the expressions it is an + argument of, to the first opcode that uses it against something the + catalog can name — the other operand of a comparison, the row a seek lands + on, the position in the record an `Insert` writes, the column of an IN + list's ephemeral table it comes back out of. One that reaches nothing + nameable is described by what the program requires of it, when it + requires anything: `MustBeInt` makes LIMIT's an integer. `sqlc.arg(name)` + becomes `?N`, numbered as sqlc numbers them, so a repeated name is one + parameter. Two things the check reports that sqlc does not: a bare column + selected alongside an aggregate is NULL over no rows, and so nullable, and + a comparison such as `x IS NULL` is an integer, since that is what SQLite + returns. The other engines have no analysis check yet. diff --git a/internal/goldeneye/clickhouse/analyze.go b/internal/goldeneye/clickhouse/analyze.go index ded6201b51..0006bf6be0 100644 --- a/internal/goldeneye/clickhouse/analyze.go +++ b/internal/goldeneye/clickhouse/analyze.go @@ -7,33 +7,14 @@ import ( "regexp" "strconv" "strings" -) - -// The output is the JSON `sqlc analyze` prints, so a case's committed -// stdout.json can be compared with it byte for byte. - -type analyzedQuery struct { - Name string `json:"name"` - Cmd string `json:"cmd"` - Columns []analyzedColumn `json:"columns"` - Params []analyzedParam `json:"params"` -} -type analyzedColumn struct { - Name string `json:"name"` - Type *typeExpr `json:"type,omitempty"` - Table string `json:"table,omitempty"` -} - -type analyzedParam struct { - Number int `json:"number"` - Column analyzedColumn `json:"column"` -} + "github.com/sqlc-dev/sqlc/internal/goldeneye/endtoend" +) // analyze runs every query against the schema and fixture and records what // ClickHouse reports about each. -func analyze(ctx context.Context, l local, schema, fixture string, queries []query) ([]analyzedQuery, error) { - out := make([]analyzedQuery, 0, len(queries)) +func analyze(ctx context.Context, l local, schema, fixture string, queries []endtoend.Query) ([]endtoend.AnalyzedQuery, error) { + out := make([]endtoend.AnalyzedQuery, 0, len(queries)) for _, q := range queries { aq, err := analyzeQuery(ctx, l, schema, fixture, q) if err != nil { @@ -44,7 +25,7 @@ func analyze(ctx context.Context, l local, schema, fixture string, queries []que return out, nil } -func analyzeQuery(ctx context.Context, l local, schema, fixture string, q query) (analyzedQuery, error) { +func analyzeQuery(ctx context.Context, l local, schema, fixture string, q endtoend.Query) (endtoend.AnalyzedQuery, error) { sql, phs := bindPlaceholders(q.SQL) explain := returnsRows(sql) @@ -62,33 +43,33 @@ func analyzeQuery(ctx context.Context, l local, schema, fixture string, q query) results, err := l.run(ctx, script.String()) if err != nil { - return analyzedQuery{}, err + return endtoend.AnalyzedQuery{}, err } - aq := analyzedQuery{ + aq := endtoend.AnalyzedQuery{ Name: q.Name, Cmd: q.Cmd, - Columns: []analyzedColumn{}, - Params: []analyzedParam{}, + Columns: []endtoend.AnalyzedColumn{}, + Params: []endtoend.AnalyzedParam{}, } if !explain { return analyzeExec(ctx, l, script.String(), sql, phs, aq) } if len(results) != 2 { - return analyzedQuery{}, fmt.Errorf("expected the query tree and one result set, got %d results", len(results)) + return endtoend.AnalyzedQuery{}, fmt.Errorf("expected the query tree and one result set, got %d results", len(results)) } var lines []string for _, row := range results[0].Data { var line string if err := json.Unmarshal(row["explain"], &line); err != nil { - return analyzedQuery{}, fmt.Errorf("reading query tree: %w", err) + return endtoend.AnalyzedQuery{}, fmt.Errorf("reading query tree: %w", err) } lines = append(lines, line) } tree, err := parseQueryTree(lines) if err != nil { - return analyzedQuery{}, err + return endtoend.AnalyzedQuery{}, err } // Names and types come from the block header of the executed query, the @@ -104,23 +85,23 @@ func analyzeQuery(ctx context.Context, l local, schema, fixture string, q query) sentinels := tree.sentinels() for i, ph := range phs { - ac := analyzedColumn{} + ac := endtoend.AnalyzedColumn{} if sentinel := sentinels[i+1]; sentinel != nil { ac = tree.paramColumn(sentinel) } if ph.Name != "" { ac.Name = ph.Name } - aq.Params = append(aq.Params, analyzedParam{Number: ph.Number, Column: ac}) + aq.Params = append(aq.Params, endtoend.AnalyzedParam{Number: ph.Number, Column: ac}) } return aq, nil } -func column(name, typ string) analyzedColumn { +func column(name, typ string) endtoend.AnalyzedColumn { if typ == "" { - return analyzedColumn{Name: name} + return endtoend.AnalyzedColumn{Name: name} } - return analyzedColumn{Name: name, Type: parseType(typ)} + return endtoend.AnalyzedColumn{Name: name, Type: parseType(typ)} } // returnsRows reports whether a statement produces a result set and so can @@ -185,7 +166,7 @@ func sentinelOrdinal(c *treeNode) (int, bool) { // paramColumn describes what a placeholder is compared with or assigned to: // the other operand of the function it is an argument of, preferring a // column over an expression, or the projected column it stands for. -func (t *queryTree) paramColumn(sentinel *treeNode) analyzedColumn { +func (t *queryTree) paramColumn(sentinel *treeNode) endtoend.AnalyzedColumn { list := sentinel.parent if list != nil && list.kind == "LIST" && list.parent != nil { switch owner := list.parent; { @@ -218,7 +199,7 @@ func (t *queryTree) paramColumn(sentinel *treeNode) analyzedColumn { } // describe turns a tree expression into a column description. -func (t *queryTree) describe(n *treeNode) analyzedColumn { +func (t *queryTree) describe(n *treeNode) endtoend.AnalyzedColumn { switch n.kind { case "COLUMN": ac := column(n.attrs["column_name"], n.attrs["result_type"]) @@ -236,7 +217,7 @@ func (t *queryTree) describe(n *treeNode) analyzedColumn { } return column(name, n.attrs["constant_value_type"]) } - return analyzedColumn{} + return endtoend.AnalyzedColumn{} } var insertValuesRe = regexp.MustCompile(`(?is)^insert\s+into\s+(?:table\s+)?([\w.` + "`" + `"]+)\s*(?:\(([^)]*)\))?\s*(?:format\s+)?values\b`) @@ -244,20 +225,20 @@ var insertValuesRe = regexp.MustCompile(`(?is)^insert\s+into\s+(?:table\s+)?([\w // analyzeExec runs a statement that returns no rows. The only parameters it // can describe are those of an INSERT ... VALUES, which map positionally // onto the target columns reported by DESCRIBE TABLE. -func analyzeExec(ctx context.Context, l local, script, sql string, phs []placeholder, aq analyzedQuery) (analyzedQuery, error) { +func analyzeExec(ctx context.Context, l local, script, sql string, phs []placeholder, aq endtoend.AnalyzedQuery) (endtoend.AnalyzedQuery, error) { m := insertValuesRe.FindStringSubmatch(sql) if m != nil { script += "DESCRIBE TABLE " + m[1] + ";\n" } results, err := l.run(ctx, script) if err != nil { - return analyzedQuery{}, err + return endtoend.AnalyzedQuery{}, err } - var targets []analyzedColumn + var targets []endtoend.AnalyzedColumn if m != nil && len(results) == 1 { - byName := map[string]analyzedColumn{} - var all []analyzedColumn + byName := map[string]endtoend.AnalyzedColumn{} + var all []endtoend.AnalyzedColumn table := strings.Trim(m[1][strings.LastIndexByte(m[1], '.')+1:], "`\"") for _, row := range results[0].Data { var name, typ string @@ -277,14 +258,14 @@ func analyzeExec(ctx context.Context, l local, script, sql string, phs []placeho } } for i, ph := range phs { - ac := analyzedColumn{} + ac := endtoend.AnalyzedColumn{} if len(targets) > 0 { ac = targets[i%len(targets)] } if ph.Name != "" { ac.Name = ph.Name } - aq.Params = append(aq.Params, analyzedParam{Number: ph.Number, Column: ac}) + aq.Params = append(aq.Params, endtoend.AnalyzedParam{Number: ph.Number, Column: ac}) } return aq, nil } diff --git a/internal/goldeneye/clickhouse/check.go b/internal/goldeneye/clickhouse/check.go index 1ad8d9cb93..be007ba6eb 100644 --- a/internal/goldeneye/clickhouse/check.go +++ b/internal/goldeneye/clickhouse/check.go @@ -1,10 +1,7 @@ package clickhouse import ( - "bytes" "context" - "encoding/json" - "fmt" "os" "github.com/sqlc-dev/sqlc/internal/goldeneye/endtoend" @@ -23,25 +20,15 @@ func Analyze(ctx context.Context, binary string, c endtoend.Case) ([]byte, error return nil, err } } - src, err := os.ReadFile(c.Query) + queries, err := c.Queries() if err != nil { return nil, err } - queries, err := parseQueries(string(src)) - if err != nil { - return nil, fmt.Errorf("%s: %w", c.Query, err) - } out, err := analyze(ctx, local{binary: binary}, string(schema), string(fixture), queries) if err != nil { return nil, err } - var buf bytes.Buffer - enc := json.NewEncoder(&buf) - enc.SetIndent("", " ") - if err := enc.Encode(out); err != nil { - return nil, err - } - return buf.Bytes(), nil + return endtoend.Encode(out) } // Check compares what ClickHouse reports for a case with the output the diff --git a/internal/goldeneye/clickhouse/queries.go b/internal/goldeneye/clickhouse/queries.go index 69e3b1ac65..a9f77877b7 100644 --- a/internal/goldeneye/clickhouse/queries.go +++ b/internal/goldeneye/clickhouse/queries.go @@ -2,59 +2,10 @@ package clickhouse import ( "fmt" - "regexp" "strings" - "unicode" -) - -// query is one entry of a sqlc query file. -type query struct { - Name string - Cmd string - SQL string -} -var headerRe = regexp.MustCompile(`^\s*--\s*name:\s*(\S+)\s+(:\S+)\s*$`) - -// parseQueries splits a sqlc query file on its `-- name: X :cmd` headers. -func parseQueries(src string) ([]query, error) { - var queries []query - var cur *query - var body []string - flush := func() error { - if cur == nil { - return nil - } - sql := strings.TrimSpace(strings.Join(body, "\n")) - sql = strings.TrimRight(sql, "; \t\r\n") - if sql == "" { - return fmt.Errorf("query %s has no body", cur.Name) - } - cur.SQL = sql - queries = append(queries, *cur) - return nil - } - for _, line := range strings.Split(src, "\n") { - if m := headerRe.FindStringSubmatch(line); m != nil { - if err := flush(); err != nil { - return nil, err - } - cur = &query{Name: m[1], Cmd: m[2]} - body = body[:0] - continue - } - if cur != nil { - body = append(body, line) - } - } - if err := flush(); err != nil { - return nil, err - } - if len(queries) == 0 { - return nil, fmt.Errorf("no queries found: expected `-- name: Name :cmd` headers") - } - return queries, nil -} + "github.com/sqlc-dev/sqlc/internal/goldeneye/endtoend" +) // placeholder is one parameter reference in a query, in order of appearance. type placeholder struct { @@ -79,74 +30,17 @@ func sentinelFor(lastWord string, ordinal int) string { return fmt.Sprintf("(NULL + %d)", ordinal) } -var namedArgRe = regexp.MustCompile(`^sqlc\.(n?arg)\(\s*'?([A-Za-z_][A-Za-z0-9_]*)'?\s*\)`) - // bindPlaceholders rewrites sqlc's parameter syntax (?, sqlc.arg(name), -// sqlc.narg(name)) into constants ClickHouse can analyze, skipping string -// literals, quoted identifiers and comments. ClickHouse binds every ? -// positionally, so each placeholder is its own parameter even when a name -// repeats, which is how sqlc numbers them too. +// sqlc.narg(name)) into constants ClickHouse can analyze. ClickHouse binds +// every ? positionally, so each placeholder is its own parameter even when +// a name repeats, which is how sqlc numbers them too. func bindPlaceholders(sql string) (string, []placeholder) { - var ( - out strings.Builder - phs []placeholder - lastWord string - i = 0 - ) - number := func(string) int { return len(phs) + 1 } - for i < len(sql) { - c := sql[i] - switch { - case c == '\'' || c == '"' || c == '`': - end := skipQuoted(sql, i) - out.WriteString(sql[i:end]) - i = end - case strings.HasPrefix(sql[i:], "--"): - end := strings.IndexByte(sql[i:], '\n') - if end < 0 { - end = len(sql) - } else { - end += i - } - out.WriteString(sql[i:end]) - i = end - case strings.HasPrefix(sql[i:], "/*"): - end := strings.Index(sql[i:], "*/") - if end < 0 { - end = len(sql) - } else { - end += i + 2 - } - out.WriteString(sql[i:end]) - i = end - case c == '?': - out.WriteString(sentinelFor(lastWord, len(phs)+1)) - phs = append(phs, placeholder{Number: number("")}) - lastWord = "" - i++ - case c == 's' && namedArgRe.MatchString(sql[i:]): - m := namedArgRe.FindStringSubmatch(sql[i:]) - out.WriteString(sentinelFor(lastWord, len(phs)+1)) - phs = append(phs, placeholder{Number: number(m[2]), Name: m[2]}) - lastWord = "" - i += len(m[0]) - case isWordByte(c): - end := i - for end < len(sql) && isWordByte(sql[end]) { - end++ - } - lastWord = sql[i:end] - out.WriteString(lastWord) - i = end - default: - if !unicode.IsSpace(rune(c)) && c != ',' && c != '(' { - lastWord = "" - } - out.WriteByte(c) - i++ - } - } - return out.String(), phs + var phs []placeholder + out := endtoend.Rewrite(sql, func(name, lastWord string) string { + phs = append(phs, placeholder{Number: len(phs) + 1, Name: name}) + return sentinelFor(lastWord, len(phs)) + }) + return out, phs } func isWordByte(c byte) bool { diff --git a/internal/goldeneye/clickhouse/types.go b/internal/goldeneye/clickhouse/types.go index 77ed8d8c19..d011d06870 100644 --- a/internal/goldeneye/clickhouse/types.go +++ b/internal/goldeneye/clickhouse/types.go @@ -3,6 +3,8 @@ package clickhouse import ( "strconv" "strings" + + "github.com/sqlc-dev/sqlc/internal/goldeneye/endtoend" ) // A type is a call expression, the way ClickHouse itself models one: a @@ -35,22 +37,8 @@ import ( // String) is a type with no arguments. Resolving names against the catalog // is the reader's job; the output only records what was said. -type typeExpr struct { - Name string `json:"name"` - Nullable bool `json:"nullable,omitempty"` - Args []typeArg `json:"args,omitempty"` -} - -type typeArg struct { - Label string `json:"label,omitempty"` - Type *typeExpr `json:"type,omitempty"` - Int *int64 `json:"int,omitempty"` - Bool *bool `json:"bool,omitempty"` - String *string `json:"string,omitempty"` -} - // parseType turns a ClickHouse type string into its expression. -func parseType(t string) *typeExpr { +func parseType(t string) *endtoend.TypeExpr { name, args := splitType(t) name = strings.ToLower(strings.TrimSpace(name)) if name == "nullable" && len(args) == 1 { @@ -61,7 +49,7 @@ func parseType(t string) *typeExpr { if name == "" { name = "nothing" } - expr := &typeExpr{Name: name} + expr := &endtoend.TypeExpr{Name: name} for _, a := range args { expr.Args = append(expr.Args, parseArg(a)) } @@ -71,7 +59,7 @@ func parseType(t string) *typeExpr { // parseArg parses one argument: a quoted string, an integer, a boolean, a // labelled argument (`lat Float64` in a Tuple, `'a' = 1` in an Enum), or a // type. -func parseArg(a string) typeArg { +func parseArg(a string) endtoend.TypeArg { a = strings.TrimSpace(a) if strings.HasPrefix(a, "'") { end := skipQuoted(a, 0) @@ -81,22 +69,22 @@ func parseArg(a string) typeArg { arg.Label = lit return arg } - return typeArg{String: &lit} + return endtoend.TypeArg{String: &lit} } if n, err := strconv.ParseInt(a, 10, 64); err == nil { - return typeArg{Int: &n} + return endtoend.TypeArg{Int: &n} } switch strings.ToLower(a) { case "true", "false": b := strings.EqualFold(a, "true") - return typeArg{Bool: &b} + return endtoend.TypeArg{Bool: &b} } if i := labelEnd(a); i > 0 { arg := parseArg(a[i+1:]) arg.Label = a[:i] return arg } - return typeArg{Type: parseType(a)} + return endtoend.TypeArg{Type: parseType(a)} } // labelEnd returns the index of the space separating a label from the type diff --git a/internal/goldeneye/cmd/goldeneye/main.go b/internal/goldeneye/cmd/goldeneye/main.go index 34abbf7c7d..13fa087f44 100644 --- a/internal/goldeneye/cmd/goldeneye/main.go +++ b/internal/goldeneye/cmd/goldeneye/main.go @@ -70,7 +70,7 @@ var engines = []engine{ {clickhouse.Engine, clickhouse.Locate, clickhouse.Version, clickhouse.Generate, clickhouse.Analyze}, {duckdb.Engine, duckdb.Locate, duckdb.Version, duckdb.Generate, nil}, {postgresql.Engine, postgresql.Locate, postgresql.Version, postgresql.Generate, nil}, - {sqlite.Engine, sqlite.Locate, sqlite.Version, sqlite.Generate, nil}, + {sqlite.Engine, sqlite.Locate, sqlite.Version, sqlite.Generate, sqlite.Analyze}, } // installer puts the binary an engine is read through in place, for the diff --git a/internal/goldeneye/endtoend/output.go b/internal/goldeneye/endtoend/output.go new file mode 100644 index 0000000000..170610bcee --- /dev/null +++ b/internal/goldeneye/endtoend/output.go @@ -0,0 +1,62 @@ +package endtoend + +import ( + "bytes" + "encoding/json" +) + +// An engine's answer is written in the JSON `sqlc analyze` prints, so that +// a case's committed stdout.json can be compared with it byte for byte. + +// AnalyzedQuery is what was found out about one query. +type AnalyzedQuery struct { + Name string `json:"name"` + Cmd string `json:"cmd"` + Columns []AnalyzedColumn `json:"columns"` + Params []AnalyzedParam `json:"params"` +} + +// AnalyzedColumn describes a result column, or the column a parameter +// stands in for. +type AnalyzedColumn struct { + Name string `json:"name"` + Type *TypeExpr `json:"type,omitempty"` + Table string `json:"table,omitempty"` +} + +// AnalyzedParam is one parameter and what it is compared with or assigned +// to. +type AnalyzedParam struct { + Number int `json:"number"` + Column AnalyzedColumn `json:"column"` +} + +// TypeExpr is a type as a call expression: a lowercased name applied to an +// ordered argument list, each argument another type, an integer, a boolean +// or a quoted string, optionally labelled. Nullability is an attribute of +// the type rather than a wrapper. +type TypeExpr struct { + Name string `json:"name"` + Nullable bool `json:"nullable,omitempty"` + Args []TypeArg `json:"args,omitempty"` +} + +// TypeArg is one argument of a TypeExpr. +type TypeArg struct { + Label string `json:"label,omitempty"` + Type *TypeExpr `json:"type,omitempty"` + Int *int64 `json:"int,omitempty"` + Bool *bool `json:"bool,omitempty"` + String *string `json:"string,omitempty"` +} + +// Encode prints the answer the way sqlc analyze does. +func Encode(queries []AnalyzedQuery) ([]byte, error) { + var buf bytes.Buffer + enc := json.NewEncoder(&buf) + enc.SetIndent("", " ") + if err := enc.Encode(queries); err != nil { + return nil, err + } + return buf.Bytes(), nil +} diff --git a/internal/goldeneye/endtoend/query.go b/internal/goldeneye/endtoend/query.go new file mode 100644 index 0000000000..465595eda7 --- /dev/null +++ b/internal/goldeneye/endtoend/query.go @@ -0,0 +1,163 @@ +package endtoend + +import ( + "fmt" + "os" + "regexp" + "strings" + "unicode" +) + +// Query is one entry of a sqlc query file. +type Query struct { + Name string + Cmd string + SQL string +} + +// Queries reads the case's query file and splits it into its queries. +func (c Case) Queries() ([]Query, error) { + src, err := os.ReadFile(c.Query) + if err != nil { + return nil, err + } + queries, err := ParseQueries(string(src)) + if err != nil { + return nil, fmt.Errorf("%s: %w", c.Query, err) + } + return queries, nil +} + +var headerRe = regexp.MustCompile(`^\s*--\s*name:\s*(\S+)\s+(:\S+)\s*$`) + +// ParseQueries splits a sqlc query file on its `-- name: X :cmd` headers. +func ParseQueries(src string) ([]Query, error) { + var queries []Query + var cur *Query + var body []string + flush := func() error { + if cur == nil { + return nil + } + sql := strings.TrimSpace(strings.Join(body, "\n")) + sql = strings.TrimRight(sql, "; \t\r\n") + if sql == "" { + return fmt.Errorf("query %s has no body", cur.Name) + } + cur.SQL = sql + queries = append(queries, *cur) + return nil + } + for _, line := range strings.Split(src, "\n") { + if m := headerRe.FindStringSubmatch(line); m != nil { + if err := flush(); err != nil { + return nil, err + } + cur = &Query{Name: m[1], Cmd: m[2]} + body = body[:0] + continue + } + if cur != nil { + body = append(body, line) + } + } + if err := flush(); err != nil { + return nil, err + } + if len(queries) == 0 { + return nil, fmt.Errorf("no queries found: expected `-- name: Name :cmd` headers") + } + return queries, nil +} + +var namedArgRe = regexp.MustCompile(`^sqlc\.(n?arg)\(\s*'?([A-Za-z_][A-Za-z0-9_]*)'?\s*\)`) + +// Rewrite replaces every parameter reference in a query — ?, sqlc.arg(name) +// and sqlc.narg(name) — with what bind returns for it, in order of +// appearance, skipping string literals, quoted identifiers and comments. +// bind is handed the name, empty for a ?, and the word before the +// reference, so that a LIMIT or OFFSET can be bound differently from a +// value. Each engine decides what a reference becomes and how the +// references are numbered. +func Rewrite(sql string, bind func(name, lastWord string) string) string { + var ( + out strings.Builder + lastWord string + i = 0 + ) + for i < len(sql) { + c := sql[i] + switch { + case c == '\'' || c == '"' || c == '`': + end := skipQuoted(sql, i) + out.WriteString(sql[i:end]) + i = end + case strings.HasPrefix(sql[i:], "--"): + end := strings.IndexByte(sql[i:], '\n') + if end < 0 { + end = len(sql) + } else { + end += i + } + out.WriteString(sql[i:end]) + i = end + case strings.HasPrefix(sql[i:], "/*"): + end := strings.Index(sql[i:], "*/") + if end < 0 { + end = len(sql) + } else { + end += i + 2 + } + out.WriteString(sql[i:end]) + i = end + case c == '?': + out.WriteString(bind("", lastWord)) + lastWord = "" + i++ + case c == 's' && namedArgRe.MatchString(sql[i:]): + m := namedArgRe.FindStringSubmatch(sql[i:]) + out.WriteString(bind(m[2], lastWord)) + lastWord = "" + i += len(m[0]) + case isWordByte(c): + end := i + for end < len(sql) && isWordByte(sql[end]) { + end++ + } + lastWord = sql[i:end] + out.WriteString(lastWord) + i = end + default: + if !unicode.IsSpace(rune(c)) && c != ',' && c != '(' { + lastWord = "" + } + out.WriteByte(c) + i++ + } + } + return out.String() +} + +func isWordByte(c byte) bool { + return c == '_' || c == '.' || c >= '0' && c <= '9' || c >= 'a' && c <= 'z' || c >= 'A' && c <= 'Z' +} + +// skipQuoted returns the index just past the quoted token starting at i, +// honouring backslash escapes and doubled quotes. +func skipQuoted(s string, i int) int { + q := s[i] + j := i + 1 + for j < len(s) { + switch { + case s[j] == '\\' && j+1 < len(s): + j += 2 + case s[j] == q && j+1 < len(s) && s[j+1] == q: + j += 2 + case s[j] == q: + return j + 1 + default: + j++ + } + } + return len(s) +} diff --git a/internal/goldeneye/sqlite/analyze.go b/internal/goldeneye/sqlite/analyze.go new file mode 100644 index 0000000000..308a293bde --- /dev/null +++ b/internal/goldeneye/sqlite/analyze.go @@ -0,0 +1,268 @@ +package sqlite + +import ( + "context" + "errors" + "fmt" + "os" + "strings" + + "github.com/sqlc-dev/sqlc/internal/goldeneye/endtoend" +) + +// The analyze cases are checked against the analysis shell, which is asked +// three things about each query. What the library reports about the +// prepared statement, through `.stats stmt`: each result column's name and, +// for one read straight from a table, its declared type and which table +// column it is, with the column's NOT NULL looked up in the catalog. What +// the query returns, over the case's fixture and again over no rows, for +// the columns the library has no answer for: an expression's type is the +// storage class of its value, since SQLite types values rather than +// expressions, and any column is nullable when a NULL comes back — an +// aggregate over no rows, the far side of an outer join. And the bytecode +// the statement compiles to, which is where the parameters are found, +// since the library reports nothing about them but their number. + +// placeholder is one parameter of a query as sqlc numbers them: each ? in +// turn, and each sqlc.arg name once, at its first appearance. SQLite +// numbers ?NNN the same way, so the query is rewritten with those. +type placeholder struct { + Number int + Name string +} + +func bind(sql string) (string, []placeholder) { + var phs []placeholder + numbers := map[string]int{} + out := endtoend.Rewrite(sql, func(name, _ string) string { + n, ok := numbers[name] + if name == "" || !ok { + n = len(phs) + 1 + phs = append(phs, placeholder{Number: n, Name: name}) + if name != "" { + numbers[name] = n + } + } + return fmt.Sprintf("?%d", n) + }) + return out, phs +} + +// Analyze runs a case's queries through the analysis shell and returns what +// SQLite reports in the JSON shape sqlc analyze prints. +func Analyze(ctx context.Context, dir string, c endtoend.Case) ([]byte, error) { + if err := checkOptions(ctx, dir, analysis); err != nil { + return nil, err + } + binary := analysis.binary(dir) + schema, err := os.ReadFile(c.Schema) + if err != nil { + return nil, err + } + var fixture []byte + if c.Fixture != "" { + if fixture, err = os.ReadFile(c.Fixture); err != nil { + return nil, err + } + } + queries, err := c.Queries() + if err != nil { + return nil, err + } + out := make([]endtoend.AnalyzedQuery, 0, len(queries)) + for _, q := range queries { + aq, err := analyzeQuery(ctx, binary, string(schema), string(fixture), q) + if err != nil { + return nil, fmt.Errorf("%s: %w", q.Name, err) + } + out = append(out, aq) + } + return endtoend.Encode(out) +} + +// Check compares what SQLite reports for a case with the output the case +// committed, returning a diff when they differ. +func Check(ctx context.Context, dir string, c endtoend.Case) (string, error) { + got, err := Analyze(ctx, dir, c) + if err != nil { + return "", err + } + return c.Compare(got) +} + +func analyzeQuery(ctx context.Context, binary, schema, fixture string, q endtoend.Query) (endtoend.AnalyzedQuery, error) { + sql, phs := bind(q.SQL) + + // What the library says: the catalog, the bytecode, and the statement + // prepared, with nothing bound to its parameters. + s := newScript() + s.sql(schema) + s.sql(fixture) + s.add(".mode json") + s.add(".explain off") + for _, cq := range catalogQueries { + s.section(cq.section) + s.sql(cq.sql) + } + s.section("explain") + s.sql("EXPLAIN " + sql) + s.section("query") + s.add(".stats stmt") + line := s.sql(sql) + s.section("end") + out, err := run(ctx, binary, s) + if err != nil { + return endtoend.AnalyzedQuery{}, err + } + if errs := out.errorsBefore(line); len(errs) > 0 { + return endtoend.AnalyzedQuery{}, errors.New(strings.Join(errs, "\n")) + } + query := out.sections["query"] + if query == nil || !query.prepared { + msg := strings.TrimSpace(out.stderr) + if msg == "" { + msg = "the statement was not prepared" + } + return endtoend.AnalyzedQuery{}, errors.New(msg) + } + cat, err := readCatalog(out) + if err != nil { + return endtoend.AnalyzedQuery{}, err + } + var prog []instr + if explain := out.sections["explain"]; explain != nil && len(explain.blocks) > 0 { + if err := explain.decode(0, &prog); err != nil { + return endtoend.AnalyzedQuery{}, fmt.Errorf("reading the bytecode: %w", err) + } + } + var names []string + for _, m := range query.columns { + names = append(names, m.Name) + } + t := newTracer(cat, names) + t.run(prog) + params := make([]endtoend.AnalyzedColumn, len(phs)) + for i, ph := range phs { + params[i] = t.param(ph.Number) + } + + // What the statement returns: over the fixture, with each parameter + // bound to a value of the column it stands in for, so that the rows + // come through the WHERE clause, and over no rows. + s = newScript() + s.sql(schema) + s.sql(fixture) + s.add(".parameter init") + for i, ph := range phs { + if v := sample(params[i]); v != "" { + s.sql(fmt.Sprintf("REPLACE INTO temp.sqlite_parameters(key, value) VALUES ('?%d', %s)", ph.Number, v)) + } + } + s.add(".mode quote") + s.section("query") + s.sql(sql) + s.section("end") + bound, err := run(ctx, binary, s) + if err != nil { + return endtoend.AnalyzedQuery{}, err + } + s = newScript() + s.sql(schema) + s.add(".mode quote") + s.section("query") + s.sql(sql) + s.section("end") + empty, err := run(ctx, binary, s) + if err != nil { + return endtoend.AnalyzedQuery{}, err + } + var rows []string + for _, o := range []*output{bound, empty} { + if q := o.sections["query"]; q != nil { + rows = append(rows, q.rows...) + } + } + + aq := endtoend.AnalyzedQuery{ + Name: q.Name, + Cmd: q.Cmd, + Columns: []endtoend.AnalyzedColumn{}, + Params: []endtoend.AnalyzedParam{}, + } + for i, m := range query.columns { + aq.Columns = append(aq.Columns, describeColumn(cat, m, classes(rows, i))) + } + for i, ph := range phs { + ac := params[i] + if ph.Name != "" { + ac.Name = ph.Name + } + aq.Params = append(aq.Params, endtoend.AnalyzedParam{Number: ph.Number, Column: ac}) + } + return aq, nil +} + +// sample is an expression for a value to bind to a parameter: one of its +// column's values in the fixture, or a value of its type when all that is +// known is the type. Empty when nothing is known. +func sample(ac endtoend.AnalyzedColumn) string { + if ac.Table != "" && ac.Name != "" { + col, tbl := quoteIdent(ac.Name), quoteIdent(ac.Table) + return fmt.Sprintf("(SELECT %s FROM %s WHERE %s IS NOT NULL LIMIT 1)", col, tbl, col) + } + if ac.Type == nil { + return "" + } + switch ac.Type.Name { + case "integer", "numeric": + return "1" + case "real": + return "1.0" + case "text": + return "'a'" + case "blob": + return "x'00'" + } + return "" +} + +func quoteIdent(s string) string { + return `"` + strings.ReplaceAll(s, `"`, `""`) + `"` +} + +// classes lists the storage classes of a column's values across rows. +func classes(rows []string, i int) []string { + var out []string + for _, row := range rows { + if c := cells(row); i < len(c) { + out = append(out, storageClass(c[i])) + } + } + return out +} + +// describeColumn describes a result column: as the table column it is read +// from when it is one, otherwise by the storage class of its values, and +// nullable when any of its values was NULL. +func describeColumn(cat *catalog, m columnMeta, classes []string) endtoend.AnalyzedColumn { + ac := endtoend.AnalyzedColumn{Name: m.Name} + if col := cat.lookup(m.Table, m.Origin); col != nil { + d := col.describe() + ac.Type, ac.Table = d.Type, d.Table + } else { + for _, class := range classes { + if class != "null" { + ac.Type = &endtoend.TypeExpr{Name: class} + break + } + } + } + if ac.Type != nil { + for _, class := range classes { + if class == "null" { + ac.Type.Nullable = true + } + } + } + return ac +} diff --git a/internal/goldeneye/sqlite/bytecode.go b/internal/goldeneye/sqlite/bytecode.go new file mode 100644 index 0000000000..00afae7773 --- /dev/null +++ b/internal/goldeneye/sqlite/bytecode.go @@ -0,0 +1,456 @@ +package sqlite + +import ( + "strconv" + "strings" + + "github.com/sqlc-dev/sqlc/internal/goldeneye/endtoend" +) + +// SQLite has nothing to say about a parameter's type: a bound value is +// whatever it is, and the column a parameter stands in for appears nowhere +// in what the library reports about a prepared statement. What it does +// report is the program it compiled the statement into. EXPLAIN prints the +// bytecode, in which a parameter is a Variable loading a register, and the +// column it stands in for is the other operand of the comparison the +// register reaches, the row the seek it drives lands on, or the position +// in the record that Insert writes. So each parameter is followed from the +// register it loads, through copies and through the expressions it is an +// argument of, to the first opcode that uses it against something the +// catalog can name, the way the ClickHouse check follows a placeholder +// through the query tree to the column it is compared with. +// +// Registers hold what the last opcode to write them wrote, and remember +// which parameters have been loaded into them, which a later constant does +// not clear: coalesce(?, 'x') overwrites the parameter's register with the +// fallback on the branch where the parameter is NULL, and the comparison +// that follows is still the parameter's. A parameter with no such use is +// described by what the program does to it before using it, when it does +// anything: MustBeInt says it has to be an integer, as LIMIT's does, and +// Affinity says which affinity it is coerced to. + +// instr is one line of EXPLAIN output. +type instr struct { + Addr int `json:"addr"` + Opcode string `json:"opcode"` + P1 int `json:"p1"` + P2 int `json:"p2"` + P3 int `json:"p3"` + P4 *string `json:"p4"` + P5 int `json:"p5"` +} + +func (in instr) p4() string { + if in.P4 == nil { + return "" + } + return *in.P4 +} + +type valueKind int + +const ( + vNone valueKind = iota + vParam // a Variable + vColumn // Column: a stored column of a cursor + vRowid // Rowid, IdxRowid: the rowid of a cursor + vConstant // a literal, with its storage class + vFunction // the result of a function call, with its name + vExpr // the result of an operator + vRecord // MakeRecord: the registers packed into a record +) + +// value is what a register holds. +type value struct { + kind valueKind + cursor int + index int + class string // vConstant + fn string // vFunction + regs []int // vRecord +} + +type register struct { + desc value + params []int // every parameter loaded into the register +} + +// tracer follows the parameters of one statement through its bytecode. +type tracer struct { + cat *catalog + names []string // the statement's result columns + // cursors is what each open cursor is on; ephemeral holds, for the + // cursors on ephemeral tables and sorters, the parameters stored in + // each column, so that a parameter put into an IN list's table comes + // back out with the Column that reads it. + cursors map[int]object + ephemeral map[int]map[int][]int + regs map[int]*register + // found is the column each parameter was found to stand in for, and + // hint what the program said about a parameter's type. + found map[int]endtoend.AnalyzedColumn + hint map[int]string +} + +func newTracer(cat *catalog, names []string) *tracer { + return &tracer{ + cat: cat, + names: names, + cursors: map[int]object{}, + ephemeral: map[int]map[int][]int{}, + regs: map[int]*register{}, + found: map[int]endtoend.AnalyzedColumn{}, + hint: map[int]string{}, + } +} + +// run walks the program in the order it runs: the prologue Init jumps to, +// where the Variables of a query are loaded, and then the body. +func (t *tracer) run(prog []instr) { + order := prog + if len(prog) > 0 && prog[0].Opcode == "Init" && prog[0].P2 > 0 && prog[0].P2 < len(prog) { + order = append(append([]instr{}, prog[prog[0].P2:]...), prog[1:prog[0].P2]...) + } + for _, in := range order { + t.step(in) + } +} + +// param describes what the parameter was found to stand in for. +func (t *tracer) param(number int) endtoend.AnalyzedColumn { + if ac, ok := t.found[number]; ok { + return ac + } + if typ := t.hint[number]; typ != "" { + return endtoend.AnalyzedColumn{Type: &endtoend.TypeExpr{Name: typ}} + } + return endtoend.AnalyzedColumn{} +} + +func (t *tracer) reg(n int) *register { + r := t.regs[n] + if r == nil { + r = ®ister{} + t.regs[n] = r + } + return r +} + +func (t *tracer) params(n int) []int { + if r := t.regs[n]; r != nil { + return r.params + } + return nil +} + +// set records a write to a register. A Variable is the parameter and +// nothing else; a column read replaces whatever the register held, since +// the compiler reuses registers for the columns it reads; a constant keeps +// the parameters, for the coalesce case above; anything else carries the +// parameters it was computed from. +func (t *tracer) set(n int, v value, params []int) { + r := t.reg(n) + r.desc = v + switch v.kind { + case vConstant: + default: + r.params = params + } +} + +func (t *tracer) copy(from, to int) { + src := t.reg(from) + dst := t.reg(to) + dst.desc = src.desc + dst.params = append([]int{}, src.params...) +} + +// assoc records the column the parameters in a register stand in for, +// keeping the first found for each. +func (t *tracer) assoc(params []int, ac endtoend.AnalyzedColumn) { + for _, p := range params { + if _, ok := t.found[p]; !ok { + t.found[p] = ac + } + } +} + +// typeHint records what the program requires of a register's parameters. +func (t *tracer) typeHint(n int, typ string) { + if typ == "" { + return + } + for _, p := range t.params(n) { + if _, ok := t.hint[p]; !ok { + t.hint[p] = typ + } + } +} + +// describe turns what a register holds into the column a parameter +// compared with it stands in for, when it can be named: a stored column +// or rowid, a constant's storage class, or a function's name. +func (t *tracer) describe(v value) (endtoend.AnalyzedColumn, bool) { + switch v.kind { + case vColumn: + return t.stored(v.cursor, v.index) + case vRowid: + if owner := t.cursors[v.cursor].owner(); owner != nil { + return owner.rowid(), true + } + case vConstant: + if v.class != "null" && v.class != "" { + return endtoend.AnalyzedColumn{Type: &endtoend.TypeExpr{Name: v.class}}, true + } + case vFunction: + return endtoend.AnalyzedColumn{Name: v.fn}, true + } + return endtoend.AnalyzedColumn{}, false +} + +// stored describes the ith stored column of a cursor. +func (t *tracer) stored(cursor, i int) (endtoend.AnalyzedColumn, bool) { + col, owner, ok := t.cursors[cursor].column(i) + switch { + case !ok: + return endtoend.AnalyzedColumn{}, false + case col != nil: + return col.describe(), true + default: + return owner.rowid(), true + } +} + +// rank orders what a parameter's fellow operands may be described by: a +// column first, then a function, then a constant. +func rank(v value) int { + switch v.kind { + case vColumn, vRowid: + return 0 + case vFunction: + return 1 + case vConstant: + if v.class != "null" && v.class != "" { + return 2 + } + } + return -1 +} + +// expression handles an opcode computing a result from operand registers: +// each parameter among the operands is described by the best other +// operand, and one with no other operand to be described by is carried +// into the result, for whatever the result is then compared with. +func (t *tracer) expression(args []int, dest int, result value) { + var carried []int + for _, a := range args { + params := t.params(a) + if len(params) == 0 { + continue + } + best := -1 + for _, b := range args { + if b == a { + continue + } + if r := rank(t.reg(b).desc); r >= 0 && (best < 0 || r < rank(t.reg(best).desc)) { + best = b + } + } + if best >= 0 { + if ac, ok := t.describe(t.reg(best).desc); ok { + t.assoc(params, ac) + continue + } + } + carried = append(carried, params...) + } + t.set(dest, result, carried) +} + +// compare handles a comparison of two registers: the parameters of each +// stand in for what the other holds. +func (t *tracer) compare(a, b int) { + if ac, ok := t.describe(t.reg(b).desc); ok { + t.assoc(t.params(a), ac) + } + if ac, ok := t.describe(t.reg(a).desc); ok { + t.assoc(t.params(b), ac) + } +} + +// store handles a register written as the ith column of a cursor's row: +// an ephemeral table remembers the parameters for the reads to come, and +// a real one is the column the parameters stand in for. +func (t *tracer) store(cursor, i, reg int) { + params := t.params(reg) + if len(params) == 0 { + return + } + if cols, ok := t.ephemeral[cursor]; ok { + cols[i] = append(cols[i], params...) + return + } + if ac, ok := t.stored(cursor, i); ok { + t.assoc(params, ac) + } +} + +// key handles registers used as a key into a cursor: n registers from +// start, or the record in start when n is zero. +func (t *tracer) key(cursor, start, n int) { + regs := t.span(start, n) + if n == 0 { + if r := t.reg(start); r.desc.kind == vRecord { + regs = r.desc.regs + } + } + for i, reg := range regs { + t.store(cursor, i, reg) + } +} + +func (t *tracer) span(start, n int) []int { + regs := make([]int, 0, max(n, 0)) + for i := 0; i < n; i++ { + regs = append(regs, start+i) + } + return regs +} + +// affinity names the type an affinity character coerces to. +func affinity(c byte) string { + switch c { + case 'A': + return "blob" + case 'B': + return "text" + case 'C': + return "numeric" + case 'D': + return "integer" + case 'E': + return "real" + } + return "" +} + +func (t *tracer) step(in instr) { + switch in.Opcode { + case "OpenRead", "OpenWrite", "ReopenIdx": + t.cursors[in.P1] = t.cat.root(in.P3, in.P2) + case "OpenDup": + t.cursors[in.P1] = t.cursors[in.P2] + if cols, ok := t.ephemeral[in.P2]; ok { + t.ephemeral[in.P1] = cols + } + case "OpenEphemeral", "OpenAutoindex", "SorterOpen", "OpenPseudo": + t.cursors[in.P1] = object{} + t.ephemeral[in.P1] = map[int][]int{} + case "Variable": + t.set(in.P2, value{kind: vParam}, []int{in.P1}) + case "Column", "VColumn": + t.set(in.P3, value{kind: vColumn, cursor: in.P1, index: in.P2}, t.ephemeral[in.P1][in.P2]) + case "Rowid", "IdxRowid": + t.set(in.P2, value{kind: vRowid, cursor: in.P1}, nil) + case "Copy": + for i := 0; i <= in.P3; i++ { + t.copy(in.P1+i, in.P2+i) + } + case "SCopy", "IntCopy": + t.copy(in.P1, in.P2) + case "Null", "SoftNull": + end := max(in.P2, in.P3) + if in.Opcode == "SoftNull" { + end = in.P1 + in.P2 = in.P1 + } + for r := in.P2; r <= end; r++ { + t.set(r, value{kind: vConstant, class: "null"}, nil) + } + case "Integer", "Int64": + t.set(in.P2, value{kind: vConstant, class: "integer"}, nil) + case "Real": + t.set(in.P2, value{kind: vConstant, class: "real"}, nil) + case "String8", "String": + t.set(in.P2, value{kind: vConstant, class: "text"}, nil) + case "Blob": + t.set(in.P2, value{kind: vConstant, class: "blob"}, nil) + case "Function", "PureFunc": + // P4 prints the function's arity, which for a variadic function + // is negative and says nothing about this call. Its arguments + // are the registers from P2 written so far, allocated together + // for the call. + name, n := parseCall(in.p4()) + if n < 0 { + n = 0 + for t.regs[in.P2+n] != nil { + n++ + } + } + t.expression(t.span(in.P2, n), in.P3, value{kind: vFunction, fn: name}) + case "Add", "Subtract", "Multiply", "Divide", "Remainder", "Concat", "BitAnd", "BitOr", "ShiftLeft", "ShiftRight": + t.expression([]int{in.P2, in.P1}, in.P3, value{kind: vExpr}) + case "MakeRecord": + t.set(in.P3, value{kind: vRecord, regs: t.span(in.P1, in.P2)}, nil) + case "Insert": + if r := t.reg(in.P2); r.desc.kind == vRecord { + for i, reg := range r.desc.regs { + t.store(in.P1, i, reg) + } + } + if owner := t.cursors[in.P1].owner(); owner != nil { + t.assoc(t.params(in.P3), owner.rowid()) + } + case "IdxInsert", "SorterInsert": + n, _ := strconv.Atoi(in.p4()) + if n > 0 { + t.key(in.P1, in.P3, n) + } else { + t.key(in.P1, in.P2, 0) + } + case "Eq", "Ne", "Lt", "Le", "Gt", "Ge": + t.compare(in.P1, in.P3) + case "SeekRowid", "NotExists": + if owner := t.cursors[in.P1].owner(); owner != nil { + t.assoc(t.params(in.P3), owner.rowid()) + } + case "SeekGE", "SeekGT", "SeekLE", "SeekLT": + // On a table with a rowid the seek is by the rowid in P3; on an + // index it is by the P4 registers from P3. + if o := t.cursors[in.P1]; o.table != nil && !o.table.withoutRowid { + t.assoc(t.params(in.P3), o.table.rowid()) + break + } + n, _ := strconv.Atoi(in.p4()) + t.key(in.P1, in.P3, n) + case "IdxGE", "IdxGT", "IdxLE", "IdxLT", "Found", "NotFound", "NoConflict": + n, _ := strconv.Atoi(in.p4()) + t.key(in.P1, in.P3, n) + case "MustBeInt": + t.typeHint(in.P1, "integer") + case "Affinity": + for i, c := range []byte(in.p4()) { + if i < in.P2 { + t.typeHint(in.P1+i, affinity(c)) + } + } + case "ResultRow": + for i := 0; i < in.P2; i++ { + if i < len(t.names) { + t.assoc(t.params(in.P1+i), endtoend.AnalyzedColumn{Name: t.names[i]}) + } + } + } +} + +// parseCall splits the `name(n)` a Function's P4 prints into the function's +// name and the number of arguments it is called with. +func parseCall(p4 string) (string, int) { + open := strings.LastIndexByte(p4, '(') + if open < 0 || !strings.HasSuffix(p4, ")") { + return p4, 0 + } + n, _ := strconv.Atoi(p4[open+1 : len(p4)-1]) + return p4[:open], n +} diff --git a/internal/goldeneye/sqlite/catalog.go b/internal/goldeneye/sqlite/catalog.go new file mode 100644 index 0000000000..81809833c2 --- /dev/null +++ b/internal/goldeneye/sqlite/catalog.go @@ -0,0 +1,294 @@ +package sqlite + +import ( + "fmt" + "strconv" + "strings" + + "github.com/sqlc-dev/sqlc/internal/goldeneye/endtoend" +) + +// catalog is what the database says about a schema: its tables and their +// columns, its indexes, and the b-tree each is stored in, which is how the +// bytecode names them. +type catalog struct { + tables map[string]*table + roots map[int]object // by root page +} + +type table struct { + name string + withoutRowid bool + cols []*tableColumn // by cid + // stored lists the columns in the order a row's record holds them, + // which is cid order less the generated columns that are not stored. + stored []*tableColumn + // pk is the PRIMARY KEY index of a WITHOUT ROWID table, whose order is + // the order the table's own b-tree stores a row in. + pk *index +} + +type tableColumn struct { + table *table + cid int + name string + declType string + notNull bool + pk int // position in the primary key, 0 for none +} + +type index struct { + name string + table *table + cids []int // by seqno: -1 for the rowid, -2 for an expression +} + +// object is what a cursor is open on: a table, an index, or, with neither +// set, something the catalog does not describe — an ephemeral table, a +// sorter, a virtual table. +type object struct { + table *table + index *index +} + +// The catalog is read with these, each in a section of its own, in json +// mode. A query that finds nothing prints nothing, so a section may be +// empty. +var catalogQueries = []struct{ section, sql string }{ + {"tables", `SELECT name, type, wr FROM pragma_table_list WHERE schema = 'main' ORDER BY name`}, + {"columns", `SELECT t.name AS tbl, c.cid, c.name, c.type, c."notnull" AS "notnull", c.pk, c.hidden + FROM pragma_table_list t, pragma_table_xinfo(t.name) c + WHERE t.schema = 'main' AND t.type IN ('table', 'virtual') ORDER BY t.name, c.cid`}, + {"indexes", `SELECT t.name AS tbl, i.name AS idx, i.origin, x.seqno, x.cid + FROM pragma_table_list t, pragma_index_list(t.name) i, pragma_index_xinfo(i.name) x + WHERE t.schema = 'main' ORDER BY t.name, i.name, x.seqno`}, + {"roots", `SELECT type, name, tbl_name, rootpage FROM sqlite_schema WHERE rootpage > 0`}, +} + +// readCatalog builds the catalog from the sections catalogQueries printed. +func readCatalog(out *output) (*catalog, error) { + var tables []struct { + Name string `json:"name"` + Type string `json:"type"` + WR int `json:"wr"` + } + var columns []struct { + Table string `json:"tbl"` + CID int `json:"cid"` + Name string `json:"name"` + Type string `json:"type"` + NotNull int `json:"notnull"` + PK int `json:"pk"` + Hidden int `json:"hidden"` + } + var indexes []struct { + Table string `json:"tbl"` + Index string `json:"idx"` + Origin string `json:"origin"` + Seqno int `json:"seqno"` + CID int `json:"cid"` + } + var roots []struct { + Type string `json:"type"` + Name string `json:"name"` + Table string `json:"tbl_name"` + Rootpage int `json:"rootpage"` + } + for name, v := range map[string]any{"tables": &tables, "columns": &columns, "indexes": &indexes, "roots": &roots} { + s := out.sections[name] + if s == nil { + return nil, fmt.Errorf("reading the catalog: no %s section in the output", name) + } + if len(s.blocks) > 0 { + if err := s.decode(0, v); err != nil { + return nil, fmt.Errorf("reading the catalog's %s: %w", name, err) + } + } + } + + c := &catalog{tables: map[string]*table{}, roots: map[int]object{}} + for _, t := range tables { + if t.Type == "table" || t.Type == "virtual" { + c.tables[t.Name] = &table{name: t.Name, withoutRowid: t.WR == 1} + } + } + for _, col := range columns { + t := c.tables[col.Table] + if t == nil { + continue + } + tc := &tableColumn{table: t, cid: col.CID, name: col.Name, declType: col.Type, notNull: col.NotNull == 1, pk: col.PK} + for len(t.cols) <= col.CID { + t.cols = append(t.cols, nil) + } + t.cols[col.CID] = tc + // A generated column that is not stored — hidden is 2 — has no + // place in the record. + if col.Hidden != 2 { + t.stored = append(t.stored, tc) + } + } + byName := map[string]*index{} + for _, ic := range indexes { + t := c.tables[ic.Table] + if t == nil { + continue + } + ix := byName[ic.Index] + if ix == nil { + ix = &index{name: ic.Index, table: t} + byName[ic.Index] = ix + if ic.Origin == "pk" && t.withoutRowid { + t.pk = ix + } + } + for len(ix.cids) <= ic.Seqno { + ix.cids = append(ix.cids, -2) + } + ix.cids[ic.Seqno] = ic.CID + } + for _, r := range roots { + switch r.Type { + case "table": + if t := c.tables[r.Name]; t != nil { + c.roots[r.Rootpage] = object{table: t} + } + case "index": + if ix := byName[r.Name]; ix != nil { + c.roots[r.Rootpage] = object{index: ix} + } + } + } + return c, nil +} + +// root returns what is stored in a b-tree of the main database. +func (c *catalog) root(db, page int) object { + if db != 0 { + return object{} + } + return c.roots[page] +} + +// lookup finds a table column by the names `.stats stmt` prints. +func (c *catalog) lookup(tbl, col string) *tableColumn { + t := c.tables[tbl] + if t == nil { + return nil + } + for _, tc := range t.cols { + if tc != nil && tc.name == col { + return tc + } + } + return nil +} + +// rowidAlias returns the column that is another name for the rowid: the +// lone INTEGER PRIMARY KEY of a table that has a rowid. +func (t *table) rowidAlias() *tableColumn { + if t.withoutRowid { + return nil + } + var alias *tableColumn + for _, tc := range t.cols { + if tc == nil || tc.pk == 0 { + continue + } + if alias != nil || !strings.EqualFold(tc.declType, "INTEGER") { + return nil + } + alias = tc + } + return alias +} + +// describe is the column as sqlc analyze describes one: its declared type, +// nullable unless declared NOT NULL or the rowid, which is never NULL. +func (tc *tableColumn) describe() endtoend.AnalyzedColumn { + typ := parseType(tc.declType) + typ.Nullable = !tc.notNull && tc.table.rowidAlias() != tc + return endtoend.AnalyzedColumn{Name: tc.name, Type: typ, Table: tc.table.name} +} + +// rowid describes the table's rowid: the column that aliases it, or the +// rowid itself. +func (t *table) rowid() endtoend.AnalyzedColumn { + if alias := t.rowidAlias(); alias != nil { + return alias.describe() + } + return endtoend.AnalyzedColumn{Name: "rowid", Type: &endtoend.TypeExpr{Name: "integer"}, Table: t.name} +} + +// column returns what the object's ith stored column is: a table column, +// or the rowid of a table. An index stores the columns it is on and then +// the rowid; a WITHOUT ROWID table is stored in the order of its primary +// key; any other table is stored in column order. +func (o object) column(i int) (*tableColumn, *table, bool) { + ix := o.index + if o.table != nil && o.table.withoutRowid { + ix = o.table.pk + } + switch { + case ix != nil: + if i < 0 || i >= len(ix.cids) { + return nil, nil, false + } + cid := ix.cids[i] + switch { + case cid == -1: + return nil, ix.table, true + case cid >= 0 && cid < len(ix.table.cols) && ix.table.cols[cid] != nil: + return ix.table.cols[cid], nil, true + } + case o.table != nil: + if i >= 0 && i < len(o.table.stored) { + return o.table.stored[i], nil, true + } + } + return nil, nil, false +} + +// owner is the table the object stores rows of. +func (o object) owner() *table { + if o.index != nil { + return o.index.table + } + return o.table +} + +// parseType reads a declared type the way sqlc's catalog does: the name +// lowercased, with whatever is in parentheses after it as arguments. A +// column declared with no type at all can hold anything. +func parseType(decl string) *endtoend.TypeExpr { + decl = strings.TrimSpace(decl) + if decl == "" { + return &endtoend.TypeExpr{Name: "any"} + } + name, args := decl, "" + if open := strings.IndexByte(decl, '('); open >= 0 && strings.HasSuffix(decl, ")") { + name, args = decl[:open], decl[open+1:len(decl)-1] + } + t := &endtoend.TypeExpr{Name: strings.ToLower(strings.TrimSpace(name))} + if strings.TrimSpace(args) == "" { + return t + } + for _, a := range strings.Split(args, ",") { + a = strings.TrimSpace(a) + switch { + case strings.HasPrefix(a, "'") && strings.HasSuffix(a, "'") && len(a) >= 2: + s := strings.ReplaceAll(a[1:len(a)-1], "''", "'") + t.Args = append(t.Args, endtoend.TypeArg{String: &s}) + case strings.EqualFold(a, "true") || strings.EqualFold(a, "false"): + b := strings.EqualFold(a, "true") + t.Args = append(t.Args, endtoend.TypeArg{Bool: &b}) + default: + if n, err := strconv.ParseInt(a, 10, 64); err == nil { + t.Args = append(t.Args, endtoend.TypeArg{Int: &n}) + } else { + t.Args = append(t.Args, endtoend.TypeArg{Type: &endtoend.TypeExpr{Name: strings.ToLower(a)}}) + } + } + } + return t +} diff --git a/internal/goldeneye/sqlite/install.go b/internal/goldeneye/sqlite/install.go index 0a0ed1d674..56ddc6495c 100644 --- a/internal/goldeneye/sqlite/install.go +++ b/internal/goldeneye/sqlite/install.go @@ -13,6 +13,7 @@ import ( "os/exec" "path/filepath" "runtime" + "slices" "strings" "sync" ) @@ -50,6 +51,26 @@ var extensions = []build{ {"enable_rtree", []string{"SQLITE_ENABLE_RTREE"}}, } +// analysis is the shell the analyze cases run through, which is no +// dialect build: nothing is generated from it. It is built with column +// metadata, so that `.stats stmt` can say which table column each result +// column of a statement comes from, and with every extension option at +// once, so that whatever a case's schema asks for is there. +var analysis = build{"analysis", append([]string{"SQLITE_ENABLE_COLUMN_METADATA"}, extensionOptions()...)} + +// extensionOptions is every option an extension build turns on, once each. +func extensionOptions() []string { + var opts []string + for _, b := range extensions { + for _, opt := range b.options { + if !slices.Contains(opts, opt) { + opts = append(opts, opt) + } + } + } + return opts +} + // asset is one downloadable amalgamation of SQLite: the zip published on // sqlite.org holding sqlite3.c, sqlite3.h and the shell's shell.c. type asset struct { @@ -87,7 +108,8 @@ func (a asset) url() string { } // cacheDir is where Install puts a version: the sources under src/, and one -// shell per build under default/ and under each option's extension name. +// shell per build under default/, under each option's extension name and +// under analysis/. func cacheDir(version string) (string, error) { dir, err := os.UserCacheDir() if err != nil { @@ -105,11 +127,18 @@ type build struct { options []string } -// builds lists the default build first, then one per extension. +// builds lists the dialect builds: the default build first, then one per +// extension. func builds() []build { return append([]build{{"default", nil}}, extensions...) } +// shells lists every build Install makes: the dialect builds and the +// analysis shell. +func shells() []build { + return append(builds(), analysis) +} + // flags are every option a build is compiled with. func (b build) flags() []string { return append(append([]string{}, defaultOptions...), b.options...) @@ -125,7 +154,7 @@ func Locate() (string, error) { if err != nil { return "", err } - for _, b := range builds() { + for _, b := range shells() { if _, err := os.Stat(b.binary(dir)); err != nil { return "", fmt.Errorf("sqlite %s is not built with %s: run `go run ./cmd/goldeneye install sqlite` in internal/goldeneye", DefaultVersion, strings.Join(b.flags(), " ")) } @@ -153,7 +182,7 @@ func Install(ctx context.Context, version, goos, goarch string, progress io.Writ return "", err } var missing []build - for _, b := range builds() { + for _, b := range shells() { if _, err := os.Stat(b.binary(dir)); err != nil { missing = append(missing, b) } diff --git a/internal/goldeneye/sqlite/shell.go b/internal/goldeneye/sqlite/shell.go new file mode 100644 index 0000000000..f17b9333ae --- /dev/null +++ b/internal/goldeneye/sqlite/shell.go @@ -0,0 +1,254 @@ +package sqlite + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "os/exec" + "regexp" + "strconv" + "strings" +) + +// The analysis shell is driven with a script on its standard input, made +// of SQL and of the shell's own dot-commands: `.print` labels the sections +// of the output so that each statement's results can be found again, +// `.mode` chooses how rows print — json for the catalog and the bytecode, +// quote for a query's own rows, since quote mode is the one that tells a +// blob from a string and a real from an integer — and `.stats stmt` makes +// the shell print what it knows about each result column of the statement +// it has just run, which with column metadata compiled in is the column's +// name, declared type, database, table and origin column. + +// script is a shell script under construction, counting its lines so that +// an error the shell reports "near line N" can be placed. +type script struct { + buf strings.Builder + line int // the number of the next line written +} + +func newScript() *script { + return &script{line: 1} +} + +// add writes text on lines of its own, and returns the line it starts on. +func (s *script) add(text string) int { + start := s.line + text = strings.TrimSpace(text) + if text == "" { + return start + } + s.buf.WriteString(text) + s.buf.WriteByte('\n') + s.line += strings.Count(text, "\n") + 1 + return start +} + +// sql writes statements, ending them with a semicolon when they lack one. +func (s *script) sql(text string) int { + text = strings.TrimRight(strings.TrimSpace(text), "; \t\r\n") + if text == "" { + return s.line + } + return s.add(text + ";") +} + +// section labels what follows, up to the next label. +func (s *script) section(name string) { + s.add(".print @@" + name) +} + +// columnMeta is what `.stats stmt` prints about one result column. Table +// and Origin name the table column a result column is read from, and are +// empty for an expression. +type columnMeta struct { + Name string + DeclType string + Database string + Table string + Origin string +} + +// section is the output between two labels. +type section struct { + // blocks holds every JSON result set printed in the section. + blocks []json.RawMessage + // rows holds every other line printed before the statement's + // statistics: in quote mode, one row each. + rows []string + // prepared says a statement's statistics were printed, which the shell + // does only for a statement it could prepare. + prepared bool + columns []columnMeta +} + +// decode reads the section's nth JSON result set. +func (s *section) decode(n int, v any) error { + if n >= len(s.blocks) { + return fmt.Errorf("expected at least %d result set(s), got %d", n+1, len(s.blocks)) + } + return json.Unmarshal(s.blocks[n], v) +} + +// output is what one run of the shell printed, by section. +type output struct { + sections map[string]*section + stderr string +} + +var errLineRe = regexp.MustCompile(`near line (\d+):`) + +// errorsBefore returns the errors the shell reported for statements before +// the given line: the ones that are not the statement under analysis +// failing to run, which it may, with nothing bound to its parameters. +func (o *output) errorsBefore(line int) []string { + var errs []string + for _, l := range strings.Split(o.stderr, "\n") { + if m := errLineRe.FindStringSubmatch(l); m != nil { + if n, _ := strconv.Atoi(m[1]); n < line { + errs = append(errs, strings.TrimSpace(l)) + } + } + } + return errs +} + +// run feeds a script to the shell over a fresh in-memory database and +// parses what it printed. A statement that fails does not stop the shell, +// so a failure is read from the output: a section whose statement has no +// statistics was not prepared, and stderr says why. +func run(ctx context.Context, binary string, s *script) (*output, error) { + cmd := exec.CommandContext(ctx, binary, ":memory:") + cmd.Stdin = strings.NewReader(s.buf.String()) + var stdout, stderr bytes.Buffer + cmd.Stdout = &stdout + cmd.Stderr = &stderr + if err := cmd.Run(); err != nil { + var exit *exec.ExitError + if !errors.As(err, &exit) { + return nil, fmt.Errorf("sqlite3: %w", err) + } + } + out := &output{sections: map[string]*section{}, stderr: stderr.String()} + var ( + cur *section + block []string // lines of the JSON result set being read + row []string // lines of the quote-mode row being read + ) + for _, line := range strings.Split(stdout.String(), "\n") { + switch { + case block != nil: + block = append(block, line) + if strings.HasSuffix(line, "]") { + cur.blocks = append(cur.blocks, json.RawMessage(strings.Join(block, "\n"))) + block = nil + } + case row != nil: + row = append(row, line) + if joined := strings.Join(row, "\n"); strings.Count(joined, "'")%2 == 0 { + cur.rows = append(cur.rows, joined) + row = nil + } + case strings.HasPrefix(line, "@@"): + cur = §ion{} + out.sections[line[2:]] = cur + case cur == nil || line == "": + case strings.HasPrefix(line, "["): + if strings.HasSuffix(line, "]") { + cur.blocks = append(cur.blocks, json.RawMessage(line)) + } else { + block = []string{line} + } + case strings.HasPrefix(line, "Number of output columns:"): + cur.prepared = true + case cur.prepared: + if i, field, value, ok := statLine(line); ok { + for len(cur.columns) <= i { + cur.columns = append(cur.columns, columnMeta{}) + } + c := &cur.columns[i] + switch field { + case "name": + c.Name = value + case "declared type": + c.DeclType = value + case "database name": + c.Database = value + case "table name": + c.Table = value + case "origin name": + c.Origin = value + } + } + default: + // A quote-mode row, complete unless a string in it spans lines. + if strings.Count(line, "'")%2 == 0 { + cur.rows = append(cur.rows, line) + } else { + row = []string{line} + } + } + } + return out, nil +} + +var statLineRe = regexp.MustCompile(`^Column (\d+) ([a-z ]+):`) + +// statLine reads one line of `.stats stmt` column output, which the shell +// prints as the label padded to 36 columns, a space and the value; a value +// the library has no answer for prints as the C library prints a null +// string. +func statLine(line string) (int, string, string, bool) { + m := statLineRe.FindStringSubmatch(line) + if m == nil { + return 0, "", "", false + } + i, _ := strconv.Atoi(m[1]) + label := m[0] + value := "" + if start := max(len(label), 36) + 1; start < len(line) { + value = line[start:] + } + if value == "(null)" { + value = "" + } + return i, m[2], value, true +} + +// cells splits a quote-mode row into its values as printed. +func cells(row string) []string { + var out []string + start := 0 + quoted := false + for i := 0; i < len(row); i++ { + switch { + case row[i] == '\'': + quoted = !quoted + case row[i] == ',' && !quoted: + out = append(out, row[start:i]) + start = i + 1 + } + } + return append(out, row[start:]) +} + +// storageClass reads the storage class of a value as quote mode prints +// it: a string is quoted, a blob is quoted with an X, NULL is spelled +// out, and a real has a point, an exponent or is infinite or not a number +// where an integer is digits. +func storageClass(cell string) string { + switch { + case cell == "NULL": + return "null" + case strings.HasPrefix(cell, "'"): + return "text" + case strings.HasPrefix(cell, "x'") || strings.HasPrefix(cell, "X'"): + return "blob" + case strings.ContainsAny(cell, ".eEIN"): + return "real" + default: + return "integer" + } +} diff --git a/internal/goldeneye/sqlite/sqlite.go b/internal/goldeneye/sqlite/sqlite.go index d8aa1b62ba..869dc0310f 100644 --- a/internal/goldeneye/sqlite/sqlite.go +++ b/internal/goldeneye/sqlite/sqlite.go @@ -181,12 +181,25 @@ type shell struct { // option lists changed would otherwise describe the wrong dialect. func readShell(ctx context.Context, dir string, b build) (*shell, error) { s := &shell{build: b, binary: b.binary(dir)} + if err := checkOptions(ctx, dir, b); err != nil { + return nil, err + } + if err := query(ctx, s.binary, functionList, &s.rows); err != nil { + return nil, err + } + return s, nil +} + +// checkOptions asks a build's shell whether it was compiled with every +// option any build turns on, and fails when the answers do not match the +// build's own list. +func checkOptions(ctx context.Context, dir string, b build) error { var used []struct { Option string `json:"option"` Used int `json:"used"` } known := map[string]bool{} - for _, b := range builds() { + for _, b := range shells() { for _, opt := range b.flags() { known[opt] = true } @@ -195,8 +208,8 @@ func readShell(ctx context.Context, dir string, b build) (*shell, error) { for _, opt := range slices.Sorted(maps.Keys(known)) { clauses = append(clauses, fmt.Sprintf("SELECT '%s' AS option, sqlite_compileoption_used('%s') AS used", opt, opt)) } - if err := query(ctx, s.binary, strings.Join(clauses, " UNION ALL "), &used); err != nil { - return nil, err + if err := query(ctx, b.binary(dir), strings.Join(clauses, " UNION ALL "), &used); err != nil { + return err } for _, u := range used { want := 0 @@ -204,13 +217,10 @@ func readShell(ctx context.Context, dir string, b build) (*shell, error) { want = 1 } if u.Used != want { - return nil, fmt.Errorf("sqlite: the %s shell was not built with the options it should have been (%s is %d): remove %s and run `go run ./cmd/goldeneye install sqlite` again", b.name, u.Option, u.Used, dir) + return fmt.Errorf("sqlite: the %s shell was not built with the options it should have been (%s is %d): remove %s and run `go run ./cmd/goldeneye install sqlite` again", b.name, u.Option, u.Used, filepath.Join(dir, b.name)) } } - if err := query(ctx, s.binary, functionList, &s.rows); err != nil { - return nil, err - } - return s, nil + return nil } // generator accumulates the functions of every build, reading their diff --git a/internal/goldeneye/sqlite/sqlite_test.go b/internal/goldeneye/sqlite/sqlite_test.go index 028f015fbf..8327090d7e 100644 --- a/internal/goldeneye/sqlite/sqlite_test.go +++ b/internal/goldeneye/sqlite/sqlite_test.go @@ -5,6 +5,7 @@ import ( "testing" "github.com/sqlc-dev/sqlc/internal/goldeneye/dialect" + "github.com/sqlc-dev/sqlc/internal/goldeneye/endtoend" ) // TestDialect verifies the committed SQLite dialect against what the pinned @@ -35,3 +36,31 @@ func TestDialect(t *testing.T) { t.Errorf("%s does not match what %s reports:\n%s", dir, version, report) } } + +// TestAnalyzeCases verifies every SQLite analyze case under +// internal/endtoend/testdata against what SQLite reports. It skips unless +// the shells are installed. +func TestAnalyzeCases(t *testing.T) { + dir, err := Locate() + if err != nil { + t.Skip(err) + } + cases, err := endtoend.Cases(Engine) + if err != nil { + t.Fatal(err) + } + if len(cases) == 0 { + t.Fatal("no sqlite analyze cases found") + } + for _, c := range cases { + t.Run(c.Name, func(t *testing.T) { + diff, err := Check(context.Background(), dir, c) + if err != nil { + t.Fatal(err) + } + if diff != "" { + t.Errorf("%s does not match what SQLite reports (-committed +sqlite):\n%s", c.Output, diff) + } + }) + } +} From 012548f2e05f259c509ad12ef4a9917a7aa09a0b Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 7 Sep 2026 17:15:59 +0000 Subject: [PATCH 2/5] sqlite: sum and abs return their argument's type, and IS never NULL Three things the goldeneye check found sqlc's analysis saying that SQLite does not. sum, abs, ceil, floor and trunc hand back an integer for an integer and a real for a real, so the dialect now spells their return type "$1", the argument's type, rather than widening to real; unixepoch, which is a real only with 'subsec', keeps real. The legacy compiler learns to resolve "$n" to the named column's or literal's type, so sum(int_val) generates NullInt64 rather than NullFloat64. IS and IS NOT test for NULL rather than propagate it, so their result is never NULL, as are SQLite's postfix ISNULL and NOTNULL, which used to be typed as their operand. And LIKE, GLOB, REGEXP and MATCH are predicates, listed with the dialect's comparison operators under the names the parser gives them, in place of the ~~ spellings it never uses. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01PdKaVjSmdkmXPeUkxetyDY --- internal/compiler/output_columns.go | 39 ++++++++++++++++++- internal/core/analyzer/expr.go | 21 +++++++++- .../analyze_expressions/sqlite/query.sql | 2 +- .../analyze_expressions/sqlite/stdout.json | 7 ++++ .../builtins/sqlite/go/aggfunc.sql.go | 8 ++-- .../builtins/sqlite/go/scalarfunc.sql.go | 4 +- .../func_match_types/sqlite/go/query.sql.go | 2 +- internal/engine/sqlite/dialect/dialect.json | 2 +- .../engine/sqlite/dialect/functions.jsonl | 12 +++--- .../engine/sqlite/dialect/operators.jsonl | 2 - internal/goldeneye/sqlite/signatures.go | 8 ++++ internal/goldeneye/sqlite/source.go | 17 +++++--- internal/goldeneye/sqlite/sqlite.go | 2 +- 13 files changed, 101 insertions(+), 25 deletions(-) diff --git a/internal/compiler/output_columns.go b/internal/compiler/output_columns.go index 557fb7d575..df67c525ef 100644 --- a/internal/compiler/output_columns.go +++ b/internal/compiler/output_columns.go @@ -3,6 +3,8 @@ package compiler import ( "errors" "fmt" + "strconv" + "strings" "github.com/sqlc-dev/sqlc/internal/config" "github.com/sqlc-dev/sqlc/internal/sql/ast" @@ -324,7 +326,7 @@ func (c *Compiler) outputColumns(qc *QueryCatalog, node ast.Node) ([]*Column, er if err == nil { cols = append(cols, &Column{ Name: name, - DataType: dataType(fun.ReturnType), + DataType: returnDataType(fun, n, tables, rtables), NotNull: !fun.ReturnTypeNullable, IsFuncCall: true, }) @@ -658,6 +660,41 @@ func (c *Compiler) sourceTables(qc *QueryCatalog, node ast.Node) ([]*Table, erro return tables, nil } +// returnDataType is the type a function call produces. A dialect spells a +// function that hands back one of its arguments, as abs and sum do, with a +// return type of "$n", the type of the nth argument, which is known when +// that argument names a column. +func returnDataType(fun *catalog.Function, call *ast.FuncCall, tables, rtables []*Table) string { + dt := dataType(fun.ReturnType) + rest, ok := strings.CutPrefix(dt, "$") + if !ok { + return dt + } + n, err := strconv.Atoi(rest) + if err != nil || n < 1 || call.Args == nil || n > len(call.Args.Items) { + return "any" + } + switch arg := call.Args.Items[n-1].(type) { + case *ast.ColumnRef: + cols, err := outputColumnRefs(&ast.ResTarget{}, tablesForRef(arg, tables, rtables), arg) + if err == nil && len(cols) == 1 { + return cols[0].DataType + } + case *ast.A_Const: + switch arg.Val.(type) { + case *ast.String: + return "text" + case *ast.Integer: + return "int" + case *ast.Float: + return "float" + case *ast.Boolean: + return "bool" + } + } + return "any" +} + func outputColumnRefs(res *ast.ResTarget, tables []*Table, node *ast.ColumnRef) ([]*Column, error) { parts := stringSlice(node.Fields) var schema, name, alias string diff --git a/internal/core/analyzer/expr.go b/internal/core/analyzer/expr.go index d034432662..904993e703 100644 --- a/internal/core/analyzer/expr.go +++ b/internal/core/analyzer/expr.go @@ -285,6 +285,12 @@ func (a *analyzer) typeAExpr(e *ast.A_Expr) (exprType, error) { if err != nil { return exprType{}, err } + // The postfix null tests SQLite has — x ISNULL, x NOTNULL, x NOT NULL — + // arrive as operators with no right operand, and are predicates that + // are never NULL. + if e.Rexpr == nil && isNullTest(opName) { + return a.boolType(false) + } rightT, err := a.typeExpr(e.Rexpr) if err != nil { return exprType{}, err @@ -305,12 +311,25 @@ func (a *analyzer) typeAExpr(e *ast.A_Expr) (exprType, error) { if err != nil { return exprType{}, err } + // An operator's result is NULL when an operand is, except for IS and + // IS NOT, which test for NULL rather than propagate it: x IS NULL and + // x IS y are never NULL, whatever x and y are. return exprType{ typeOID: overload.ResultTypeOID, - nullable: leftT.nullable || rightT.nullable, + nullable: (leftT.nullable || rightT.nullable) && !isNullTest(opName), }, nil } +// isNullTest reports whether an operator compares with NULL as a value +// rather than propagating it, so that its result is never NULL. +func isNullTest(opName string) bool { + switch opName { + case "IS", "IS NOT", "ISNULL", "NOTNULL", "NOT NULL": + return true + } + return false +} + // typeQuantifiedExpr types "x = ANY($1)" and "x > ALL(...)": the right side // holds values of the left side's type, and the result is a predicate. func (a *analyzer) typeQuantifiedExpr(e *ast.A_Expr) (exprType, error) { diff --git a/internal/endtoend/testdata/analyze_expressions/sqlite/query.sql b/internal/endtoend/testdata/analyze_expressions/sqlite/query.sql index 8913215427..04fdc2f8a0 100644 --- a/internal/endtoend/testdata/analyze_expressions/sqlite/query.sql +++ b/internal/endtoend/testdata/analyze_expressions/sqlite/query.sql @@ -3,7 +3,7 @@ SELECT count(*) AS total, max(id) AS latest, min(created) AS first FROM posts WHERE user_id = ?; -- name: UserScores :one -SELECT avg(score) AS mean, sum(score) AS sum, group_concat(name) AS names +SELECT avg(score) AS mean, sum(score) AS sum, sum(id) AS ids, group_concat(name) AS names FROM users; -- name: ListUsers :many diff --git a/internal/endtoend/testdata/analyze_expressions/sqlite/stdout.json b/internal/endtoend/testdata/analyze_expressions/sqlite/stdout.json index 3c6eab97a0..24326932cd 100644 --- a/internal/endtoend/testdata/analyze_expressions/sqlite/stdout.json +++ b/internal/endtoend/testdata/analyze_expressions/sqlite/stdout.json @@ -55,6 +55,13 @@ "nullable": true } }, + { + "name": "ids", + "type": { + "name": "integer", + "nullable": true + } + }, { "name": "names", "type": { diff --git a/internal/endtoend/testdata/builtins/sqlite/go/aggfunc.sql.go b/internal/endtoend/testdata/builtins/sqlite/go/aggfunc.sql.go index de2a1fa89c..0c2abf1aca 100644 --- a/internal/endtoend/testdata/builtins/sqlite/go/aggfunc.sql.go +++ b/internal/endtoend/testdata/builtins/sqlite/go/aggfunc.sql.go @@ -135,9 +135,9 @@ const getSumInt = `-- name: GetSumInt :one SELECT sum(int_val) FROM test ` -func (q *Queries) GetSumInt(ctx context.Context) (sql.NullFloat64, error) { +func (q *Queries) GetSumInt(ctx context.Context) (sql.NullInt64, error) { row := q.db.QueryRowContext(ctx, getSumInt) - var sum sql.NullFloat64 + var sum sql.NullInt64 err := row.Scan(&sum) return sum, err } @@ -146,9 +146,9 @@ const getSumText = `-- name: GetSumText :one SELECT sum(text_val) FROM test ` -func (q *Queries) GetSumText(ctx context.Context) (sql.NullFloat64, error) { +func (q *Queries) GetSumText(ctx context.Context) (sql.NullString, error) { row := q.db.QueryRowContext(ctx, getSumText) - var sum sql.NullFloat64 + var sum sql.NullString err := row.Scan(&sum) return sum, err } diff --git a/internal/endtoend/testdata/builtins/sqlite/go/scalarfunc.sql.go b/internal/endtoend/testdata/builtins/sqlite/go/scalarfunc.sql.go index 06ec0436ce..d3f92da6b0 100644 --- a/internal/endtoend/testdata/builtins/sqlite/go/scalarfunc.sql.go +++ b/internal/endtoend/testdata/builtins/sqlite/go/scalarfunc.sql.go @@ -14,9 +14,9 @@ const getAbs = `-- name: GetAbs :one SELECT abs(int_val) FROM test ` -func (q *Queries) GetAbs(ctx context.Context) (float64, error) { +func (q *Queries) GetAbs(ctx context.Context) (int64, error) { row := q.db.QueryRowContext(ctx, getAbs) - var abs float64 + var abs int64 err := row.Scan(&abs) return abs, err } diff --git a/internal/endtoend/testdata/func_match_types/sqlite/go/query.sql.go b/internal/endtoend/testdata/func_match_types/sqlite/go/query.sql.go index 8e56ea38b6..156216f2d7 100644 --- a/internal/endtoend/testdata/func_match_types/sqlite/go/query.sql.go +++ b/internal/endtoend/testdata/func_match_types/sqlite/go/query.sql.go @@ -19,7 +19,7 @@ GROUP BY author type AuthorPagesRow struct { Author string NumBooks int64 - TotalPages sql.NullFloat64 + TotalPages sql.NullInt64 } func (q *Queries) AuthorPages(ctx context.Context) ([]AuthorPagesRow, error) { diff --git a/internal/engine/sqlite/dialect/dialect.json b/internal/engine/sqlite/dialect/dialect.json index ba49e48300..36129df83a 100644 --- a/internal/engine/sqlite/dialect/dialect.json +++ b/internal/engine/sqlite/dialect/dialect.json @@ -7,7 +7,7 @@ "bool": "boolean" }, "bool": "boolean", - "comparison": ["=", "==", "<>", "!=", "<", "<=", ">", ">=", "IS", "IS NOT"], + "comparison": ["=", "==", "<>", "!=", "<", "<=", ">", ">=", "IS", "IS NOT", "LIKE", "NOT LIKE", "GLOB", "NOT GLOB", "REGEXP", "NOT REGEXP", "MATCH", "NOT MATCH"], "comparison_categories": "BNSDU", "arithmetic": ["+", "-", "*", "/", "%"], "arithmetic_categories": "N", diff --git a/internal/engine/sqlite/dialect/functions.jsonl b/internal/engine/sqlite/dialect/functions.jsonl index da25d4d581..0e3b763dc9 100644 --- a/internal/engine/sqlite/dialect/functions.jsonl +++ b/internal/engine/sqlite/dialect/functions.jsonl @@ -1,6 +1,6 @@ {"name":"-\u003e","args":[{"type":"any"},{"type":"any"}],"returns":"text","nullable":true} {"name":"-\u003e\u003e","args":[{"type":"any"},{"type":"any"}],"returns":"any","nullable":true} -{"name":"abs","args":[{"type":"any"}],"returns":"real"} +{"name":"abs","args":[{"type":"any"}],"returns":"$1"} {"name":"acos","args":[{"type":"real"}],"returns":"real"} {"name":"acosh","args":[{"type":"real"}],"returns":"real"} {"name":"asin","args":[{"type":"real"}],"returns":"real"} @@ -9,8 +9,8 @@ {"name":"atan2","args":[{"type":"real"},{"type":"real"}],"returns":"real"} {"name":"atanh","args":[{"type":"real"}],"returns":"real"} {"name":"avg","kind":"a","args":[{"type":"any"}],"returns":"real","nullable":true} -{"name":"ceil","args":[{"type":"any"}],"returns":"real"} -{"name":"ceiling","args":[{"type":"any"}],"returns":"real"} +{"name":"ceil","args":[{"type":"any"}],"returns":"$1"} +{"name":"ceiling","args":[{"type":"any"}],"returns":"$1"} {"name":"changes","returns":"integer"} {"name":"char","args":[{"type":"integer","mode":"v"}],"returns":"text"} {"name":"coalesce","args":[{"type":"any"},{"type":"any"},{"type":"any","mode":"v"}],"returns":"any"} @@ -30,7 +30,7 @@ {"name":"dense_rank","kind":"w","returns":"integer"} {"name":"exp","args":[{"type":"real"}],"returns":"real"} {"name":"first_value","kind":"w","args":[{"type":"any"}],"returns":"any","nullable":true} -{"name":"floor","args":[{"type":"any"}],"returns":"real"} +{"name":"floor","args":[{"type":"any"}],"returns":"$1"} {"name":"format","args":[{"type":"text","has_default":true},{"type":"any","mode":"v"}],"returns":"text"} {"name":"glob","args":[{"type":"text"},{"type":"text"}],"returns":"integer"} {"name":"group_concat","kind":"a","args":[{"type":"text"}],"returns":"text","nullable":true} @@ -136,7 +136,7 @@ {"name":"substring","args":[{"type":"any"},{"type":"integer"}],"returns":"text"} {"name":"substring","args":[{"type":"any"},{"type":"integer"},{"type":"integer"}],"returns":"text"} {"name":"subtype","args":[{"type":"any"}],"returns":"integer"} -{"name":"sum","kind":"a","args":[{"type":"any"}],"returns":"real","nullable":true} +{"name":"sum","kind":"a","args":[{"type":"any"}],"returns":"$1","nullable":true} {"name":"tan","args":[{"type":"real"}],"returns":"real"} {"name":"tanh","args":[{"type":"real"}],"returns":"real"} {"name":"time","args":[{"type":"any","mode":"v"}],"returns":"text","nullable":true} @@ -145,7 +145,7 @@ {"name":"total_changes","returns":"integer"} {"name":"trim","args":[{"type":"text"}],"returns":"text"} {"name":"trim","args":[{"type":"text"},{"type":"text"}],"returns":"text"} -{"name":"trunc","args":[{"type":"any"}],"returns":"real"} +{"name":"trunc","args":[{"type":"any"}],"returns":"$1"} {"name":"typeof","args":[{"type":"any"}],"returns":"text"} {"name":"unhex","args":[{"type":"text"}],"returns":"blob","nullable":true} {"name":"unhex","args":[{"type":"text"},{"type":"text"}],"returns":"blob","nullable":true} diff --git a/internal/engine/sqlite/dialect/operators.jsonl b/internal/engine/sqlite/dialect/operators.jsonl index 1312de4e53..9ed8beaf64 100644 --- a/internal/engine/sqlite/dialect/operators.jsonl +++ b/internal/engine/sqlite/dialect/operators.jsonl @@ -1,3 +1 @@ {"name": "||", "left": "text", "right": "text", "result": "text"} -{"name": "~~", "left": "text", "right": "text", "result": "boolean"} -{"name": "!~~", "left": "text", "right": "text", "result": "boolean"} diff --git a/internal/goldeneye/sqlite/signatures.go b/internal/goldeneye/sqlite/signatures.go index 121bf36507..1158e27bd6 100644 --- a/internal/goldeneye/sqlite/signatures.go +++ b/internal/goldeneye/sqlite/signatures.go @@ -57,6 +57,14 @@ var inlineReturns = map[string]string{ "INLINEFUNC_sqlite_offset": "integer", } +// realResults are the functions that return an integer or a real on +// something other than which their argument is, and so are typed real +// rather than as the argument: unixepoch takes a time string and returns a +// real only with the 'subsec' modifier. +var realResults = map[string]bool{ + "unixepoch": true, +} + // omitted are functions the dialect leaves out: ones that exist for their // side effect and return nothing a query can use, and ones an extension // uses to pass pointers to itself. diff --git a/internal/goldeneye/sqlite/source.go b/internal/goldeneye/sqlite/source.go index 1599e839f9..7e66a204b5 100644 --- a/internal/goldeneye/sqlite/source.go +++ b/internal/goldeneye/sqlite/source.go @@ -441,10 +441,13 @@ func single(kinds map[string]bool) string { // signature derives what the source says a SQL function returns and takes. // A result of one kind is that type. A function that returns one of its // arguments, or a mixture of kinds, takes the type of its first argument, -// which the seed spells "any" — except that integer and real together widen -// to real, as SQLite's own arithmetic does, and text and blob together to -// text, since a function that returns either is handing back the bytes it -// was given, and the legacy compiler cannot follow "any" to an argument. +// which the seed spells "any" — except that integer and real together are +// the argument's own type, spelled "$1", since abs, ceil and sum hand back +// an integer for an integer and a real for a real, unless the function is +// one of the few whose choice turns on something else; and text and blob +// together are text, since a function that returns either is handing back +// the bytes it was given, and the legacy compiler cannot follow "any" to +// an argument. func (s *source) signature(name string) (signature, error) { r, ok := s.regs[strings.ToLower(name)] if !ok { @@ -481,7 +484,11 @@ func (s *source) signature(name string) (signature, error) { // registration says which this form gets. sig.Returns = r.json case len(kinds) == 2 && kinds["integer"] && kinds["real"]: - sig.Returns = "real" + if realResults[strings.ToLower(name)] { + sig.Returns = "real" + } else { + sig.Returns = "$1" + } case len(kinds) == 2 && kinds["text"] && kinds["blob"]: sig.Returns = "text" default: diff --git a/internal/goldeneye/sqlite/sqlite.go b/internal/goldeneye/sqlite/sqlite.go index 869dc0310f..8b30af4c15 100644 --- a/internal/goldeneye/sqlite/sqlite.go +++ b/internal/goldeneye/sqlite/sqlite.go @@ -396,7 +396,7 @@ func Generate(ctx context.Context, dir string) (dialect.Files, error) { } } var stale []string - for _, list := range []map[string]bool{omitted, nullable} { + for _, list := range []map[string]bool{omitted, nullable, realResults} { for name := range list { if !g.reported[name] { stale = append(stale, name) From 4c5cc40397f126814391711994f98e976fe75f78 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 8 Sep 2026 02:40:20 +0000 Subject: [PATCH 3/5] sqlite: leave the legacy compiler out of the "$1" return type The legacy compiler takes a dialect function's return type as it is spelled, so it is not taught to follow "$1" to an argument; sum, abs, ceil, floor and trunc generate an untyped value on that path, the way max and min already do, and the analysis core resolves them. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01PdKaVjSmdkmXPeUkxetyDY --- internal/compiler/output_columns.go | 39 +------------------ .../builtins/sqlite/go/aggfunc.sql.go | 8 ++-- .../builtins/sqlite/go/mathfunc.sql.go | 16 ++++---- .../builtins/sqlite/go/scalarfunc.sql.go | 4 +- .../func_match_types/sqlite/go/query.sql.go | 3 +- 5 files changed, 16 insertions(+), 54 deletions(-) diff --git a/internal/compiler/output_columns.go b/internal/compiler/output_columns.go index df67c525ef..557fb7d575 100644 --- a/internal/compiler/output_columns.go +++ b/internal/compiler/output_columns.go @@ -3,8 +3,6 @@ package compiler import ( "errors" "fmt" - "strconv" - "strings" "github.com/sqlc-dev/sqlc/internal/config" "github.com/sqlc-dev/sqlc/internal/sql/ast" @@ -326,7 +324,7 @@ func (c *Compiler) outputColumns(qc *QueryCatalog, node ast.Node) ([]*Column, er if err == nil { cols = append(cols, &Column{ Name: name, - DataType: returnDataType(fun, n, tables, rtables), + DataType: dataType(fun.ReturnType), NotNull: !fun.ReturnTypeNullable, IsFuncCall: true, }) @@ -660,41 +658,6 @@ func (c *Compiler) sourceTables(qc *QueryCatalog, node ast.Node) ([]*Table, erro return tables, nil } -// returnDataType is the type a function call produces. A dialect spells a -// function that hands back one of its arguments, as abs and sum do, with a -// return type of "$n", the type of the nth argument, which is known when -// that argument names a column. -func returnDataType(fun *catalog.Function, call *ast.FuncCall, tables, rtables []*Table) string { - dt := dataType(fun.ReturnType) - rest, ok := strings.CutPrefix(dt, "$") - if !ok { - return dt - } - n, err := strconv.Atoi(rest) - if err != nil || n < 1 || call.Args == nil || n > len(call.Args.Items) { - return "any" - } - switch arg := call.Args.Items[n-1].(type) { - case *ast.ColumnRef: - cols, err := outputColumnRefs(&ast.ResTarget{}, tablesForRef(arg, tables, rtables), arg) - if err == nil && len(cols) == 1 { - return cols[0].DataType - } - case *ast.A_Const: - switch arg.Val.(type) { - case *ast.String: - return "text" - case *ast.Integer: - return "int" - case *ast.Float: - return "float" - case *ast.Boolean: - return "bool" - } - } - return "any" -} - func outputColumnRefs(res *ast.ResTarget, tables []*Table, node *ast.ColumnRef) ([]*Column, error) { parts := stringSlice(node.Fields) var schema, name, alias string diff --git a/internal/endtoend/testdata/builtins/sqlite/go/aggfunc.sql.go b/internal/endtoend/testdata/builtins/sqlite/go/aggfunc.sql.go index 0c2abf1aca..c0949d0997 100644 --- a/internal/endtoend/testdata/builtins/sqlite/go/aggfunc.sql.go +++ b/internal/endtoend/testdata/builtins/sqlite/go/aggfunc.sql.go @@ -135,9 +135,9 @@ const getSumInt = `-- name: GetSumInt :one SELECT sum(int_val) FROM test ` -func (q *Queries) GetSumInt(ctx context.Context) (sql.NullInt64, error) { +func (q *Queries) GetSumInt(ctx context.Context) (any, error) { row := q.db.QueryRowContext(ctx, getSumInt) - var sum sql.NullInt64 + var sum any err := row.Scan(&sum) return sum, err } @@ -146,9 +146,9 @@ const getSumText = `-- name: GetSumText :one SELECT sum(text_val) FROM test ` -func (q *Queries) GetSumText(ctx context.Context) (sql.NullString, error) { +func (q *Queries) GetSumText(ctx context.Context) (any, error) { row := q.db.QueryRowContext(ctx, getSumText) - var sum sql.NullString + var sum any err := row.Scan(&sum) return sum, err } diff --git a/internal/endtoend/testdata/builtins/sqlite/go/mathfunc.sql.go b/internal/endtoend/testdata/builtins/sqlite/go/mathfunc.sql.go index 152d80ad46..01d6d4872f 100644 --- a/internal/endtoend/testdata/builtins/sqlite/go/mathfunc.sql.go +++ b/internal/endtoend/testdata/builtins/sqlite/go/mathfunc.sql.go @@ -90,9 +90,9 @@ const getCeil = `-- name: GetCeil :one SELECT ceil(1.0) ` -func (q *Queries) GetCeil(ctx context.Context) (float64, error) { +func (q *Queries) GetCeil(ctx context.Context) (any, error) { row := q.db.QueryRowContext(ctx, getCeil) - var ceil float64 + var ceil any err := row.Scan(&ceil) return ceil, err } @@ -101,9 +101,9 @@ const getCeilin = `-- name: GetCeilin :one SELECT ceiling(1.0) ` -func (q *Queries) GetCeilin(ctx context.Context) (float64, error) { +func (q *Queries) GetCeilin(ctx context.Context) (any, error) { row := q.db.QueryRowContext(ctx, getCeilin) - var ceiling float64 + var ceiling any err := row.Scan(&ceiling) return ceiling, err } @@ -156,9 +156,9 @@ const getFloor = `-- name: GetFloor :one SELECT floor(1.0) ` -func (q *Queries) GetFloor(ctx context.Context) (float64, error) { +func (q *Queries) GetFloor(ctx context.Context) (any, error) { row := q.db.QueryRowContext(ctx, getFloor) - var floor float64 + var floor any err := row.Scan(&floor) return floor, err } @@ -321,9 +321,9 @@ const getTrunc = `-- name: GetTrunc :one SELECT trunc(1.0) ` -func (q *Queries) GetTrunc(ctx context.Context) (float64, error) { +func (q *Queries) GetTrunc(ctx context.Context) (any, error) { row := q.db.QueryRowContext(ctx, getTrunc) - var trunc float64 + var trunc any err := row.Scan(&trunc) return trunc, err } diff --git a/internal/endtoend/testdata/builtins/sqlite/go/scalarfunc.sql.go b/internal/endtoend/testdata/builtins/sqlite/go/scalarfunc.sql.go index d3f92da6b0..3823aef9ba 100644 --- a/internal/endtoend/testdata/builtins/sqlite/go/scalarfunc.sql.go +++ b/internal/endtoend/testdata/builtins/sqlite/go/scalarfunc.sql.go @@ -14,9 +14,9 @@ const getAbs = `-- name: GetAbs :one SELECT abs(int_val) FROM test ` -func (q *Queries) GetAbs(ctx context.Context) (int64, error) { +func (q *Queries) GetAbs(ctx context.Context) (any, error) { row := q.db.QueryRowContext(ctx, getAbs) - var abs int64 + var abs any err := row.Scan(&abs) return abs, err } diff --git a/internal/endtoend/testdata/func_match_types/sqlite/go/query.sql.go b/internal/endtoend/testdata/func_match_types/sqlite/go/query.sql.go index 156216f2d7..46c3aae8bf 100644 --- a/internal/endtoend/testdata/func_match_types/sqlite/go/query.sql.go +++ b/internal/endtoend/testdata/func_match_types/sqlite/go/query.sql.go @@ -7,7 +7,6 @@ package querytest import ( "context" - "database/sql" ) const authorPages = `-- name: AuthorPages :many @@ -19,7 +18,7 @@ GROUP BY author type AuthorPagesRow struct { Author string NumBooks int64 - TotalPages sql.NullInt64 + TotalPages any } func (q *Queries) AuthorPages(ctx context.Context) ([]AuthorPagesRow, error) { From 550bdbb4c8877abb81fbb3becd6e1a631f5e61b0 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 8 Sep 2026 03:33:44 +0000 Subject: [PATCH 4/5] sqlite: write sum and abs once per numeric type instead of as "$1" A function that returns an integer for an integer and a real for a real is now written the way PostgreSQL's catalog writes sum: once over any, returning the real a text or blob argument gets, and once more per spelling types.jsonl gives integer and real, returning that type. The analysis core picks the overload whose parameter is the argument's type, so sum over an INTEGER or BIGINT column is an integer; the legacy compiler resolves by arity alone and takes the first overload, the one over any, so what it generates is unchanged. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01PdKaVjSmdkmXPeUkxetyDY --- .../builtins/sqlite/go/aggfunc.sql.go | 8 +- .../builtins/sqlite/go/mathfunc.sql.go | 16 ++-- .../builtins/sqlite/go/scalarfunc.sql.go | 4 +- .../func_match_types/sqlite/go/query.sql.go | 3 +- .../engine/sqlite/dialect/functions.jsonl | 90 +++++++++++++++++-- internal/goldeneye/README.md | 11 ++- internal/goldeneye/dialect/dialect.go | 19 ++++ internal/goldeneye/sqlite/signatures.go | 8 +- internal/goldeneye/sqlite/source.go | 19 ++-- internal/goldeneye/sqlite/sqlite.go | 39 +++++++- 10 files changed, 178 insertions(+), 39 deletions(-) diff --git a/internal/endtoend/testdata/builtins/sqlite/go/aggfunc.sql.go b/internal/endtoend/testdata/builtins/sqlite/go/aggfunc.sql.go index c0949d0997..de2a1fa89c 100644 --- a/internal/endtoend/testdata/builtins/sqlite/go/aggfunc.sql.go +++ b/internal/endtoend/testdata/builtins/sqlite/go/aggfunc.sql.go @@ -135,9 +135,9 @@ const getSumInt = `-- name: GetSumInt :one SELECT sum(int_val) FROM test ` -func (q *Queries) GetSumInt(ctx context.Context) (any, error) { +func (q *Queries) GetSumInt(ctx context.Context) (sql.NullFloat64, error) { row := q.db.QueryRowContext(ctx, getSumInt) - var sum any + var sum sql.NullFloat64 err := row.Scan(&sum) return sum, err } @@ -146,9 +146,9 @@ const getSumText = `-- name: GetSumText :one SELECT sum(text_val) FROM test ` -func (q *Queries) GetSumText(ctx context.Context) (any, error) { +func (q *Queries) GetSumText(ctx context.Context) (sql.NullFloat64, error) { row := q.db.QueryRowContext(ctx, getSumText) - var sum any + var sum sql.NullFloat64 err := row.Scan(&sum) return sum, err } diff --git a/internal/endtoend/testdata/builtins/sqlite/go/mathfunc.sql.go b/internal/endtoend/testdata/builtins/sqlite/go/mathfunc.sql.go index 01d6d4872f..152d80ad46 100644 --- a/internal/endtoend/testdata/builtins/sqlite/go/mathfunc.sql.go +++ b/internal/endtoend/testdata/builtins/sqlite/go/mathfunc.sql.go @@ -90,9 +90,9 @@ const getCeil = `-- name: GetCeil :one SELECT ceil(1.0) ` -func (q *Queries) GetCeil(ctx context.Context) (any, error) { +func (q *Queries) GetCeil(ctx context.Context) (float64, error) { row := q.db.QueryRowContext(ctx, getCeil) - var ceil any + var ceil float64 err := row.Scan(&ceil) return ceil, err } @@ -101,9 +101,9 @@ const getCeilin = `-- name: GetCeilin :one SELECT ceiling(1.0) ` -func (q *Queries) GetCeilin(ctx context.Context) (any, error) { +func (q *Queries) GetCeilin(ctx context.Context) (float64, error) { row := q.db.QueryRowContext(ctx, getCeilin) - var ceiling any + var ceiling float64 err := row.Scan(&ceiling) return ceiling, err } @@ -156,9 +156,9 @@ const getFloor = `-- name: GetFloor :one SELECT floor(1.0) ` -func (q *Queries) GetFloor(ctx context.Context) (any, error) { +func (q *Queries) GetFloor(ctx context.Context) (float64, error) { row := q.db.QueryRowContext(ctx, getFloor) - var floor any + var floor float64 err := row.Scan(&floor) return floor, err } @@ -321,9 +321,9 @@ const getTrunc = `-- name: GetTrunc :one SELECT trunc(1.0) ` -func (q *Queries) GetTrunc(ctx context.Context) (any, error) { +func (q *Queries) GetTrunc(ctx context.Context) (float64, error) { row := q.db.QueryRowContext(ctx, getTrunc) - var trunc any + var trunc float64 err := row.Scan(&trunc) return trunc, err } diff --git a/internal/endtoend/testdata/builtins/sqlite/go/scalarfunc.sql.go b/internal/endtoend/testdata/builtins/sqlite/go/scalarfunc.sql.go index 3823aef9ba..06ec0436ce 100644 --- a/internal/endtoend/testdata/builtins/sqlite/go/scalarfunc.sql.go +++ b/internal/endtoend/testdata/builtins/sqlite/go/scalarfunc.sql.go @@ -14,9 +14,9 @@ const getAbs = `-- name: GetAbs :one SELECT abs(int_val) FROM test ` -func (q *Queries) GetAbs(ctx context.Context) (any, error) { +func (q *Queries) GetAbs(ctx context.Context) (float64, error) { row := q.db.QueryRowContext(ctx, getAbs) - var abs any + var abs float64 err := row.Scan(&abs) return abs, err } diff --git a/internal/endtoend/testdata/func_match_types/sqlite/go/query.sql.go b/internal/endtoend/testdata/func_match_types/sqlite/go/query.sql.go index 46c3aae8bf..8e56ea38b6 100644 --- a/internal/endtoend/testdata/func_match_types/sqlite/go/query.sql.go +++ b/internal/endtoend/testdata/func_match_types/sqlite/go/query.sql.go @@ -7,6 +7,7 @@ package querytest import ( "context" + "database/sql" ) const authorPages = `-- name: AuthorPages :many @@ -18,7 +19,7 @@ GROUP BY author type AuthorPagesRow struct { Author string NumBooks int64 - TotalPages any + TotalPages sql.NullFloat64 } func (q *Queries) AuthorPages(ctx context.Context) ([]AuthorPagesRow, error) { diff --git a/internal/engine/sqlite/dialect/functions.jsonl b/internal/engine/sqlite/dialect/functions.jsonl index 0e3b763dc9..18012549de 100644 --- a/internal/engine/sqlite/dialect/functions.jsonl +++ b/internal/engine/sqlite/dialect/functions.jsonl @@ -1,6 +1,19 @@ {"name":"-\u003e","args":[{"type":"any"},{"type":"any"}],"returns":"text","nullable":true} {"name":"-\u003e\u003e","args":[{"type":"any"},{"type":"any"}],"returns":"any","nullable":true} -{"name":"abs","args":[{"type":"any"}],"returns":"$1"} +{"name":"abs","args":[{"type":"any"}],"returns":"real"} +{"name":"abs","args":[{"type":"integer"}],"returns":"integer"} +{"name":"abs","args":[{"type":"int"}],"returns":"integer"} +{"name":"abs","args":[{"type":"tinyint"}],"returns":"integer"} +{"name":"abs","args":[{"type":"smallint"}],"returns":"integer"} +{"name":"abs","args":[{"type":"mediumint"}],"returns":"integer"} +{"name":"abs","args":[{"type":"bigint"}],"returns":"integer"} +{"name":"abs","args":[{"type":"unsigned big int"}],"returns":"integer"} +{"name":"abs","args":[{"type":"int2"}],"returns":"integer"} +{"name":"abs","args":[{"type":"int8"}],"returns":"integer"} +{"name":"abs","args":[{"type":"real"}],"returns":"real"} +{"name":"abs","args":[{"type":"double"}],"returns":"real"} +{"name":"abs","args":[{"type":"double precision"}],"returns":"real"} +{"name":"abs","args":[{"type":"float"}],"returns":"real"} {"name":"acos","args":[{"type":"real"}],"returns":"real"} {"name":"acosh","args":[{"type":"real"}],"returns":"real"} {"name":"asin","args":[{"type":"real"}],"returns":"real"} @@ -9,8 +22,34 @@ {"name":"atan2","args":[{"type":"real"},{"type":"real"}],"returns":"real"} {"name":"atanh","args":[{"type":"real"}],"returns":"real"} {"name":"avg","kind":"a","args":[{"type":"any"}],"returns":"real","nullable":true} -{"name":"ceil","args":[{"type":"any"}],"returns":"$1"} -{"name":"ceiling","args":[{"type":"any"}],"returns":"$1"} +{"name":"ceil","args":[{"type":"any"}],"returns":"real"} +{"name":"ceil","args":[{"type":"integer"}],"returns":"integer"} +{"name":"ceil","args":[{"type":"int"}],"returns":"integer"} +{"name":"ceil","args":[{"type":"tinyint"}],"returns":"integer"} +{"name":"ceil","args":[{"type":"smallint"}],"returns":"integer"} +{"name":"ceil","args":[{"type":"mediumint"}],"returns":"integer"} +{"name":"ceil","args":[{"type":"bigint"}],"returns":"integer"} +{"name":"ceil","args":[{"type":"unsigned big int"}],"returns":"integer"} +{"name":"ceil","args":[{"type":"int2"}],"returns":"integer"} +{"name":"ceil","args":[{"type":"int8"}],"returns":"integer"} +{"name":"ceil","args":[{"type":"real"}],"returns":"real"} +{"name":"ceil","args":[{"type":"double"}],"returns":"real"} +{"name":"ceil","args":[{"type":"double precision"}],"returns":"real"} +{"name":"ceil","args":[{"type":"float"}],"returns":"real"} +{"name":"ceiling","args":[{"type":"any"}],"returns":"real"} +{"name":"ceiling","args":[{"type":"integer"}],"returns":"integer"} +{"name":"ceiling","args":[{"type":"int"}],"returns":"integer"} +{"name":"ceiling","args":[{"type":"tinyint"}],"returns":"integer"} +{"name":"ceiling","args":[{"type":"smallint"}],"returns":"integer"} +{"name":"ceiling","args":[{"type":"mediumint"}],"returns":"integer"} +{"name":"ceiling","args":[{"type":"bigint"}],"returns":"integer"} +{"name":"ceiling","args":[{"type":"unsigned big int"}],"returns":"integer"} +{"name":"ceiling","args":[{"type":"int2"}],"returns":"integer"} +{"name":"ceiling","args":[{"type":"int8"}],"returns":"integer"} +{"name":"ceiling","args":[{"type":"real"}],"returns":"real"} +{"name":"ceiling","args":[{"type":"double"}],"returns":"real"} +{"name":"ceiling","args":[{"type":"double precision"}],"returns":"real"} +{"name":"ceiling","args":[{"type":"float"}],"returns":"real"} {"name":"changes","returns":"integer"} {"name":"char","args":[{"type":"integer","mode":"v"}],"returns":"text"} {"name":"coalesce","args":[{"type":"any"},{"type":"any"},{"type":"any","mode":"v"}],"returns":"any"} @@ -30,7 +69,20 @@ {"name":"dense_rank","kind":"w","returns":"integer"} {"name":"exp","args":[{"type":"real"}],"returns":"real"} {"name":"first_value","kind":"w","args":[{"type":"any"}],"returns":"any","nullable":true} -{"name":"floor","args":[{"type":"any"}],"returns":"$1"} +{"name":"floor","args":[{"type":"any"}],"returns":"real"} +{"name":"floor","args":[{"type":"integer"}],"returns":"integer"} +{"name":"floor","args":[{"type":"int"}],"returns":"integer"} +{"name":"floor","args":[{"type":"tinyint"}],"returns":"integer"} +{"name":"floor","args":[{"type":"smallint"}],"returns":"integer"} +{"name":"floor","args":[{"type":"mediumint"}],"returns":"integer"} +{"name":"floor","args":[{"type":"bigint"}],"returns":"integer"} +{"name":"floor","args":[{"type":"unsigned big int"}],"returns":"integer"} +{"name":"floor","args":[{"type":"int2"}],"returns":"integer"} +{"name":"floor","args":[{"type":"int8"}],"returns":"integer"} +{"name":"floor","args":[{"type":"real"}],"returns":"real"} +{"name":"floor","args":[{"type":"double"}],"returns":"real"} +{"name":"floor","args":[{"type":"double precision"}],"returns":"real"} +{"name":"floor","args":[{"type":"float"}],"returns":"real"} {"name":"format","args":[{"type":"text","has_default":true},{"type":"any","mode":"v"}],"returns":"text"} {"name":"glob","args":[{"type":"text"},{"type":"text"}],"returns":"integer"} {"name":"group_concat","kind":"a","args":[{"type":"text"}],"returns":"text","nullable":true} @@ -136,7 +188,20 @@ {"name":"substring","args":[{"type":"any"},{"type":"integer"}],"returns":"text"} {"name":"substring","args":[{"type":"any"},{"type":"integer"},{"type":"integer"}],"returns":"text"} {"name":"subtype","args":[{"type":"any"}],"returns":"integer"} -{"name":"sum","kind":"a","args":[{"type":"any"}],"returns":"$1","nullable":true} +{"name":"sum","kind":"a","args":[{"type":"any"}],"returns":"real","nullable":true} +{"name":"sum","kind":"a","args":[{"type":"integer"}],"returns":"integer","nullable":true} +{"name":"sum","kind":"a","args":[{"type":"int"}],"returns":"integer","nullable":true} +{"name":"sum","kind":"a","args":[{"type":"tinyint"}],"returns":"integer","nullable":true} +{"name":"sum","kind":"a","args":[{"type":"smallint"}],"returns":"integer","nullable":true} +{"name":"sum","kind":"a","args":[{"type":"mediumint"}],"returns":"integer","nullable":true} +{"name":"sum","kind":"a","args":[{"type":"bigint"}],"returns":"integer","nullable":true} +{"name":"sum","kind":"a","args":[{"type":"unsigned big int"}],"returns":"integer","nullable":true} +{"name":"sum","kind":"a","args":[{"type":"int2"}],"returns":"integer","nullable":true} +{"name":"sum","kind":"a","args":[{"type":"int8"}],"returns":"integer","nullable":true} +{"name":"sum","kind":"a","args":[{"type":"real"}],"returns":"real","nullable":true} +{"name":"sum","kind":"a","args":[{"type":"double"}],"returns":"real","nullable":true} +{"name":"sum","kind":"a","args":[{"type":"double precision"}],"returns":"real","nullable":true} +{"name":"sum","kind":"a","args":[{"type":"float"}],"returns":"real","nullable":true} {"name":"tan","args":[{"type":"real"}],"returns":"real"} {"name":"tanh","args":[{"type":"real"}],"returns":"real"} {"name":"time","args":[{"type":"any","mode":"v"}],"returns":"text","nullable":true} @@ -145,7 +210,20 @@ {"name":"total_changes","returns":"integer"} {"name":"trim","args":[{"type":"text"}],"returns":"text"} {"name":"trim","args":[{"type":"text"},{"type":"text"}],"returns":"text"} -{"name":"trunc","args":[{"type":"any"}],"returns":"$1"} +{"name":"trunc","args":[{"type":"any"}],"returns":"real"} +{"name":"trunc","args":[{"type":"integer"}],"returns":"integer"} +{"name":"trunc","args":[{"type":"int"}],"returns":"integer"} +{"name":"trunc","args":[{"type":"tinyint"}],"returns":"integer"} +{"name":"trunc","args":[{"type":"smallint"}],"returns":"integer"} +{"name":"trunc","args":[{"type":"mediumint"}],"returns":"integer"} +{"name":"trunc","args":[{"type":"bigint"}],"returns":"integer"} +{"name":"trunc","args":[{"type":"unsigned big int"}],"returns":"integer"} +{"name":"trunc","args":[{"type":"int2"}],"returns":"integer"} +{"name":"trunc","args":[{"type":"int8"}],"returns":"integer"} +{"name":"trunc","args":[{"type":"real"}],"returns":"real"} +{"name":"trunc","args":[{"type":"double"}],"returns":"real"} +{"name":"trunc","args":[{"type":"double precision"}],"returns":"real"} +{"name":"trunc","args":[{"type":"float"}],"returns":"real"} {"name":"typeof","args":[{"type":"any"}],"returns":"text"} {"name":"unhex","args":[{"type":"text"}],"returns":"blob","nullable":true} {"name":"unhex","args":[{"type":"text"},{"type":"text"}],"returns":"blob","nullable":true} diff --git a/internal/goldeneye/README.md b/internal/goldeneye/README.md index a9ddf3bb23..6e0276573f 100644 --- a/internal/goldeneye/README.md +++ b/internal/goldeneye/README.md @@ -80,8 +80,15 @@ the hand-written files alone, and the checks do not look at them. `sqlite/signatures.go`, since a SQLite function returns NULL as often by setting no result as by saying so. The pinned release is the one the main module's driver embeds. SQLite has no catalog of types or operators, so - `types.jsonl` and `operators.jsonl` are hand-written. `install` builds one - more shell, for the analysis check below; nothing is generated from it. + `types.jsonl` and `operators.jsonl` are hand-written. A function that + returns an integer for an integer and a real for a real — `abs`, `ceil`, + `floor`, `trunc`, `sum` — is written once over `any`, returning the real + that a text or blob argument gets, and once more per spelling + `types.jsonl` gives integer and real, returning that type, the way + PostgreSQL's catalog has a `sum` per numeric type; the overload over `any` + comes first, since the legacy compiler resolves by arity alone and takes + it. `install` builds one more shell, for the analysis check below; + nothing is generated from it. ## Layout diff --git a/internal/goldeneye/dialect/dialect.go b/internal/goldeneye/dialect/dialect.go index 3db8aab463..6085ec9b2f 100644 --- a/internal/goldeneye/dialect/dialect.go +++ b/internal/goldeneye/dialect/dialect.go @@ -118,6 +118,25 @@ func JSONL[T any](records []T) ([]byte, error) { return buf.Bytes(), nil } +// ReadTypes reads the hand-written types.jsonl of a dialect directory, for +// a generator that writes a function once per spelling of a type. +func ReadTypes(dir string) ([]Type, error) { + blob, err := os.ReadFile(filepath.Join(dir, TypesFile)) + if err != nil { + return nil, err + } + var types []Type + dec := json.NewDecoder(bytes.NewReader(blob)) + for dec.More() { + var t Type + if err := dec.Decode(&t); err != nil { + return nil, fmt.Errorf("%s: %w", filepath.Join(dir, TypesFile), err) + } + types = append(types, t) + } + return types, nil +} + // Dir returns the dialect directory of an engine, // internal/engine//dialect, found relative to this source file so // the working directory does not matter. diff --git a/internal/goldeneye/sqlite/signatures.go b/internal/goldeneye/sqlite/signatures.go index 1158e27bd6..5551061bd5 100644 --- a/internal/goldeneye/sqlite/signatures.go +++ b/internal/goldeneye/sqlite/signatures.go @@ -10,10 +10,14 @@ import "github.com/sqlc-dev/sqlc/internal/goldeneye/dialect" // many arguments an overload takes, and how many of them it requires, // comes from the shell. Nullable is decided afterwards: for an aggregate by // running it over no rows, for a scalar by the nullable list below. +// Numeric says the function returns an integer for an integer argument +// and a real for a real one, and Returns what it returns for anything +// else. type signature struct { Args []string Variadic string Returns string + Numeric bool Nullable bool } @@ -59,8 +63,8 @@ var inlineReturns = map[string]string{ // realResults are the functions that return an integer or a real on // something other than which their argument is, and so are typed real -// rather than as the argument: unixepoch takes a time string and returns a -// real only with the 'subsec' modifier. +// alone: unixepoch takes a time string and returns a real only with the +// 'subsec' modifier. var realResults = map[string]bool{ "unixepoch": true, } diff --git a/internal/goldeneye/sqlite/source.go b/internal/goldeneye/sqlite/source.go index 7e66a204b5..d7c551891e 100644 --- a/internal/goldeneye/sqlite/source.go +++ b/internal/goldeneye/sqlite/source.go @@ -442,12 +442,12 @@ func single(kinds map[string]bool) string { // A result of one kind is that type. A function that returns one of its // arguments, or a mixture of kinds, takes the type of its first argument, // which the seed spells "any" — except that integer and real together are -// the argument's own type, spelled "$1", since abs, ceil and sum hand back -// an integer for an integer and a real for a real, unless the function is -// one of the few whose choice turns on something else; and text and blob -// together are text, since a function that returns either is handing back -// the bytes it was given, and the legacy compiler cannot follow "any" to -// an argument. +// real, and Numeric, since abs, ceil and sum hand back an integer for an +// integer and a real for a real, and a real for the text or blob they +// convert, unless the function is one of the few whose choice turns on +// something else; and text and blob together are text, since a function +// that returns either is handing back the bytes it was given, and the +// legacy compiler cannot follow "any" to an argument. func (s *source) signature(name string) (signature, error) { r, ok := s.regs[strings.ToLower(name)] if !ok { @@ -484,11 +484,8 @@ func (s *source) signature(name string) (signature, error) { // registration says which this form gets. sig.Returns = r.json case len(kinds) == 2 && kinds["integer"] && kinds["real"]: - if realResults[strings.ToLower(name)] { - sig.Returns = "real" - } else { - sig.Returns = "$1" - } + sig.Returns = "real" + sig.Numeric = !realResults[strings.ToLower(name)] case len(kinds) == 2 && kinds["text"] && kinds["blob"]: sig.Returns = "text" default: diff --git a/internal/goldeneye/sqlite/sqlite.go b/internal/goldeneye/sqlite/sqlite.go index 8b30af4c15..c176a55b36 100644 --- a/internal/goldeneye/sqlite/sqlite.go +++ b/internal/goldeneye/sqlite/sqlite.go @@ -231,6 +231,9 @@ type generator struct { ctx context.Context src *source reported map[string]bool + // numeric holds every spelling types.jsonl gives integer and real, in + // its order, for the functions written once per spelling. + numeric map[string][]string } // overload is one row of a shell's list with what was found out about it. @@ -283,13 +286,30 @@ func (g *generator) functions(s *shell, base bool) ([]dialect.Function, error) { if o.kind == "a" { isNullable = empty[o.row.key()] == "null" } - funcs = append(funcs, dialect.Function{ + fn := dialect.Function{ Name: o.row.Name, Kind: o.kind, Args: o.sig.args(o.row.NArg), Returns: o.sig.Returns, Nullable: isNullable, - }) + } + funcs = append(funcs, fn) + // A function that returns an integer for an integer and a real for + // a real is written once more per spelling of each, the way + // PostgreSQL's catalog has a sum per numeric type. The overload over + // any comes first: the analysis core picks the overload whose + // parameter is the argument's type, but the legacy compiler takes + // the first of the right arity, and keeps what it had. + if o.sig.Numeric && len(fn.Args) > 0 { + for _, typ := range []string{"integer", "real"} { + for _, spelling := range g.numeric[typ] { + typed := fn + typed.Args = append([]dialect.Arg{{Type: spelling}}, fn.Args[1:]...) + typed.Returns = typ + funcs = append(funcs, typed) + } + } + } } return funcs, nil } @@ -365,7 +385,20 @@ func Generate(ctx context.Context, dir string) (dialect.Files, error) { if err != nil { return nil, err } - g := &generator{ctx: ctx, src: src, reported: map[string]bool{}} + g := &generator{ctx: ctx, src: src, reported: map[string]bool{}, numeric: map[string][]string{}} + dialectDir, err := dialect.Dir(Engine) + if err != nil { + return nil, err + } + types, err := dialect.ReadTypes(dialectDir) + if err != nil { + return nil, err + } + for _, t := range types { + if t.Name == "integer" || t.Name == "real" { + g.numeric[t.Name] = append([]string{t.Name}, t.Aliases...) + } + } all := builds() base, err := readShell(ctx, dir, all[0]) if err != nil { From 81f0531cf2dfeb0470d410f96a668f670be8a25d Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 9 Sep 2026 03:29:48 +0000 Subject: [PATCH 5/5] goldeneye: move the shape of an engine's answer into its own package MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The JSON sqlc analyze prints — a query, its columns, its parameters and their types — lived in endtoend beside the case finder. It is its own thing, so it moves to the analysis package, and the Analyzed prefix goes with it: analysis.Query, analysis.Column, analysis.Param. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01PdKaVjSmdkmXPeUkxetyDY --- internal/goldeneye/README.md | 4 +- .../output.go => analysis/analysis.go} | 37 +++++++------- internal/goldeneye/clickhouse/analyze.go | 51 ++++++++++--------- internal/goldeneye/clickhouse/check.go | 3 +- internal/goldeneye/clickhouse/types.go | 16 +++--- internal/goldeneye/sqlite/analyze.go | 43 ++++++++-------- internal/goldeneye/sqlite/bytecode.go | 28 +++++----- internal/goldeneye/sqlite/catalog.go | 24 ++++----- internal/goldeneye/sqlite/install.go | 6 +-- 9 files changed, 107 insertions(+), 105 deletions(-) rename internal/goldeneye/{endtoend/output.go => analysis/analysis.go} (58%) diff --git a/internal/goldeneye/README.md b/internal/goldeneye/README.md index 6e0276573f..7139a16ac9 100644 --- a/internal/goldeneye/README.md +++ b/internal/goldeneye/README.md @@ -96,8 +96,8 @@ the hand-written files alone, and the checks do not look at them. `internal/core/seed`, and the helpers that write a generated set of files into an engine directory or diff it against what is committed. - `endtoend/` — finds the analyze cases, splits their query files, and - holds the shape of an engine's answer, which it compares with a case's - committed output. + compares an engine's answer with a case's committed output. +- `analysis/` — the shape of that answer: the JSON `sqlc analyze` prints. - `postgresql/`, `duckdb/`, `clickhouse/`, `sqlite/` — one package per engine, each exposing `Locate`, `Version` and `Generate`, `Analyze` where the engine has an analysis check, and tests that run the checks. diff --git a/internal/goldeneye/endtoend/output.go b/internal/goldeneye/analysis/analysis.go similarity index 58% rename from internal/goldeneye/endtoend/output.go rename to internal/goldeneye/analysis/analysis.go index 170610bcee..fe907766a3 100644 --- a/internal/goldeneye/endtoend/output.go +++ b/internal/goldeneye/analysis/analysis.go @@ -1,34 +1,33 @@ -package endtoend +// Package analysis is the shape of an engine's answer about a query: the +// JSON `sqlc analyze` prints, so that a case's committed stdout.json can be +// compared with what the database itself reports, byte for byte. +package analysis import ( "bytes" "encoding/json" ) -// An engine's answer is written in the JSON `sqlc analyze` prints, so that -// a case's committed stdout.json can be compared with it byte for byte. - -// AnalyzedQuery is what was found out about one query. -type AnalyzedQuery struct { - Name string `json:"name"` - Cmd string `json:"cmd"` - Columns []AnalyzedColumn `json:"columns"` - Params []AnalyzedParam `json:"params"` +// Query is what was found out about one query. +type Query struct { + Name string `json:"name"` + Cmd string `json:"cmd"` + Columns []Column `json:"columns"` + Params []Param `json:"params"` } -// AnalyzedColumn describes a result column, or the column a parameter -// stands in for. -type AnalyzedColumn struct { +// Column describes a result column, or the column a parameter stands in +// for. +type Column struct { Name string `json:"name"` Type *TypeExpr `json:"type,omitempty"` Table string `json:"table,omitempty"` } -// AnalyzedParam is one parameter and what it is compared with or assigned -// to. -type AnalyzedParam struct { - Number int `json:"number"` - Column AnalyzedColumn `json:"column"` +// Param is one parameter and what it is compared with or assigned to. +type Param struct { + Number int `json:"number"` + Column Column `json:"column"` } // TypeExpr is a type as a call expression: a lowercased name applied to an @@ -51,7 +50,7 @@ type TypeArg struct { } // Encode prints the answer the way sqlc analyze does. -func Encode(queries []AnalyzedQuery) ([]byte, error) { +func Encode(queries []Query) ([]byte, error) { var buf bytes.Buffer enc := json.NewEncoder(&buf) enc.SetIndent("", " ") diff --git a/internal/goldeneye/clickhouse/analyze.go b/internal/goldeneye/clickhouse/analyze.go index 0006bf6be0..ff2148fdf5 100644 --- a/internal/goldeneye/clickhouse/analyze.go +++ b/internal/goldeneye/clickhouse/analyze.go @@ -8,13 +8,14 @@ import ( "strconv" "strings" + "github.com/sqlc-dev/sqlc/internal/goldeneye/analysis" "github.com/sqlc-dev/sqlc/internal/goldeneye/endtoend" ) // analyze runs every query against the schema and fixture and records what // ClickHouse reports about each. -func analyze(ctx context.Context, l local, schema, fixture string, queries []endtoend.Query) ([]endtoend.AnalyzedQuery, error) { - out := make([]endtoend.AnalyzedQuery, 0, len(queries)) +func analyze(ctx context.Context, l local, schema, fixture string, queries []endtoend.Query) ([]analysis.Query, error) { + out := make([]analysis.Query, 0, len(queries)) for _, q := range queries { aq, err := analyzeQuery(ctx, l, schema, fixture, q) if err != nil { @@ -25,7 +26,7 @@ func analyze(ctx context.Context, l local, schema, fixture string, queries []end return out, nil } -func analyzeQuery(ctx context.Context, l local, schema, fixture string, q endtoend.Query) (endtoend.AnalyzedQuery, error) { +func analyzeQuery(ctx context.Context, l local, schema, fixture string, q endtoend.Query) (analysis.Query, error) { sql, phs := bindPlaceholders(q.SQL) explain := returnsRows(sql) @@ -43,33 +44,33 @@ func analyzeQuery(ctx context.Context, l local, schema, fixture string, q endtoe results, err := l.run(ctx, script.String()) if err != nil { - return endtoend.AnalyzedQuery{}, err + return analysis.Query{}, err } - aq := endtoend.AnalyzedQuery{ + aq := analysis.Query{ Name: q.Name, Cmd: q.Cmd, - Columns: []endtoend.AnalyzedColumn{}, - Params: []endtoend.AnalyzedParam{}, + Columns: []analysis.Column{}, + Params: []analysis.Param{}, } if !explain { return analyzeExec(ctx, l, script.String(), sql, phs, aq) } if len(results) != 2 { - return endtoend.AnalyzedQuery{}, fmt.Errorf("expected the query tree and one result set, got %d results", len(results)) + return analysis.Query{}, fmt.Errorf("expected the query tree and one result set, got %d results", len(results)) } var lines []string for _, row := range results[0].Data { var line string if err := json.Unmarshal(row["explain"], &line); err != nil { - return endtoend.AnalyzedQuery{}, fmt.Errorf("reading query tree: %w", err) + return analysis.Query{}, fmt.Errorf("reading query tree: %w", err) } lines = append(lines, line) } tree, err := parseQueryTree(lines) if err != nil { - return endtoend.AnalyzedQuery{}, err + return analysis.Query{}, err } // Names and types come from the block header of the executed query, the @@ -85,23 +86,23 @@ func analyzeQuery(ctx context.Context, l local, schema, fixture string, q endtoe sentinels := tree.sentinels() for i, ph := range phs { - ac := endtoend.AnalyzedColumn{} + ac := analysis.Column{} if sentinel := sentinels[i+1]; sentinel != nil { ac = tree.paramColumn(sentinel) } if ph.Name != "" { ac.Name = ph.Name } - aq.Params = append(aq.Params, endtoend.AnalyzedParam{Number: ph.Number, Column: ac}) + aq.Params = append(aq.Params, analysis.Param{Number: ph.Number, Column: ac}) } return aq, nil } -func column(name, typ string) endtoend.AnalyzedColumn { +func column(name, typ string) analysis.Column { if typ == "" { - return endtoend.AnalyzedColumn{Name: name} + return analysis.Column{Name: name} } - return endtoend.AnalyzedColumn{Name: name, Type: parseType(typ)} + return analysis.Column{Name: name, Type: parseType(typ)} } // returnsRows reports whether a statement produces a result set and so can @@ -166,7 +167,7 @@ func sentinelOrdinal(c *treeNode) (int, bool) { // paramColumn describes what a placeholder is compared with or assigned to: // the other operand of the function it is an argument of, preferring a // column over an expression, or the projected column it stands for. -func (t *queryTree) paramColumn(sentinel *treeNode) endtoend.AnalyzedColumn { +func (t *queryTree) paramColumn(sentinel *treeNode) analysis.Column { list := sentinel.parent if list != nil && list.kind == "LIST" && list.parent != nil { switch owner := list.parent; { @@ -199,7 +200,7 @@ func (t *queryTree) paramColumn(sentinel *treeNode) endtoend.AnalyzedColumn { } // describe turns a tree expression into a column description. -func (t *queryTree) describe(n *treeNode) endtoend.AnalyzedColumn { +func (t *queryTree) describe(n *treeNode) analysis.Column { switch n.kind { case "COLUMN": ac := column(n.attrs["column_name"], n.attrs["result_type"]) @@ -217,7 +218,7 @@ func (t *queryTree) describe(n *treeNode) endtoend.AnalyzedColumn { } return column(name, n.attrs["constant_value_type"]) } - return endtoend.AnalyzedColumn{} + return analysis.Column{} } var insertValuesRe = regexp.MustCompile(`(?is)^insert\s+into\s+(?:table\s+)?([\w.` + "`" + `"]+)\s*(?:\(([^)]*)\))?\s*(?:format\s+)?values\b`) @@ -225,20 +226,20 @@ var insertValuesRe = regexp.MustCompile(`(?is)^insert\s+into\s+(?:table\s+)?([\w // analyzeExec runs a statement that returns no rows. The only parameters it // can describe are those of an INSERT ... VALUES, which map positionally // onto the target columns reported by DESCRIBE TABLE. -func analyzeExec(ctx context.Context, l local, script, sql string, phs []placeholder, aq endtoend.AnalyzedQuery) (endtoend.AnalyzedQuery, error) { +func analyzeExec(ctx context.Context, l local, script, sql string, phs []placeholder, aq analysis.Query) (analysis.Query, error) { m := insertValuesRe.FindStringSubmatch(sql) if m != nil { script += "DESCRIBE TABLE " + m[1] + ";\n" } results, err := l.run(ctx, script) if err != nil { - return endtoend.AnalyzedQuery{}, err + return analysis.Query{}, err } - var targets []endtoend.AnalyzedColumn + var targets []analysis.Column if m != nil && len(results) == 1 { - byName := map[string]endtoend.AnalyzedColumn{} - var all []endtoend.AnalyzedColumn + byName := map[string]analysis.Column{} + var all []analysis.Column table := strings.Trim(m[1][strings.LastIndexByte(m[1], '.')+1:], "`\"") for _, row := range results[0].Data { var name, typ string @@ -258,14 +259,14 @@ func analyzeExec(ctx context.Context, l local, script, sql string, phs []placeho } } for i, ph := range phs { - ac := endtoend.AnalyzedColumn{} + ac := analysis.Column{} if len(targets) > 0 { ac = targets[i%len(targets)] } if ph.Name != "" { ac.Name = ph.Name } - aq.Params = append(aq.Params, endtoend.AnalyzedParam{Number: ph.Number, Column: ac}) + aq.Params = append(aq.Params, analysis.Param{Number: ph.Number, Column: ac}) } return aq, nil } diff --git a/internal/goldeneye/clickhouse/check.go b/internal/goldeneye/clickhouse/check.go index be007ba6eb..fab53b8a08 100644 --- a/internal/goldeneye/clickhouse/check.go +++ b/internal/goldeneye/clickhouse/check.go @@ -4,6 +4,7 @@ import ( "context" "os" + "github.com/sqlc-dev/sqlc/internal/goldeneye/analysis" "github.com/sqlc-dev/sqlc/internal/goldeneye/endtoend" ) @@ -28,7 +29,7 @@ func Analyze(ctx context.Context, binary string, c endtoend.Case) ([]byte, error if err != nil { return nil, err } - return endtoend.Encode(out) + return analysis.Encode(out) } // Check compares what ClickHouse reports for a case with the output the diff --git a/internal/goldeneye/clickhouse/types.go b/internal/goldeneye/clickhouse/types.go index d011d06870..57f6caf731 100644 --- a/internal/goldeneye/clickhouse/types.go +++ b/internal/goldeneye/clickhouse/types.go @@ -4,7 +4,7 @@ import ( "strconv" "strings" - "github.com/sqlc-dev/sqlc/internal/goldeneye/endtoend" + "github.com/sqlc-dev/sqlc/internal/goldeneye/analysis" ) // A type is a call expression, the way ClickHouse itself models one: a @@ -38,7 +38,7 @@ import ( // is the reader's job; the output only records what was said. // parseType turns a ClickHouse type string into its expression. -func parseType(t string) *endtoend.TypeExpr { +func parseType(t string) *analysis.TypeExpr { name, args := splitType(t) name = strings.ToLower(strings.TrimSpace(name)) if name == "nullable" && len(args) == 1 { @@ -49,7 +49,7 @@ func parseType(t string) *endtoend.TypeExpr { if name == "" { name = "nothing" } - expr := &endtoend.TypeExpr{Name: name} + expr := &analysis.TypeExpr{Name: name} for _, a := range args { expr.Args = append(expr.Args, parseArg(a)) } @@ -59,7 +59,7 @@ func parseType(t string) *endtoend.TypeExpr { // parseArg parses one argument: a quoted string, an integer, a boolean, a // labelled argument (`lat Float64` in a Tuple, `'a' = 1` in an Enum), or a // type. -func parseArg(a string) endtoend.TypeArg { +func parseArg(a string) analysis.TypeArg { a = strings.TrimSpace(a) if strings.HasPrefix(a, "'") { end := skipQuoted(a, 0) @@ -69,22 +69,22 @@ func parseArg(a string) endtoend.TypeArg { arg.Label = lit return arg } - return endtoend.TypeArg{String: &lit} + return analysis.TypeArg{String: &lit} } if n, err := strconv.ParseInt(a, 10, 64); err == nil { - return endtoend.TypeArg{Int: &n} + return analysis.TypeArg{Int: &n} } switch strings.ToLower(a) { case "true", "false": b := strings.EqualFold(a, "true") - return endtoend.TypeArg{Bool: &b} + return analysis.TypeArg{Bool: &b} } if i := labelEnd(a); i > 0 { arg := parseArg(a[i+1:]) arg.Label = a[:i] return arg } - return endtoend.TypeArg{Type: parseType(a)} + return analysis.TypeArg{Type: parseType(a)} } // labelEnd returns the index of the space separating a label from the type diff --git a/internal/goldeneye/sqlite/analyze.go b/internal/goldeneye/sqlite/analyze.go index 308a293bde..b1a8003dff 100644 --- a/internal/goldeneye/sqlite/analyze.go +++ b/internal/goldeneye/sqlite/analyze.go @@ -7,6 +7,7 @@ import ( "os" "strings" + "github.com/sqlc-dev/sqlc/internal/goldeneye/analysis" "github.com/sqlc-dev/sqlc/internal/goldeneye/endtoend" ) @@ -51,10 +52,10 @@ func bind(sql string) (string, []placeholder) { // Analyze runs a case's queries through the analysis shell and returns what // SQLite reports in the JSON shape sqlc analyze prints. func Analyze(ctx context.Context, dir string, c endtoend.Case) ([]byte, error) { - if err := checkOptions(ctx, dir, analysis); err != nil { + if err := checkOptions(ctx, dir, analysisShell); err != nil { return nil, err } - binary := analysis.binary(dir) + binary := analysisShell.binary(dir) schema, err := os.ReadFile(c.Schema) if err != nil { return nil, err @@ -69,7 +70,7 @@ func Analyze(ctx context.Context, dir string, c endtoend.Case) ([]byte, error) { if err != nil { return nil, err } - out := make([]endtoend.AnalyzedQuery, 0, len(queries)) + out := make([]analysis.Query, 0, len(queries)) for _, q := range queries { aq, err := analyzeQuery(ctx, binary, string(schema), string(fixture), q) if err != nil { @@ -77,7 +78,7 @@ func Analyze(ctx context.Context, dir string, c endtoend.Case) ([]byte, error) { } out = append(out, aq) } - return endtoend.Encode(out) + return analysis.Encode(out) } // Check compares what SQLite reports for a case with the output the case @@ -90,7 +91,7 @@ func Check(ctx context.Context, dir string, c endtoend.Case) (string, error) { return c.Compare(got) } -func analyzeQuery(ctx context.Context, binary, schema, fixture string, q endtoend.Query) (endtoend.AnalyzedQuery, error) { +func analyzeQuery(ctx context.Context, binary, schema, fixture string, q endtoend.Query) (analysis.Query, error) { sql, phs := bind(q.SQL) // What the library says: the catalog, the bytecode, and the statement @@ -112,10 +113,10 @@ func analyzeQuery(ctx context.Context, binary, schema, fixture string, q endtoen s.section("end") out, err := run(ctx, binary, s) if err != nil { - return endtoend.AnalyzedQuery{}, err + return analysis.Query{}, err } if errs := out.errorsBefore(line); len(errs) > 0 { - return endtoend.AnalyzedQuery{}, errors.New(strings.Join(errs, "\n")) + return analysis.Query{}, errors.New(strings.Join(errs, "\n")) } query := out.sections["query"] if query == nil || !query.prepared { @@ -123,16 +124,16 @@ func analyzeQuery(ctx context.Context, binary, schema, fixture string, q endtoen if msg == "" { msg = "the statement was not prepared" } - return endtoend.AnalyzedQuery{}, errors.New(msg) + return analysis.Query{}, errors.New(msg) } cat, err := readCatalog(out) if err != nil { - return endtoend.AnalyzedQuery{}, err + return analysis.Query{}, err } var prog []instr if explain := out.sections["explain"]; explain != nil && len(explain.blocks) > 0 { if err := explain.decode(0, &prog); err != nil { - return endtoend.AnalyzedQuery{}, fmt.Errorf("reading the bytecode: %w", err) + return analysis.Query{}, fmt.Errorf("reading the bytecode: %w", err) } } var names []string @@ -141,7 +142,7 @@ func analyzeQuery(ctx context.Context, binary, schema, fixture string, q endtoen } t := newTracer(cat, names) t.run(prog) - params := make([]endtoend.AnalyzedColumn, len(phs)) + params := make([]analysis.Column, len(phs)) for i, ph := range phs { params[i] = t.param(ph.Number) } @@ -164,7 +165,7 @@ func analyzeQuery(ctx context.Context, binary, schema, fixture string, q endtoen s.section("end") bound, err := run(ctx, binary, s) if err != nil { - return endtoend.AnalyzedQuery{}, err + return analysis.Query{}, err } s = newScript() s.sql(schema) @@ -174,7 +175,7 @@ func analyzeQuery(ctx context.Context, binary, schema, fixture string, q endtoen s.section("end") empty, err := run(ctx, binary, s) if err != nil { - return endtoend.AnalyzedQuery{}, err + return analysis.Query{}, err } var rows []string for _, o := range []*output{bound, empty} { @@ -183,11 +184,11 @@ func analyzeQuery(ctx context.Context, binary, schema, fixture string, q endtoen } } - aq := endtoend.AnalyzedQuery{ + aq := analysis.Query{ Name: q.Name, Cmd: q.Cmd, - Columns: []endtoend.AnalyzedColumn{}, - Params: []endtoend.AnalyzedParam{}, + Columns: []analysis.Column{}, + Params: []analysis.Param{}, } for i, m := range query.columns { aq.Columns = append(aq.Columns, describeColumn(cat, m, classes(rows, i))) @@ -197,7 +198,7 @@ func analyzeQuery(ctx context.Context, binary, schema, fixture string, q endtoen if ph.Name != "" { ac.Name = ph.Name } - aq.Params = append(aq.Params, endtoend.AnalyzedParam{Number: ph.Number, Column: ac}) + aq.Params = append(aq.Params, analysis.Param{Number: ph.Number, Column: ac}) } return aq, nil } @@ -205,7 +206,7 @@ func analyzeQuery(ctx context.Context, binary, schema, fixture string, q endtoen // sample is an expression for a value to bind to a parameter: one of its // column's values in the fixture, or a value of its type when all that is // known is the type. Empty when nothing is known. -func sample(ac endtoend.AnalyzedColumn) string { +func sample(ac analysis.Column) string { if ac.Table != "" && ac.Name != "" { col, tbl := quoteIdent(ac.Name), quoteIdent(ac.Table) return fmt.Sprintf("(SELECT %s FROM %s WHERE %s IS NOT NULL LIMIT 1)", col, tbl, col) @@ -244,15 +245,15 @@ func classes(rows []string, i int) []string { // describeColumn describes a result column: as the table column it is read // from when it is one, otherwise by the storage class of its values, and // nullable when any of its values was NULL. -func describeColumn(cat *catalog, m columnMeta, classes []string) endtoend.AnalyzedColumn { - ac := endtoend.AnalyzedColumn{Name: m.Name} +func describeColumn(cat *catalog, m columnMeta, classes []string) analysis.Column { + ac := analysis.Column{Name: m.Name} if col := cat.lookup(m.Table, m.Origin); col != nil { d := col.describe() ac.Type, ac.Table = d.Type, d.Table } else { for _, class := range classes { if class != "null" { - ac.Type = &endtoend.TypeExpr{Name: class} + ac.Type = &analysis.TypeExpr{Name: class} break } } diff --git a/internal/goldeneye/sqlite/bytecode.go b/internal/goldeneye/sqlite/bytecode.go index 00afae7773..2a0e0dd281 100644 --- a/internal/goldeneye/sqlite/bytecode.go +++ b/internal/goldeneye/sqlite/bytecode.go @@ -4,7 +4,7 @@ import ( "strconv" "strings" - "github.com/sqlc-dev/sqlc/internal/goldeneye/endtoend" + "github.com/sqlc-dev/sqlc/internal/goldeneye/analysis" ) // SQLite has nothing to say about a parameter's type: a bound value is @@ -88,7 +88,7 @@ type tracer struct { regs map[int]*register // found is the column each parameter was found to stand in for, and // hint what the program said about a parameter's type. - found map[int]endtoend.AnalyzedColumn + found map[int]analysis.Column hint map[int]string } @@ -99,7 +99,7 @@ func newTracer(cat *catalog, names []string) *tracer { cursors: map[int]object{}, ephemeral: map[int]map[int][]int{}, regs: map[int]*register{}, - found: map[int]endtoend.AnalyzedColumn{}, + found: map[int]analysis.Column{}, hint: map[int]string{}, } } @@ -117,14 +117,14 @@ func (t *tracer) run(prog []instr) { } // param describes what the parameter was found to stand in for. -func (t *tracer) param(number int) endtoend.AnalyzedColumn { +func (t *tracer) param(number int) analysis.Column { if ac, ok := t.found[number]; ok { return ac } if typ := t.hint[number]; typ != "" { - return endtoend.AnalyzedColumn{Type: &endtoend.TypeExpr{Name: typ}} + return analysis.Column{Type: &analysis.TypeExpr{Name: typ}} } - return endtoend.AnalyzedColumn{} + return analysis.Column{} } func (t *tracer) reg(n int) *register { @@ -167,7 +167,7 @@ func (t *tracer) copy(from, to int) { // assoc records the column the parameters in a register stand in for, // keeping the first found for each. -func (t *tracer) assoc(params []int, ac endtoend.AnalyzedColumn) { +func (t *tracer) assoc(params []int, ac analysis.Column) { for _, p := range params { if _, ok := t.found[p]; !ok { t.found[p] = ac @@ -190,7 +190,7 @@ func (t *tracer) typeHint(n int, typ string) { // describe turns what a register holds into the column a parameter // compared with it stands in for, when it can be named: a stored column // or rowid, a constant's storage class, or a function's name. -func (t *tracer) describe(v value) (endtoend.AnalyzedColumn, bool) { +func (t *tracer) describe(v value) (analysis.Column, bool) { switch v.kind { case vColumn: return t.stored(v.cursor, v.index) @@ -200,20 +200,20 @@ func (t *tracer) describe(v value) (endtoend.AnalyzedColumn, bool) { } case vConstant: if v.class != "null" && v.class != "" { - return endtoend.AnalyzedColumn{Type: &endtoend.TypeExpr{Name: v.class}}, true + return analysis.Column{Type: &analysis.TypeExpr{Name: v.class}}, true } case vFunction: - return endtoend.AnalyzedColumn{Name: v.fn}, true + return analysis.Column{Name: v.fn}, true } - return endtoend.AnalyzedColumn{}, false + return analysis.Column{}, false } // stored describes the ith stored column of a cursor. -func (t *tracer) stored(cursor, i int) (endtoend.AnalyzedColumn, bool) { +func (t *tracer) stored(cursor, i int) (analysis.Column, bool) { col, owner, ok := t.cursors[cursor].column(i) switch { case !ok: - return endtoend.AnalyzedColumn{}, false + return analysis.Column{}, false case col != nil: return col.describe(), true default: @@ -438,7 +438,7 @@ func (t *tracer) step(in instr) { case "ResultRow": for i := 0; i < in.P2; i++ { if i < len(t.names) { - t.assoc(t.params(in.P1+i), endtoend.AnalyzedColumn{Name: t.names[i]}) + t.assoc(t.params(in.P1+i), analysis.Column{Name: t.names[i]}) } } } diff --git a/internal/goldeneye/sqlite/catalog.go b/internal/goldeneye/sqlite/catalog.go index 81809833c2..8077543325 100644 --- a/internal/goldeneye/sqlite/catalog.go +++ b/internal/goldeneye/sqlite/catalog.go @@ -5,7 +5,7 @@ import ( "strconv" "strings" - "github.com/sqlc-dev/sqlc/internal/goldeneye/endtoend" + "github.com/sqlc-dev/sqlc/internal/goldeneye/analysis" ) // catalog is what the database says about a schema: its tables and their @@ -205,19 +205,19 @@ func (t *table) rowidAlias() *tableColumn { // describe is the column as sqlc analyze describes one: its declared type, // nullable unless declared NOT NULL or the rowid, which is never NULL. -func (tc *tableColumn) describe() endtoend.AnalyzedColumn { +func (tc *tableColumn) describe() analysis.Column { typ := parseType(tc.declType) typ.Nullable = !tc.notNull && tc.table.rowidAlias() != tc - return endtoend.AnalyzedColumn{Name: tc.name, Type: typ, Table: tc.table.name} + return analysis.Column{Name: tc.name, Type: typ, Table: tc.table.name} } // rowid describes the table's rowid: the column that aliases it, or the // rowid itself. -func (t *table) rowid() endtoend.AnalyzedColumn { +func (t *table) rowid() analysis.Column { if alias := t.rowidAlias(); alias != nil { return alias.describe() } - return endtoend.AnalyzedColumn{Name: "rowid", Type: &endtoend.TypeExpr{Name: "integer"}, Table: t.name} + return analysis.Column{Name: "rowid", Type: &analysis.TypeExpr{Name: "integer"}, Table: t.name} } // column returns what the object's ith stored column is: a table column, @@ -260,16 +260,16 @@ func (o object) owner() *table { // parseType reads a declared type the way sqlc's catalog does: the name // lowercased, with whatever is in parentheses after it as arguments. A // column declared with no type at all can hold anything. -func parseType(decl string) *endtoend.TypeExpr { +func parseType(decl string) *analysis.TypeExpr { decl = strings.TrimSpace(decl) if decl == "" { - return &endtoend.TypeExpr{Name: "any"} + return &analysis.TypeExpr{Name: "any"} } name, args := decl, "" if open := strings.IndexByte(decl, '('); open >= 0 && strings.HasSuffix(decl, ")") { name, args = decl[:open], decl[open+1:len(decl)-1] } - t := &endtoend.TypeExpr{Name: strings.ToLower(strings.TrimSpace(name))} + t := &analysis.TypeExpr{Name: strings.ToLower(strings.TrimSpace(name))} if strings.TrimSpace(args) == "" { return t } @@ -278,15 +278,15 @@ func parseType(decl string) *endtoend.TypeExpr { switch { case strings.HasPrefix(a, "'") && strings.HasSuffix(a, "'") && len(a) >= 2: s := strings.ReplaceAll(a[1:len(a)-1], "''", "'") - t.Args = append(t.Args, endtoend.TypeArg{String: &s}) + t.Args = append(t.Args, analysis.TypeArg{String: &s}) case strings.EqualFold(a, "true") || strings.EqualFold(a, "false"): b := strings.EqualFold(a, "true") - t.Args = append(t.Args, endtoend.TypeArg{Bool: &b}) + t.Args = append(t.Args, analysis.TypeArg{Bool: &b}) default: if n, err := strconv.ParseInt(a, 10, 64); err == nil { - t.Args = append(t.Args, endtoend.TypeArg{Int: &n}) + t.Args = append(t.Args, analysis.TypeArg{Int: &n}) } else { - t.Args = append(t.Args, endtoend.TypeArg{Type: &endtoend.TypeExpr{Name: strings.ToLower(a)}}) + t.Args = append(t.Args, analysis.TypeArg{Type: &analysis.TypeExpr{Name: strings.ToLower(a)}}) } } } diff --git a/internal/goldeneye/sqlite/install.go b/internal/goldeneye/sqlite/install.go index 56ddc6495c..d09a38013a 100644 --- a/internal/goldeneye/sqlite/install.go +++ b/internal/goldeneye/sqlite/install.go @@ -51,12 +51,12 @@ var extensions = []build{ {"enable_rtree", []string{"SQLITE_ENABLE_RTREE"}}, } -// analysis is the shell the analyze cases run through, which is no +// analysisShell is the shell the analyze cases run through, which is no // dialect build: nothing is generated from it. It is built with column // metadata, so that `.stats stmt` can say which table column each result // column of a statement comes from, and with every extension option at // once, so that whatever a case's schema asks for is there. -var analysis = build{"analysis", append([]string{"SQLITE_ENABLE_COLUMN_METADATA"}, extensionOptions()...)} +var analysisShell = build{"analysis", append([]string{"SQLITE_ENABLE_COLUMN_METADATA"}, extensionOptions()...)} // extensionOptions is every option an extension build turns on, once each. func extensionOptions() []string { @@ -136,7 +136,7 @@ func builds() []build { // shells lists every build Install makes: the dialect builds and the // analysis shell. func shells() []build { - return append(builds(), analysis) + return append(builds(), analysisShell) } // flags are every option a build is compiled with.